diff --git a/src/SIL.Machine.Morphology.HermitCrab/AnalysisAffixTemplateRule.cs b/src/SIL.Machine.Morphology.HermitCrab/AnalysisAffixTemplateRule.cs index 9044a8488..f467f934d 100644 --- a/src/SIL.Machine.Morphology.HermitCrab/AnalysisAffixTemplateRule.cs +++ b/src/SIL.Machine.Morphology.HermitCrab/AnalysisAffixTemplateRule.cs @@ -14,6 +14,7 @@ internal class AnalysisAffixTemplateRule : IRule private readonly Morpher _morpher; private readonly AffixTemplate _template; private readonly List> _rules; + private readonly RequiredSlotMaterial _lookahead; public AnalysisAffixTemplateRule(Morpher morpher, AffixTemplate template) { @@ -26,17 +27,47 @@ public AnalysisAffixTemplateRule(Morpher morpher, AffixTemplate template) FreezableEqualityComparer.Default )) ); + _lookahead = new RequiredSlotMaterial(template); } public IEnumerable Apply(Word input) { - if (!_morpher.RuleSelector(_template)) + if (!CanApply(input)) return Enumerable.Empty(); + int[] fired = null; + if (RequiredSlotCannotUnapply(input, _rules.Count - 1, ref fired)) + { + if (_morpher.TraceManager.IsTracing) + _morpher.TraceManager.EndUnapplyTemplate(_template, input, false, FailureReason.None); + return Enumerable.Empty(); + } + + Word inWord = input.Clone(); + inWord.Freeze(); + + var output = new HashSet(FreezableEqualityComparer.Default); + if (_morpher.MaxDegreeOfParallelism == 1) + ApplySlots(inWord, _rules.Count - 1, output, fired); + else + ParallelApplySlots(inWord, output, fired); + + return output; + } + + internal AffixTemplate Template => _template; + + internal IReadOnlyList> SlotRules => _rules; + + internal bool CanApply(Word input) + { + if (!_morpher.RuleSelector(_template)) + return false; + // A check only: adding the template's required features here would make SyntacticFeatureStruct // depend on whether a template was un-applied, which Word.ValueEquals assumes cannot happen. if (!input.SyntacticFeatureStruct.IsUnifiable(_template.RequiredSyntacticFeatureStruct)) - return Enumerable.Empty(); + return false; if (_morpher.TraceManager.IsTracing) _morpher.TraceManager.BeginUnapplyTemplate(_template, input); @@ -58,27 +89,82 @@ public IEnumerable Apply(Word input) FailureReason.NonPartialRuleProhibitedAfterFinalTemplate ); } - return Enumerable.Empty(); + return false; } - Word inWord = input.Clone(); - inWord.Freeze(); + return true; + } - var output = new HashSet(FreezableEqualityComparer.Default); - if (_morpher.MaxDegreeOfParallelism == 1) - ApplySlots(inWord, _rules.Count - 1, output); - else - ParallelApplySlots(inWord, output); + // Testing runs from the next slot to unapply and stops after an impure slot, which makes the test + // for any slot behind it unsound. Shadow mode prunes nothing and carries the fired slots in fired. + internal bool RequiredSlotCannotUnapply(Word word, int lastSlot, ref int[] fired) + { + RequiredSlotLookaheadMode mode = _morpher.RequiredSlotLookahead; + if (mode == RequiredSlotLookaheadMode.Off) + return false; - return output; + long unifications = 0; + bool cannot = false; + for (int j = lastSlot; j >= 0; j--) + { + if ( + !_template.Slots[j].Optional + && !IsFired(fired, j) + && _lookahead.CannotUnapply(j, word.Shape, ref unifications) + ) + { + if (mode == RequiredSlotLookaheadMode.On) + { + cannot = true; + break; + } + fired = fired == null ? new[] { j } : fired.Append(j).ToArray(); + _morpher.RequiredSlotShadow?.Invoke("fired", _template, j); + } + if (!_lookahead.IsPure(j)) + break; + } + if (unifications != 0) + _morpher.RequiredSlotCheckObserver?.Invoke(unifications); + return cannot; } - private void ApplySlots(Word inWord, int index, HashSet output) + internal static bool IsFired(int[] fired, int slot) + { + return fired != null && Array.IndexOf(fired, slot) >= 0; + } + + internal void ReportShadow(string kind, int[] fired, int slot) + { + if (fired == null || _morpher.RequiredSlotShadow == null) + return; + if (slot < 0) + { + foreach (int j in fired) + _morpher.RequiredSlotShadow(kind, _template, j); + } + else if (IsFired(fired, slot)) + { + _morpher.RequiredSlotShadow(kind, _template, slot); + } + } + + private void ApplySlots(Word inWord, int index, HashSet output, int[] fired) { for (int i = index; i >= 0; i--) { foreach (Word outWord in _rules[i].Apply(inWord)) - ApplySlots(outWord, i - 1, output); + { + ReportShadow("violated", fired, i); + int[] outFired = fired; + if (RequiredSlotCannotUnapply(outWord, i - 1, ref outFired)) + { + if (_morpher.TraceManager.IsTracing) + _morpher.TraceManager.EndUnapplyTemplate(_template, outWord, false, FailureReason.None); + continue; + } + ApplySlots(outWord, i - 1, output, outFired); + } if (!_template.Slots[i].Optional) { @@ -90,16 +176,17 @@ private void ApplySlots(Word inWord, int index, HashSet output) if (_morpher.TraceManager.IsTracing) _morpher.TraceManager.EndUnapplyTemplate(_template, inWord, true, FailureReason.None); + ReportShadow("completed-under-fired", fired, -1); output.Add(inWord); } - private void ParallelApplySlots(Word inWord, HashSet output) + private void ParallelApplySlots(Word inWord, HashSet output, int[] fired) { ParallelOptions parallelOptions = _morpher.CreateParallelOptions(); var outStack = new ConcurrentStack(); - var from = new ConcurrentStack>(); - from.Push(Tuple.Create(inWord, _rules.Count - 1)); - var to = new ConcurrentStack>(); + var from = new ConcurrentStack>(); + from.Push(Tuple.Create(inWord, _rules.Count - 1, fired)); + var to = new ConcurrentStack>(); while (!from.IsEmpty) { to.Clear(); @@ -111,9 +198,25 @@ private void ParallelApplySlots(Word inWord, HashSet output) bool add = true; for (int i = work.Item2; i >= 0; i--) { - Tuple[] workItems = _rules[i] + int next = i - 1; + Tuple[] workItems = _rules[i] .Apply(work.Item1) - .Select(res => Tuple.Create(res, i - 1)) + .Select(res => + { + ReportShadow("violated", work.Item3, i); + int[] resFired = work.Item3; + if (!RequiredSlotCannotUnapply(res, next, ref resFired)) + return Tuple.Create(res, next, resFired); + if (_morpher.TraceManager.IsTracing) + _morpher.TraceManager.EndUnapplyTemplate( + _template, + res, + false, + FailureReason.None + ); + return null; + }) + .Where(item => item != null) .ToArray(); if (workItems.Length > 0) to.PushRange(workItems); @@ -141,11 +244,12 @@ private void ParallelApplySlots(Word inWord, HashSet output) true, FailureReason.None ); + ReportShadow("completed-under-fired", work.Item3, -1); outStack.Push(work.Item1); } } ); - ConcurrentStack> temp = from; + ConcurrentStack> temp = from; from = to; to = temp; } diff --git a/src/SIL.Machine.Morphology.HermitCrab/AnalysisAffixTemplatesRule.cs b/src/SIL.Machine.Morphology.HermitCrab/AnalysisAffixTemplatesRule.cs new file mode 100644 index 000000000..5103c13cd --- /dev/null +++ b/src/SIL.Machine.Morphology.HermitCrab/AnalysisAffixTemplatesRule.cs @@ -0,0 +1,223 @@ +using System.Collections.Generic; +using System.Linq; +using SIL.Machine.Annotations; +using SIL.Machine.Rules; +using SIL.ObjectModel; + +namespace SIL.Machine.Morphology.HermitCrab +{ + internal class AnalysisAffixTemplatesRule : RuleBatch + { + private readonly Morpher _morpher; + private readonly AnalysisAffixTemplateRule[] _templates; + private readonly bool[][] _slotOptionality; + private readonly SlotNode _root = new SlotNode(); + private readonly bool _hasSharedSlots; + + public AnalysisAffixTemplatesRule(Morpher morpher, IEnumerable> rules) + : base(rules, false, FreezableEqualityComparer.Default) + { + _morpher = morpher; + _templates = Rules.OfType().ToArray(); + if (_templates.Length != Rules.Count) + return; + + _root.Templates.AddRange(Enumerable.Range(0, _templates.Length)); + _slotOptionality = _templates.Select(t => t.Template.Slots.Select(s => s.Optional).ToArray()).ToArray(); + for (int t = 0; t < _templates.Length; t++) + { + AnalysisAffixTemplateRule template = _templates[t]; + SlotNode node = _root; + for (int s = template.Template.Slots.Count - 1; s >= 0; s--) + { + AffixTemplateSlot slot = template.Template.Slots[s]; + SlotNode child = node.Children.FirstOrDefault(n => n.Matches(slot)); + if (child == null) + { + child = new SlotNode + { + Slot = slot, + Optional = slot.Optional, + Rule = template.SlotRules[s], + Depth = node.Depth + 1, + }; + node.Children.Add(child); + } + else + { + _hasSharedSlots = true; + } + child.Templates.Add(t); + node = child; + } + node.Terminals.Add(t); + } + } + + public override IEnumerable Apply(Word input) + { + // Use the original walk when no tail is shared, when tracing (per-template Begin/End events), for + // parallel slots, under an alternative limit, or if slot optionality changed after the trie was built. + if ( + !_hasSharedSlots + || _morpher.TraceManager.IsTracing + || _morpher.MaxDegreeOfParallelism != 1 + || MaxAlternatives > 0 + || !SlotsUnchanged() + ) + return base.Apply(input); + + var outputs = new HashSet[_templates.Length]; + for (int t = 0; t < _templates.Length; t++) + { + if (_templates[t].CanApply(input)) + outputs[t] = new HashSet(Comparer); + } + + if (outputs.Any(o => o != null)) + { + Word inWord = input.Clone(); + inWord.Freeze(); + Walk(_root, inWord, new WalkState(_templates, outputs, _morpher), null); + } + + // Template order determines which equal word survives and the order seen by analysis folding. + var output = new HashSet(Comparer); + foreach (HashSet templateOutput in outputs) + { + if (templateOutput != null) + output.UnionWith(templateOutput); + } + return output; + } + + private bool SlotsUnchanged() + { + for (int t = 0; t < _templates.Length; t++) + { + IList slots = _templates[t].Template.Slots; + bool[] optionality = _slotOptionality[t]; + if (slots.Count != optionality.Length) + return false; + for (int s = 0; s < slots.Count; s++) + { + if (slots[s].Optional != optionality[s]) + return false; + } + } + return true; + } + + // A template that fails the look-ahead on a word is dead below it while its siblings go on. + // Marks are set on entry and undone on return, so the mask is one bool per template per call. + private void Walk(SlotNode node, Word input, WalkState state, int[][] fired) + { + List marked = null; + if (state.Mode != RequiredSlotLookaheadMode.Off) + { + foreach (int t in node.Templates) + { + if (state.Outputs[t] == null || state.Dead[t]) + continue; + AnalysisAffixTemplateRule template = _templates[t]; + int[] templateFired = fired?[t]; + if ( + !template.RequiredSlotCannotUnapply( + input, + template.Template.Slots.Count - 1 - node.Depth, + ref templateFired + ) + ) + { + if (!ReferenceEquals(templateFired, fired?[t])) + { + fired = fired == null ? new int[_templates.Length][] : (int[][])fired.Clone(); + fired[t] = templateFired; + } + continue; + } + state.Dead[t] = true; + if (marked == null) + marked = new List(); + marked.Add(t); + } + } + + foreach (int t in node.Terminals) + { + if (state.Outputs[t] == null || state.Dead[t]) + continue; + _templates[t].ReportShadow("completed-under-fired", fired?[t], -1); + state.Outputs[t].Add(input); + } + + foreach (SlotNode child in node.Children) + { + if (!AnyLive(child.Templates, state)) + continue; + + foreach (Word word in child.Rule.Apply(input)) + { + if (fired != null) + { + foreach (int t in child.Templates) + _templates[t] + .ReportShadow("violated", fired[t], _templates[t].SlotRules.Count - child.Depth); + } + Walk(child, word, state, fired); + } + if (child.Optional) + Walk(child, input, state, fired); + } + + if (marked != null) + { + foreach (int t in marked) + state.Dead[t] = false; + } + } + + private static bool AnyLive(List templates, WalkState state) + { + for (int i = 0; i < templates.Count; i++) + { + if (state.Outputs[templates[i]] != null && !state.Dead[templates[i]]) + return true; + } + return false; + } + + private sealed class WalkState + { + public WalkState(AnalysisAffixTemplateRule[] templates, HashSet[] outputs, Morpher morpher) + { + Outputs = outputs; + Dead = new bool[templates.Length]; + Mode = morpher.RequiredSlotLookahead; + } + + public HashSet[] Outputs { get; } + public bool[] Dead { get; } + public RequiredSlotLookaheadMode Mode { get; } + } + + private sealed class SlotNode + { + public AffixTemplateSlot Slot { get; set; } + public bool Optional { get; set; } + public int Depth { get; set; } + public IRule Rule { get; set; } + public List Children { get; } = new List(); + public List Templates { get; } = new List(); + public List Terminals { get; } = new List(); + + public bool Matches(AffixTemplateSlot slot) + { + if (Optional != slot.Optional || Slot.Rules.Count != slot.Rules.Count) + return false; + + return Slot.Rules.Zip(slot.Rules, (a, b) => ReferenceEquals(a, b)).All(equal => equal); + } + } + } +} diff --git a/src/SIL.Machine.Morphology.HermitCrab/AnalysisStratumRule.cs b/src/SIL.Machine.Morphology.HermitCrab/AnalysisStratumRule.cs index ef48637ac..811471d74 100644 --- a/src/SIL.Machine.Morphology.HermitCrab/AnalysisStratumRule.cs +++ b/src/SIL.Machine.Morphology.HermitCrab/AnalysisStratumRule.cs @@ -25,10 +25,9 @@ public AnalysisStratumRule(Morpher morpher, Stratum stratum) _prulesRule = new LinearRuleCascade( stratum.PhonologicalRules.Select(prule => CompilePhonologicalRule(prule, morpher)).Reverse() ); - _templatesRule = new RuleBatch( - stratum.AffixTemplates.Select(template => CompileAffixTemplate(template, morpher)), - false, - FreezableEqualityComparer.Default + _templatesRule = new AnalysisAffixTemplatesRule( + morpher, + stratum.AffixTemplates.Select(template => CompileAffixTemplate(template, morpher)) ); _mrulesRule = null; IEnumerable> mrules = stratum diff --git a/src/SIL.Machine.Morphology.HermitCrab/Morpher.cs b/src/SIL.Machine.Morphology.HermitCrab/Morpher.cs index d0675509e..ca906f91e 100644 --- a/src/SIL.Machine.Morphology.HermitCrab/Morpher.cs +++ b/src/SIL.Machine.Morphology.HermitCrab/Morpher.cs @@ -83,6 +83,26 @@ public ITraceManager TraceManager public int DeletionReapplications { get; set; } + /// + /// Whether affix template walks drop partials in which a required slot can no longer unapply because + /// the material it inserts is absent. See . + /// + internal RequiredSlotLookaheadMode RequiredSlotLookahead { get; set; } = RequiredSlotLookaheadMode.On; + + /// + /// Called in with a kind and the template and slot it is about: + /// "fired" (the test is true for a required slot), "violated" (a rule of a fired slot then unapplied under + /// that partial) and "completed-under-fired" (the template walk completed with a fired slot on its path). + /// Any "violated" or "completed-under-fired" is a soundness failure. + /// + internal Action RequiredSlotShadow { get; set; } + + /// + /// Called after each required-slot check that unified, with the number of unifications it performed. + /// Null unless a diagnostic sets it; it may be called from several threads. + /// + internal Action RequiredSlotCheckObserver { get; set; } + public int MaxStemCount { get; set; } /// diff --git a/src/SIL.Machine.Morphology.HermitCrab/RequiredSlotMaterial.cs b/src/SIL.Machine.Morphology.HermitCrab/RequiredSlotMaterial.cs new file mode 100644 index 000000000..27e60caa6 --- /dev/null +++ b/src/SIL.Machine.Morphology.HermitCrab/RequiredSlotMaterial.cs @@ -0,0 +1,264 @@ +using System.Collections.Generic; +using System.Linq; +using SIL.Machine.Annotations; +using SIL.Machine.FeatureModel; +using SIL.Machine.Matching; +using SIL.Machine.Morphology.HermitCrab.MorphologicalRules; + +namespace SIL.Machine.Morphology.HermitCrab +{ + /// + /// How uses . + /// + internal enum RequiredSlotLookaheadMode + { + /// The look-ahead is not evaluated. + Off, + + /// The look-ahead is evaluated and reported to Morpher.RequiredSlotShadow, never pruning. + Shadow, + + /// The look-ahead prunes template partials. + On, + } + + /// + /// A necessary condition for a required affix template slot to unapply later in a template walk: some + /// allomorph of some rule in the slot must have every inserted segment unify with a segment of the shape. + /// + /// + /// The tables are built once from the template and depend only on the grammar. + /// + /// Soundness. The analysis pattern of an allomorph holds one top-level constraint per inserted segment + /// (, ), + /// and the rule can only unapply if each constraint unifies with a distinct segment node of the shape. + /// A rule is pure when every right-hand side action copies an input part or inserts material, and every + /// left-hand part is copied by a that must capture a segment, or has no + /// segment to restore. Then only copies input + /// nodes and never untruncates a segment. Unification can only narrow the feature structure of a copied + /// node, so a constraint that unifies with no node of the shape unifies with no node of the shape after + /// any number of pure unapplications. Inside one template walk only the rules of that template's slots + /// run, and the stratum's phonological rules ran before the walk. The test for slot j is therefore used + /// only when every rule in the slots still to be walked after j is pure. Optional nodes count as present + /// because the matcher can match them. Anything that cannot be summarised (a null affix, a constraint with + /// variables, an unknown rule type) leaves the slot unprunable. + /// + internal class RequiredSlotMaterial + { + private readonly SlotSummary[] _slots; + + public RequiredSlotMaterial(AffixTemplate template) + { + _slots = template.Slots.Select(Summarize).ToArray(); + } + + /// + /// Whether every rule of the slot is pure, so that unapplying the slot leaves the segments needed by + /// slots before it no easier to find. + /// + public bool IsPure(int slot) + { + return _slots[slot].IsPure; + } + + /// + /// True if no allomorph of the slot can unapply to a word with this shape, provided that every slot + /// walked between this word and the slot is pure. + /// + /// The slot to test. + /// The shape of the partial. + /// Incremented by the feature structure unifications performed. + public bool CannotUnapply(int slot, Shape shape, ref long unifications) + { + SlotSummary summary = _slots[slot]; + if (summary.MayAlwaysUnapply) + return false; + + // The constraints of a slot repeat across its allomorphs, so each distinct one is looked up once. + ulong known = 0; + ulong present = 0; + foreach (int[] requirements in summary.Allomorphs) + { + bool all = true; + foreach (int index in requirements) + { + bool found; + ulong bit = index < 64 ? 1UL << index : 0; + if ((known & bit) != 0) + { + found = (present & bit) != 0; + } + else + { + found = IsPresent(summary.Distinct[index], shape, ref unifications); + known |= bit; + if (found) + present |= bit; + } + if (!found) + { + all = false; + break; + } + } + if (all) + return false; + } + return true; + } + + private static bool IsPresent(FeatureStruct required, Shape shape, ref long unifications) + { + foreach (ShapeNode node in shape) + { + if (node.Type() != HCFeatureSystem.Segment) + continue; + + unifications++; + if (node.Annotation.FeatureStruct.IsUnifiable(required, false, null)) + return true; + } + return false; + } + + private static SlotSummary Summarize(AffixTemplateSlot slot) + { + var summary = new SlotSummary(); + var allomorphs = new List(); + foreach (MorphemicMorphologicalRule rule in slot.Rules) + { + IList ruleAllomorphs; + switch (rule) + { + case AffixProcessRule affixRule: + ruleAllomorphs = affixRule.Allomorphs; + break; + case RealizationalAffixProcessRule realizationalRule: + ruleAllomorphs = realizationalRule.Allomorphs; + break; + default: + summary.MayAlwaysUnapply = true; + summary.IsPure = false; + continue; + } + + foreach (AffixProcessAllomorph allomorph in ruleAllomorphs) + { + if (!IsPure(allomorph)) + summary.IsPure = false; + + FeatureStruct[] requirements = GetRequirements(allomorph); + if (requirements.Length == 0) + summary.MayAlwaysUnapply = true; + else + allomorphs.Add(requirements); + } + } + var distinct = new List(); + summary.Allomorphs = allomorphs + .Select(requirements => + requirements + .Select(fs => + { + int index = distinct.FindIndex(d => d.ValueEquals(fs)); + if (index < 0) + { + distinct.Add(fs); + index = distinct.Count - 1; + } + return index; + }) + .ToArray() + ) + .ToArray(); + summary.Distinct = distinct.ToArray(); + return summary; + } + + private static FeatureStruct[] GetRequirements(AffixProcessAllomorph allomorph) + { + var requirements = new List(); + foreach (MorphologicalOutputAction action in allomorph.Rhs) + { + if (action is InsertSegments insertSegments) + { + foreach (ShapeNode node in insertSegments.Segments.Shape) + { + if (node.Type() != HCFeatureSystem.Boundary) + AddRequirement(requirements, node.Annotation.FeatureStruct); + } + } + else if (action is InsertSimpleContext insertContext) + { + AddRequirement(requirements, insertContext.SimpleContext.FeatureStruct); + } + } + return requirements.ToArray(); + } + + // A constraint with variables is bound by earlier matches, and one that is not a segment is never + // matched against the segments the matcher considers, so neither can be tested here. + private static void AddRequirement(List requirements, FeatureStruct fs) + { + if (fs.HasVariables || (FeatureSymbol)fs.GetValue(HCFeatureSystem.Type) != HCFeatureSystem.Segment) + return; + + FeatureStruct copy = fs.Clone(); + copy.Freeze(); + requirements.Add(copy); + } + + private static bool IsPure(AffixProcessAllomorph allomorph) + { + var copied = new HashSet(); + foreach (MorphologicalOutputAction action in allomorph.Rhs) + { + if (action is CopyFromInput) + copied.Add(action.PartName); + else if (!(action is InsertSegments || action is InsertSimpleContext)) + return false; + } + + foreach (Pattern part in allomorph.Lhs) + { + if (!HasSegmentConstraint(part)) + continue; + if (!copied.Contains(part.Name) || MinSegments(part) == 0) + return false; + } + return true; + } + + private static bool HasSegmentConstraint(PatternNode node) + { + if (node is Constraint constraint) + return constraint.Type() == HCFeatureSystem.Segment; + return node.Children.Any(HasSegmentConstraint); + } + + // The fewest segments a successful match of the node must consume. A part matching none is captured + // as a null range, and GenerateShape then untruncates it, which adds segments that are not copies. + private static int MinSegments(PatternNode node) + { + switch (node) + { + case Constraint constraint: + return constraint.Type() == HCFeatureSystem.Segment ? 1 : 0; + case Quantifier quantifier: + return quantifier.MinOccur * quantifier.Children.Sum(MinSegments); + case Alternation alternation: + return alternation.Children.Count == 0 ? 0 : alternation.Children.Min(MinSegments); + default: + return node.Children.Sum(MinSegments); + } + } + + private class SlotSummary + { + public bool IsPure = true; + public bool MayAlwaysUnapply; + public int[][] Allomorphs; + public FeatureStruct[] Distinct; + } + } +} diff --git a/tests/SIL.Machine.Morphology.HermitCrab.Tests/AnalysisAffixTemplateRuleTests.cs b/tests/SIL.Machine.Morphology.HermitCrab.Tests/AnalysisAffixTemplateRuleTests.cs index ed5cabc36..439f0b4d1 100644 --- a/tests/SIL.Machine.Morphology.HermitCrab.Tests/AnalysisAffixTemplateRuleTests.cs +++ b/tests/SIL.Machine.Morphology.HermitCrab.Tests/AnalysisAffixTemplateRuleTests.cs @@ -1,4 +1,4 @@ -using NUnit.Framework; +using NUnit.Framework; using SIL.Machine.Annotations; using SIL.Machine.FeatureModel; using SIL.Machine.Matching; @@ -69,6 +69,409 @@ public void Apply_SlotLeftEmpty_LeavesTheSameStateAsNotEnteringTheTemplate() }); } + // Slots are listed in synthesis order and unapplied from the last to the first, so slot 0 is the suffix + // closest to the stem and a word ends with the material of its highest slot. + private static AffixTemplateSlot Slot(bool optional, params MorphemicMorphologicalRule[] rules) => + new(rules) { Optional = optional }; + + private static AffixTemplate TemplateOf(params AffixTemplateSlot[] slots) + { + var template = new AffixTemplate { Name = "lookahead_template" }; + foreach (AffixTemplateSlot slot in slots) + template.Slots.Add(slot); + return template; + } + + // The analysis of a rule that deletes a final segment in synthesis: it adds the segment back, so the + // unapplication produces a shape that is not made of copies of the input. + private AffixProcessRule SegmentRestoringRule(string name, string restored) + { + var rule = new AffixProcessRule + { + Id = name, + Name = name, + Gloss = name, + RequiredSyntacticFeatureStruct = FeatureStruct.New().Value, + OutSyntacticFeatureStruct = FeatureStruct.New().Value, + }; + rule.Allomorphs.Add( + new AffixProcessAllomorph + { + Lhs = + { + Pattern.New("1").Annotation(Any).OneOrMore.Value, + Pattern.New("2").Annotation(Character(Table3, restored)).Value, + }, + Rhs = { new CopyFromInput("1") }, + } + ); + return rule; + } + + private AffixProcessRule SuffixRule(string name, string insert) => + Suffix(name, FeatureStruct.New().Value, FeatureStruct.New().Value, Table3, insert); + + private Word Input(string form, bool lastNodeOptional = false) + { + var input = new Word(Morphophonemic, Morphophonemic.CharacterDefinitionTable.Segment(form)); + if (lastNodeOptional) + input.Shape.Last.Annotation.Optional = true; + return input; + } + + private sealed record WalkResult(List Parses, int RuleAttempts); + + private WalkResult Walk( + AffixTemplate template, + Word input, + RequiredSlotLookaheadMode mode, + int maxDegreeOfParallelism = 1, + bool tracing = false + ) + { + int attempts = 0; + var morpher = new Morpher(TraceManager, Language, maxDegreeOfParallelism) + { + RequiredSlotLookahead = mode, + RuleSelector = rule => + { + if (rule is AffixProcessRule) + System.Threading.Interlocked.Increment(ref attempts); + return true; + }, + }; + TraceManager.IsTracing = tracing; + if (tracing) + TraceManager.AnalyzeWord(Language, input); + input.Freeze(); + IRule rule = template.CompileAnalysisRule(morpher); + List parses = rule.Apply(input) + .Select(w => w.ToString() + " " + string.Join(",", w.AllomorphsInMorphOrder.Select(a => a.Morpheme.Gloss))) + .OrderBy(p => p, System.StringComparer.Ordinal) + .ToList(); + return new WalkResult(parses, attempts); + } + + private (WalkResult Off, WalkResult On) WalkBoth(AffixTemplate template, string form, bool lastNodeOptional = false) + { + WalkResult off = Walk(template, Input(form, lastNodeOptional), RequiredSlotLookaheadMode.Off); + WalkResult on = Walk(template, Input(form, lastNodeOptional), RequiredSlotLookaheadMode.On); + Assert.That(on.Parses, Is.EqualTo(off.Parses), "the look-ahead must not change the parses"); + return (off, on); + } + + // stem + required -ti + optional -u/-f/-v + optional -z + private AffixTemplate RequiredInnerSuffixTemplate() => + TemplateOf( + Slot(false, SuffixRule("ti", "ti")), + Slot(true, SuffixRule("u", "u"), SuffixRule("f", "f"), SuffixRule("v", "v")), + Slot(true, SuffixRule("z", "z")) + ); + + [Test] + public void Apply_RequiredSlotLiteralAbsent_PrunesBeforeTryingTheOptionalSlots() + { + AffixTemplate template = RequiredInnerSuffixTemplate(); + + (WalkResult off, WalkResult on) = WalkBoth(template, "sagufz"); + + Assert.Multiple(() => + { + Assert.That(off.Parses, Is.Empty); + Assert.That(off.RuleAttempts, Is.GreaterThan(0)); + Assert.That(on.RuleAttempts, Is.EqualTo(0), "no rule can supply -ti, so no slot rule is attempted"); + }); + } + + [Test] + public void Apply_RequiredSlotLiteralPresent_DoesNotPrune() + { + AffixTemplate template = RequiredInnerSuffixTemplate(); + + (WalkResult off, WalkResult on) = WalkBoth(template, "sagtiuz"); + + Assert.Multiple(() => + { + Assert.That(off.Parses, Is.Not.Empty); + Assert.That(on.RuleAttempts, Is.EqualTo(off.RuleAttempts)); + }); + } + + [Test] + public void Apply_RequiredSlotLiteralLostAfterAnOptionalSlot_PrunesTheInnerPartial() + { + AffixTemplate template = RequiredInnerSuffixTemplate(); + + // -z unapplies, which leaves "sagtuu"; no node can still be the i of -ti + (WalkResult off, WalkResult on) = WalkBoth(template, "sagttz"); + + Assert.Multiple(() => + { + Assert.That(off.Parses, Is.Empty); + Assert.That(on.RuleAttempts, Is.LessThan(off.RuleAttempts)); + }); + } + + [Test] + public void Apply_ImpureSlotBetween_DoesNotPruneForTheInnerRequiredSlot() + { + // the middle slot restores the i that -ti needs, so "sagt" is a stem plus -ti once it has run + AffixTemplate template = TemplateOf( + Slot(false, SuffixRule("ti", "ti")), + Slot(true, SegmentRestoringRule("restore_i", "i")) + ); + + (WalkResult off, WalkResult on) = WalkBoth(template, "sagt"); + + Assert.That(on.Parses, Is.Not.Empty, "the parse that needs the restored segment must survive"); + Assert.That(on.RuleAttempts, Is.EqualTo(off.RuleAttempts)); + } + + [Test] + public void Apply_SeveralRequiredSlots_PrunesWhicheverLiteralIsAbsent() + { + // -u is required as well as -ti; the word has neither + AffixTemplate template = TemplateOf( + Slot(false, SuffixRule("ti", "ti")), + Slot(true, SuffixRule("z", "z")), + Slot(false, SuffixRule("u", "u")) + ); + + (WalkResult off, WalkResult on) = WalkBoth(template, "sagfz"); + + Assert.That(on.RuleAttempts, Is.EqualTo(0)); + Assert.That(off.RuleAttempts, Is.GreaterThan(0)); + } + + [Test] + public void Apply_OptionalNodeThatUnifiesWithTheLiteral_CountsAsPresent() + { + AffixTemplate template = TemplateOf(Slot(false, SuffixRule("ti", "ti"))); + + // "sagti" where the i stands for a deleted segment: the matcher may match or skip an optional node + (WalkResult off, WalkResult on) = WalkBoth(template, "sagti", lastNodeOptional: true); + + Assert.Multiple(() => + { + Assert.That(off.Parses, Is.Not.Empty); + Assert.That(on.Parses, Is.EqualTo(off.Parses)); + }); + } + + [Test] + public void Apply_RequiredSlotWithNullAffix_IsNeverPruned() + { + AffixProcessRule withNull = SuffixRule("ti_or_null", "ti"); + withNull.Allomorphs.Add( + new AffixProcessAllomorph + { + Lhs = { Pattern.New("1").Annotation(Any).OneOrMore.Value }, + Rhs = { new CopyFromInput("1") }, + } + ); + AffixTemplate template = TemplateOf(Slot(false, withNull), Slot(true, SuffixRule("z", "z"))); + + (WalkResult off, WalkResult on) = WalkBoth(template, "sagz"); + + Assert.Multiple(() => + { + Assert.That(on.Parses, Is.Not.Empty); + Assert.That(on.RuleAttempts, Is.EqualTo(off.RuleAttempts)); + }); + } + + // The word lacks the t and i that the required inner slot inserts, so a pure optional rule that inserts "z" + // lets the walk prune at entry, and an impure one does not. + private (WalkResult Off, WalkResult On) WalkWithOptionalSlot( + Pattern[] lhs, + MorphologicalOutputAction[] rhs + ) + { + var rule = new AffixProcessRule + { + Id = "z_slot", + Name = "z_slot", + Gloss = "z_slot", + RequiredSyntacticFeatureStruct = FeatureStruct.New().Value, + OutSyntacticFeatureStruct = FeatureStruct.New().Value, + }; + var allomorph = new AffixProcessAllomorph(); + foreach (Pattern part in lhs) + allomorph.Lhs.Add(part); + foreach (MorphologicalOutputAction action in rhs) + allomorph.Rhs.Add(action); + allomorph.Rhs.Add(new InsertSegments(Table3, "z")); + rule.Allomorphs.Add(allomorph); + AffixTemplate template = TemplateOf(Slot(false, SuffixRule("ti", "ti")), Slot(true, rule)); + return WalkBoth(template, "sagz"); + } + + [Test] + public void Apply_CopiedPartThatMayMatchNothing_MakesTheSlotImpure() + { + (WalkResult off, WalkResult on) = WalkWithOptionalSlot( + new[] { Pattern.New("1").Annotation(Any).ZeroOrMore.Value }, + new MorphologicalOutputAction[] { new CopyFromInput("1") } + ); + + Assert.That(on.RuleAttempts, Is.GreaterThan(0), "the optional rule is still tried at entry"); + } + + [Test] + public void Apply_ModifyFromInputInASlot_MakesTheSlotImpure() + { + FeatureStruct voiced = FeatureStruct + .New(Language.PhonologicalFeatureSystem) + .Symbol(HCFeatureSystem.Segment) + .Symbol("vd+") + .Value; + + (WalkResult off, WalkResult on) = WalkWithOptionalSlot( + new[] { Pattern.New("1").Annotation(Any).OneOrMore.Value }, + new MorphologicalOutputAction[] + { + new ModifyFromInput("1", voiced, System.Array.Empty()), + } + ); + + Assert.That(on.RuleAttempts, Is.GreaterThan(0), "the optional rule is still tried at entry"); + } + + [Test] + public void Apply_CopiedAlternationOfSegments_KeepsTheSlotPure() + { + (WalkResult off, WalkResult on) = WalkWithOptionalSlot( + new[] + { + Pattern.New("1").Annotation(Any).OneOrMore.Value, + Pattern + .New("2") + .Annotation(Character(Table3, "i")) + .Or.Annotation(Character(Table3, "u")) + .Value, + }, + new MorphologicalOutputAction[] { new CopyFromInput("1"), new CopyFromInput("2") } + ); + + Assert.That(off.RuleAttempts, Is.GreaterThan(0)); + Assert.That(on.RuleAttempts, Is.EqualTo(0)); + } + + [Test] + public void Apply_CopiedAlternationWithAnOptionalBranch_MakesTheSlotImpure() + { + (WalkResult off, WalkResult on) = WalkWithOptionalSlot( + new[] + { + Pattern.New("1").Annotation(Any).OneOrMore.Value, + Pattern + .New("2") + .Annotation(Character(Table3, "i")) + .Optional.Or.Annotation(Character(Table3, "u")) + .Value, + }, + new MorphologicalOutputAction[] { new CopyFromInput("1"), new CopyFromInput("2") } + ); + + Assert.That(on.RuleAttempts, Is.GreaterThan(0), "the optional rule is still tried at entry"); + } + + [Test] + public void Apply_SlotOptionalityChangedAfterCompiling_IsReadLive() + { + AffixTemplate template = RequiredInnerSuffixTemplate(); + var morpher = new Morpher(TraceManager, Language, 1) { RequiredSlotLookahead = RequiredSlotLookaheadMode.On }; + IRule rule = template.CompileAnalysisRule(morpher); + Word input = Input("sagu"); + input.Freeze(); + + Assert.That(rule.Apply(input), Is.Empty, "slot 0 is required and -ti is absent"); + + template.Slots[0].Optional = true; + Assert.That(rule.Apply(input), Is.Not.Empty, "an optional slot is not tested"); + + template.Slots[0].Optional = false; + Assert.That(rule.Apply(input), Is.Empty); + } + + [TestCase(0)] + [TestCase(4)] + public void Apply_Parallel_ReturnsTheSameParsesAsSequential(int maxDegreeOfParallelism) + { + AffixTemplate template = RequiredInnerSuffixTemplate(); + + foreach (string form in new[] { "sagtiuz", "sagufz", "sagttz", "sagtif" }) + { + WalkResult sequential = Walk(template, Input(form), RequiredSlotLookaheadMode.On); + WalkResult parallel = Walk(template, Input(form), RequiredSlotLookaheadMode.On, maxDegreeOfParallelism); + WalkResult parallelOff = Walk(template, Input(form), RequiredSlotLookaheadMode.Off, maxDegreeOfParallelism); + Assert.That(parallel.Parses, Is.EqualTo(sequential.Parses), form); + Assert.That(parallel.Parses, Is.EqualTo(parallelOff.Parses), form); + } + } + + [Test] + public void Apply_Tracing_ShowsAPrunedTemplateAsAFailedOutput() + { + AffixTemplate template = RequiredInnerSuffixTemplate(); + Word input = Input("sagufz"); + + Walk(template, input, RequiredSlotLookaheadMode.On, tracing: true); + TraceManager.IsTracing = false; + + var root = (Trace)input.CurrentTrace; + Trace[] children = root.Children.ToArray(); + Assert.Multiple(() => + { + Assert.That( + children.Select(t => t.Type), + Is.EqualTo(new[] { TraceType.TemplateAnalysisInput, TraceType.TemplateAnalysisOutput }) + ); + Assert.That(children[1].Output, Is.Null, "the partial is reported as not unapplied"); + }); + } + + [TestCase(1)] + [TestCase(0)] + public void Apply_Shadow_ReportsFiredPointsAndNeverViolatesOrChangesTheParses(int maxDegreeOfParallelism) + { + AffixTemplate template = RequiredInnerSuffixTemplate(); + var events = new List(); + + foreach (string form in new[] { "sagtiuz", "sagufz", "sagttz", "sagtif", "sagu" }) + { + WalkResult off = Walk(template, Input(form), RequiredSlotLookaheadMode.Off, maxDegreeOfParallelism); + Word input = Input(form); + input.Freeze(); + var morpher = new Morpher(TraceManager, Language, maxDegreeOfParallelism) + { + RequiredSlotLookahead = RequiredSlotLookaheadMode.Shadow, + RequiredSlotShadow = (kind, t, slot) => + { + lock (events) + events.Add(kind); + }, + }; + TraceManager.IsTracing = false; + string[] parses = template + .CompileAnalysisRule(morpher) + .Apply(input) + .Select(w => + w.ToString() + " " + string.Join(",", w.AllomorphsInMorphOrder.Select(a => a.Morpheme.Gloss)) + ) + .OrderBy(p => p, System.StringComparer.Ordinal) + .ToArray(); + Assert.That(parses, Is.EqualTo(off.Parses), form); + } + + Assert.Multiple(() => + { + Assert.That(events.Count(e => e == "fired"), Is.GreaterThan(0)); + Assert.That(events.Count(e => e == "violated"), Is.EqualTo(0)); + Assert.That(events.Count(e => e == "completed-under-fired"), Is.EqualTo(0)); + }); + } + private AffixTemplate VerbTemplate(FeatureStruct required) { var template = new AffixTemplate { Name = "verb_template", RequiredSyntacticFeatureStruct = required }; diff --git a/tests/SIL.Machine.Morphology.HermitCrab.Tests/AnalysisAffixTemplatesRuleTests.cs b/tests/SIL.Machine.Morphology.HermitCrab.Tests/AnalysisAffixTemplatesRuleTests.cs new file mode 100644 index 000000000..d3b1c82a9 --- /dev/null +++ b/tests/SIL.Machine.Morphology.HermitCrab.Tests/AnalysisAffixTemplatesRuleTests.cs @@ -0,0 +1,373 @@ +using NUnit.Framework; +using SIL.Machine.Annotations; +using SIL.Machine.FeatureModel; +using SIL.Machine.Matching; +using SIL.Machine.Morphology.HermitCrab.MorphologicalRules; +using SIL.Machine.Rules; +using SIL.ObjectModel; + +namespace SIL.Machine.Morphology.HermitCrab; + +[TestFixture] +public class AnalysisAffixTemplatesRuleTests : HermitCrabTestBase +{ + [TestCase(0)] + [TestCase(1)] + [TestCase(2)] + [TestCase(3)] + public void SharedWalkMatchesPerTemplateWalkInOrder(int variant) + { + var d = Suffix("d", "d"); + var dd = Suffix("dd", "dd"); + var s = Suffix("s", "s"); + AffixTemplateSlot Tail(bool optional = false) => new(d, dd) { Optional = optional }; + AffixTemplateSlot Inner() => new(s) { Optional = (variant & 1) == 0 }; + AffixTemplate[] templates = + [ + Template(Inner(), Tail()), + Template(Tail()), + Template(Inner(), Tail(true)), + Template(new AffixTemplateSlot(d) { Optional = true }, Inner(), Tail()), + Template(new AffixTemplateSlot(s), Tail()), + Template(), + Template(Inner(), Tail()), + Template(new AffixTemplateSlot(dd, d)), + Template(new AffixTemplateSlot()), + ]; + templates[4].RequiredSyntacticFeatureStruct = Pos("N"); + templates[3].IsFinal = false; + if ((variant & 2) != 0) + Array.Reverse(templates); + + var morpher = new Morpher(TraceManager, Language, maxDegreeOfParallelism: 1) + { + RuleSelector = rule => variant != 3 || !ReferenceEquals(rule, templates[1]), + }; + IRule[] rules = templates.Select(t => t.CompileAnalysisRule(morpher)).ToArray(); + var shared = new AnalysisAffixTemplatesRule(morpher, rules); + var own = new RuleBatch(rules, false, FreezableEqualityComparer.Default); + + int compared = 0; + foreach (string text in Inputs(4)) + { + foreach (FeatureStruct fs in new[] { FeatureStruct.New().Value, Pos("N"), Pos("V") }) + { + foreach ( + FinalTemplateState finalState in new[] { FinalTemplateState.None, FinalTemplateState.NonTemplate } + ) + { + Word input = Input(text); + input.SyntacticFeatureStruct = fs; + input.FinalTemplateState = finalState; + input.Freeze(); + + Word[] expected = own.Apply(input).ToArray(); + Word[] actual = shared.Apply(input).ToArray(); + Assert.That(actual, Is.EqualTo(expected).Using(FreezableEqualityComparer.Default), text); + Assert.That( + actual.Select(w => new AnalysisStratumRule.AnalysisMergeKey(w)), + Is.EqualTo(expected.Select(w => new AnalysisStratumRule.AnalysisMergeKey(w))), + text + ); + compared += actual.Length; + } + } + } + Assert.That(compared, Is.GreaterThan(1000)); + } + + [Test] + public void UntracedBatteryRunsSharedSlotBatchOnce() + { + var d = Suffix("d", "d"); + var morpher = new Morpher(TraceManager, Language, maxDegreeOfParallelism: 1); + IRule[] rules = new[] + { + Template(new AffixTemplateSlot(d)), + Template(new AffixTemplateSlot(d)), + } + .Select(t => t.CompileAnalysisRule(morpher)) + .ToArray(); + var own = new RuleBatch(rules, false, FreezableEqualityComparer.Default); + var shared = new AnalysisAffixTemplatesRule(morpher, rules); + Word input = Input("sagd"); + input.Freeze(); + + Word[] expected = own.Apply(input).ToArray(); + Assert.That(d.ApplyCount, Is.EqualTo(2)); + d.ApplyCount = 0; + Assert.That(shared.Apply(input), Is.EqualTo(expected).Using(FreezableEqualityComparer.Default)); + Assert.That(d.ApplyCount, Is.EqualTo(1), "two identical tails must execute one slot batch"); + _ = shared.Apply(input).ToArray(); + Assert.That(d.ApplyCount, Is.EqualTo(2), "each battery call must execute its own batch"); + } + + [TestCase(false, 2)] + [TestCase(true, 1)] + public void TracedAndParallelBatteriesKeepPerTemplateWalk(bool tracing, int degree) + { + var d = Suffix("d", "d"); + var traceManager = new TraceManager { IsTracing = tracing }; + var morpher = new Morpher(traceManager, Language, maxDegreeOfParallelism: degree); + var shared = new AnalysisAffixTemplatesRule( + morpher, + new[] { Template(new AffixTemplateSlot(d)), Template(new AffixTemplateSlot(d)) }.Select(t => + t.CompileAnalysisRule(morpher) + ) + ); + Word input = Input("sagd"); + input.Freeze(); + if (tracing) + traceManager.AnalyzeWord(Language, input); + _ = shared.Apply(input).ToArray(); + Assert.That(d.ApplyCount, Is.EqualTo(2)); + } + + [Test] + public void SharedWalkPreservesBatteryAlternativeLimit() + { + var d = Suffix("d", "d"); + var morpher = new Morpher(TraceManager, Language, maxDegreeOfParallelism: 1); + IRule[] rules = new[] + { + Template(new AffixTemplateSlot(d) { Optional = true }), + Template(new AffixTemplateSlot(d) { Optional = true }), + } + .Select(t => t.CompileAnalysisRule(morpher)) + .ToArray(); + var shared = new AnalysisAffixTemplatesRule(morpher, rules) { MaxAlternatives = 1 }; + var own = new RuleBatch(rules, false, FreezableEqualityComparer.Default) + { + MaxAlternatives = 1, + }; + Word input = Input("sagd"); + input.Freeze(); + Assert.Throws(() => own.Apply(input).ToArray()); + Assert.Throws(() => shared.Apply(input).ToArray()); + } + + [Test] + public void AlternativeLimitStopsBeforeLaterTemplateAdmission() + { + var d = Suffix("d", "d"); + AffixTemplate[] templates = + [ + Template(new AffixTemplateSlot(d) { Optional = true }), + Template(new AffixTemplateSlot(d) { Optional = true }), + Template(new AffixTemplateSlot(Suffix("s", "s"))), + ]; + var morpher = new Morpher(TraceManager, Language, maxDegreeOfParallelism: 1) + { + RuleSelector = rule => + !ReferenceEquals(rule, templates[2]) + ? true + : throw new InvalidOperationException("later template must not be reached"), + }; + IRule[] rules = templates.Select(t => t.CompileAnalysisRule(morpher)).ToArray(); + var own = new RuleBatch(rules, false, FreezableEqualityComparer.Default) + { + MaxAlternatives = 1, + }; + var shared = new AnalysisAffixTemplatesRule(morpher, rules) { MaxAlternatives = 1 }; + Word input = Input("sagd"); + input.Freeze(); + + Assert.Throws(() => own.Apply(input).ToArray()); + Assert.Throws(() => shared.Apply(input).ToArray()); + } + + [TestCase(false, 1)] + [TestCase(true, 0)] + public void ChangedSlotOptionalityPreservesPerTemplateWalk(bool optional, int changedTemplate) + { + var d = Suffix("d", "d"); + AffixTemplate[] templates = + [ + Template(new AffixTemplateSlot(d) { Optional = optional }), + Template(new AffixTemplateSlot(d) { Optional = optional }), + ]; + var morpher = new Morpher(TraceManager, Language, maxDegreeOfParallelism: 1); + IRule[] rules = templates.Select(t => t.CompileAnalysisRule(morpher)).ToArray(); + var own = new RuleBatch(rules, false, FreezableEqualityComparer.Default); + var shared = new AnalysisAffixTemplatesRule(morpher, rules); + templates[changedTemplate].Slots[0].Optional = !optional; + Word input = Input("sag"); + input.Freeze(); + + Assert.That(shared.Apply(input), Is.EqualTo(own.Apply(input)).Using(FreezableEqualityComparer.Default)); + } + + private sealed record LookaheadRun(string[] Output, int InnerA, int InnerS, int Tail, List Shadow); + + // Two templates share the tail slot "ad". Template A needs an inner "a" suffix, which the tail's unapplication + // consumes from "sad"; template B has an optional inner "s" slot and still yields its outputs. + private LookaheadRun RunSharedTail(RequiredSlotLookaheadMode mode, string text, bool innerAIsOptional = false) + { + CountingSuffix a = Suffix("a", "a"); + CountingSuffix s = Suffix("s", "s"); + CountingSuffix tail = Suffix("ad", "ad"); + var shadow = new List(); + var morpher = new Morpher(TraceManager, Language, maxDegreeOfParallelism: 1) + { + RequiredSlotLookahead = mode, + RequiredSlotShadow = (kind, template, slot) => shadow.Add(kind), + }; + IRule[] rules = new[] + { + Template(new AffixTemplateSlot(a) { Optional = innerAIsOptional }, new AffixTemplateSlot(tail)), + Template(new AffixTemplateSlot(s) { Optional = true }, new AffixTemplateSlot(tail)), + } + .Select(t => t.CompileAnalysisRule(morpher)) + .ToArray(); + var shared = new AnalysisAffixTemplatesRule(morpher, rules); + Word input = Input(text); + input.Freeze(); + string[] output = shared.Apply(input).Select(Signature).ToArray(); + return new LookaheadRun(output, a.ApplyCount, s.ApplyCount, tail.ApplyCount, shadow); + } + + [Test] + public void LookaheadPrunesOnlyTheTemplateWhoseRequiredSlotIsGone() + { + LookaheadRun off = RunSharedTail(RequiredSlotLookaheadMode.Off, "sad"); + LookaheadRun on = RunSharedTail(RequiredSlotLookaheadMode.On, "sad"); + + Assert.That(off.InnerA, Is.EqualTo(1)); + Assert.That(on.InnerA, Is.EqualTo(0), "the required inner suffix cannot unapply once the tail took the a"); + Assert.That(on.Tail, Is.EqualTo(off.Tail), "the shared tail still runs once"); + Assert.That(on.InnerS, Is.EqualTo(off.InnerS), "the other template is unaffected"); + Assert.That(off.Output, Is.Not.Empty); + Assert.That(on.Output, Is.EqualTo(off.Output)); + } + + [Test] + public void LookaheadKeepsEveryAttemptWhenBothTemplatesAreFeasible() + { + LookaheadRun off = RunSharedTail(RequiredSlotLookaheadMode.Off, "saad"); + LookaheadRun on = RunSharedTail(RequiredSlotLookaheadMode.On, "saad"); + + Assert.That((on.InnerA, on.InnerS, on.Tail), Is.EqualTo((off.InnerA, off.InnerS, off.Tail))); + Assert.That(on.Output, Is.EqualTo(off.Output)); + } + + [TestCase("sad", false)] + [TestCase("saad", false)] + [TestCase("sad", true)] + [TestCase("dd", false)] + public void ShadowLookaheadOnTheSharedWalkNeverViolatesOrChangesTheOutput(string text, bool optional) + { + LookaheadRun off = RunSharedTail(RequiredSlotLookaheadMode.Off, text, optional); + LookaheadRun shadow = RunSharedTail(RequiredSlotLookaheadMode.Shadow, text, optional); + + Assert.That(shadow.Output, Is.EqualTo(off.Output)); + Assert.That((shadow.InnerA, shadow.InnerS, shadow.Tail), Is.EqualTo((off.InnerA, off.InnerS, off.Tail))); + Assert.That(shadow.Shadow, Does.Not.Contain("violated")); + Assert.That(shadow.Shadow, Does.Not.Contain("completed-under-fired")); + if (text == "sad" && !optional) + Assert.That(shadow.Shadow, Does.Contain("fired")); + } + + [Test] + public void LookaheadSharedWalkMatchesPerTemplateWalkOnEveryShortInput() + { + int compared = 0; + foreach (string text in Inputs(5)) + { + foreach ( + RequiredSlotLookaheadMode mode in new[] { RequiredSlotLookaheadMode.Off, RequiredSlotLookaheadMode.On } + ) + { + CountingSuffix a = Suffix("a", "a"); + CountingSuffix s = Suffix("s", "s"); + CountingSuffix tail = Suffix("ad", "ad"); + var morpher = new Morpher(TraceManager, Language, maxDegreeOfParallelism: 1) + { + RequiredSlotLookahead = mode, + }; + IRule[] rules = new[] + { + Template(new AffixTemplateSlot(a), new AffixTemplateSlot(tail)), + Template( + new AffixTemplateSlot(s) { Optional = true }, + new AffixTemplateSlot(a), + new AffixTemplateSlot(tail) + ), + Template(new AffixTemplateSlot(s), new AffixTemplateSlot(tail)), + } + .Select(t => t.CompileAnalysisRule(morpher)) + .ToArray(); + var own = new RuleBatch(rules, false, FreezableEqualityComparer.Default); + var shared = new AnalysisAffixTemplatesRule(morpher, rules); + Word input = Input(text); + input.Freeze(); + Word[] expected = own.Apply(input).ToArray(); + Word[] actual = shared.Apply(input).ToArray(); + Assert.That(actual, Is.EqualTo(expected).Using(FreezableEqualityComparer.Default), text); + compared += actual.Length; + } + } + Assert.That(compared, Is.GreaterThan(10)); + } + + private static string Signature(Word w) => w + " " + string.Join(",", w.MorphologicalRules.Select(r => r.Name)); + + private Word Input(string text) => new(Morphophonemic, Table3.Segment(text)); + + private FeatureStruct Pos(string symbol) => FeatureStruct.New(Language.SyntacticFeatureSystem).Symbol(symbol).Value; + + private static AffixTemplate Template(params AffixTemplateSlot[] slots) + { + var template = new AffixTemplate(); + foreach (AffixTemplateSlot slot in slots) + template.Slots.Add(slot); + return template; + } + + private CountingSuffix Suffix(string name, string suffix) + { + var rule = new CountingSuffix { Name = name, MaxApplicationCount = 3 }; + rule.Allomorphs.Add( + new AffixProcessAllomorph + { + Lhs = + { + Pattern + .New("stem") + .Annotation(FeatureStruct.New().Symbol(HCFeatureSystem.Segment).Value) + .OneOrMore.Value, + }, + Rhs = { new CopyFromInput("stem"), new InsertSegments(Table3, suffix) }, + } + ); + return rule; + } + + private static IEnumerable Inputs(int maxLength) + { + string[] frontier = [""]; + yield return ""; + for (int length = 1; length <= maxLength; length++) + { + frontier = frontier.SelectMany(w => new[] { "s", "a", "d" }.Select(c => w + c)).ToArray(); + foreach (string word in frontier) + yield return word; + } + } + + private sealed class CountingSuffix : AffixProcessRule + { + public int ApplyCount; + + public override IRule CompileAnalysisRule(Morpher morpher) => + new CountingRule(this, base.CompileAnalysisRule(morpher)); + + private sealed class CountingRule(CountingSuffix owner, IRule rule) : IRule + { + public IEnumerable Apply(Word input) + { + Interlocked.Increment(ref owner.ApplyCount); + return rule.Apply(input); + } + } + } +}