Prune analysis rule outputs that contain no root when no later rule can restore stem material

src/SIL.Machine.Morphology.HermitCrab/Morpher.cs

@@ -25,6 +25,7 @@ public class Morpher : IMorphologicalAnalyzer, IMorphologicalGenerator
         private readonly ITraceManager _traceManager;
         private readonly ReadOnlyObservableCollection<Morpheme> _morphemes;
         private readonly IList<RootAllomorph> _lexicalPatterns = new List<RootAllomorph>();
+        private readonly Dictionary<Stratum, RootSubstringPruneStratum> _rootSubstringPrune;
 
         public Morpher(ITraceManager traceManager, Language lang, int maxDegreeOfParallelism = 0)
         {
@@ -52,6 +53,7 @@ public Morpher(ITraceManager traceManager, Language lang, int maxDegreeOfParalle
                 morphemes.AddRange(stratum.MorphologicalRules.OfType<AffixProcessRule>());
                 morphemes.AddRange(stratum.AffixTemplates.SelectMany(t => t.Slots).SelectMany(s => s.Rules).Distinct());
             }
+            _rootSubstringPrune = RootSubstringPruneStratum.Compute(lang, _allomorphTries, _lexicalPatterns.Count > 0);
             _analysisRule = lang.CompileAnalysisRule(this);
             _synthesisRule = lang.CompileSynthesisRule(this);
             MaxStemCount = 2;
@@ -424,6 +426,113 @@ internal IEnumerable<RootAllomorph> SearchRootAllomorphs(Stratum stratum, Shape
             return alloSearcher.Search(shape).Distinct();
         }
 
+        /// <summary>
+        /// Controls the root-substring prune of <see cref="MayContainRoot"/>. Shadow evaluates the predicate and
+        /// reports through <see cref="RootSubstringShadow"/> without pruning.
+        /// </summary>
+        internal RootSubstringPruneMode RootSubstringPrune { get; set; } = RootSubstringPruneMode.On;
+
+        /// <summary>Called in shadow mode with the rule output and the rule type that would have been pruned.</summary>
+        internal Action<Word, string> RootSubstringShadow { get; set; }
+
+        /// <summary>
+        /// Diagnostic hook, called once per trie transduction. Null in production, so the hot path keeps no shared
+        /// counter.
+        /// </summary>
+        internal Action RootSubstringProbed { get; set; }
+
+        internal IReadOnlyDictionary<Stratum, RootSubstringPruneStratum> RootSubstringPruneStrata =>
+            _rootSubstringPrune;
+
+        /// <summary>
+        /// Whether an analysis rule output may still lead to a lexical hit. False only when the prune is on for
+        /// the output's stratum and no root occurs as a contiguous run of its segments. The probe is skipped when
+        /// the output kept at least as many segments as its input in the same stratum: the rule then removed no
+        /// segment-bearing material, so under the flag's copy conditions every root run of the input is still a
+        /// run of the output. Skipping only ever keeps a word, so it cannot lose a parse.
+        /// </summary>
+        internal bool MayContainRoot(Word input, Word word, string site)
+        {
+            if (
+                RootSubstringPrune == RootSubstringPruneMode.Off
+                || word.Stratum == null
+                || !_rootSubstringPrune.TryGetValue(word.Stratum, out RootSubstringPruneStratum info)
+                || !info.Enabled
+            )
+            {
+                return true;
+            }
+
+            bool sameStratum = input.Stratum == word.Stratum;
+            int outputSegments = SegmentCount(word);
+            if (sameStratum && outputSegments >= SegmentCount(input))
+            {
+                if (word.RootWindowStart == 0)
+                    word.RootWitness = input.RootWitness;
+                return true;
+            }
+
+            // Under the flag's copy conditions the output is a run of the input's segments starting at
+            // RootWindowStart, so a root run of the input inside that window is a root run of the output.
+            if (sameStratum && word.RootWindowStart >= 0)
+            {
+                RootRun witness = input.RootWitness;
+                if (witness == null)
+                {
+                    TryFindRootRun(info, input, out RootRun found);
+                    witness = input.RootWitness = found ?? RootRun.None;
+                }
+                int start = word.RootWindowStart;
+                if (witness.Start >= start && witness.End <= start + outputSegments)
+                {
+                    word.RootWitness = new RootRun(witness.Start - start, witness.End - start);
+                    return true;
+                }
+            }
+
+            if (TryFindRootRun(info, word, out RootRun run))
+            {
+                word.RootWitness = run ?? RootRun.None;
+                return true;
+            }
+
+            if (RootSubstringPrune == RootSubstringPruneMode.Shadow)
+            {
+                RootSubstringShadow?.Invoke(word, site);
+                return true;
+            }
+            return false;
+        }
+
+        internal bool RootSubstringPruneActive(Stratum stratum) =>
+            RootSubstringPrune != RootSubstringPruneMode.Off
+            && stratum != null
+            && _rootSubstringPrune.TryGetValue(stratum, out RootSubstringPruneStratum info)
+            && info.Enabled;
+
+        private bool TryFindRootRun(RootSubstringPruneStratum info, Word word, out RootRun run)
+        {
+            foreach (RootAllomorphTrie trie in info.Tries)
+            {
+                RootSubstringProbed?.Invoke();
+                if (trie.TryFindRootRun(word.Shape, out run))
+                    return true;
+            }
+            run = null;
+            return false;
+        }
+
+        private static int SegmentCount(Word word)
+        {
+            int count = 0;
+            foreach (ShapeNode node in word.Shape)
+            {
+                if (node.Type() == HCFeatureSystem.Segment)
+                    count++;
+            }
+            return count;
+        }
+
         internal IEnumerable<Word> LexicalLookup(Word input)
         {
             if (_traceManager.IsTracing)

src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisAffixProcessAllomorphRuleSpec.cs

@@ -20,6 +20,8 @@ public override Word ApplyRhs(PatternRule<Word, ShapeNode> rule, Match<Word, Sha
         {
             Word output = match.Input.Clone();
             GenerateShape(_allomorph.Lhs, output.Shape, match);
+            if (PruningMorpher != null && PruningMorpher.RootSubstringPruneActive(output.Stratum))
+                output.RootWindowStart = CopiedWindowStart(_allomorph.Lhs, match);
             return output;
         }
     }

src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisAffixProcessRule.cs

@@ -93,7 +93,8 @@ out FeatureStruct syntacticFS
                     outWord.Freeze();
                     if (_morpher.TraceManager.IsTracing)
                         _morpher.TraceManager.MorphologicalRuleUnapplied(_rule, i, input, outWord);
-                    output.Add(outWord);
+                    if (_morpher.MayContainRoot(input, outWord, nameof(AnalysisAffixProcessRule)))
+                        output.Add(outWord);
                     unapplied = true;
                 }
 

src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisCompoundingRule.cs

@@ -154,7 +154,8 @@ out FeatureStruct syntacticFS
                     outWord.Freeze();
                     if (_morpher.TraceManager.IsTracing)
                         _morpher.TraceManager.MorphologicalRuleUnapplied(_rule, i, input, outWord);
-                    output.Add(outWord);
+                    if (_morpher.MayContainRoot(input, outWord, nameof(AnalysisCompoundingRule)))
+                        output.Add(outWord);
                     unapplied = true;
                 }
 

src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisCompoundingSubruleRuleSpec.cs

@@ -25,6 +25,8 @@ public override Word ApplyRhs(PatternRule<Word, ShapeNode> rule, Match<Word, Sha
         {
             Word output = match.Input.Clone();
             GenerateShape(_subrule.HeadLhs, output.Shape, match);
+            if (PruningMorpher != null && PruningMorpher.RootSubstringPruneActive(output.Stratum))
+                output.RootWindowStart = CopiedWindowStart(_subrule.HeadLhs, match);
             var nonHeadShape = new Shape(begin => new ShapeNode(
                 begin ? HCFeatureSystem.LeftSideAnchor : HCFeatureSystem.RightSideAnchor
             ));

src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisMorphologicalTransform.cs

@@ -178,6 +178,35 @@ public void GenerateShape(IList<Pattern<Word, ShapeNode>> lhs, Shape shape, Matc
                 AddPartNodes(part, match, shape);
         }
 
+        /// <summary>
+        /// The number of segments of the match's input before the first part the output copies, or -1 if it
+        /// copies none.
+        /// </summary>
+        protected int CopiedWindowStart(IList<Pattern<Word, ShapeNode>> lhs, Match<Word, ShapeNode> match)
+        {
+            foreach (Pattern<Word, ShapeNode> part in lhs)
+            {
+                if (!_capturedParts.TryGetValue(part.Name, out int count))
+                    continue;
+                for (int i = 0; i < count; i++)
+                {
+                    GroupCapture<ShapeNode> group = match.GroupCaptures[GetGroupName(part.Name, i)];
+                    if (!group.Success)
+                        continue;
+                    int segments = 0;
+                    foreach (ShapeNode node in match.Input.Shape)
+                    {
+                        if (node == group.Range.Start)
+                            return segments;
+                        if (node.Type() == HCFeatureSystem.Segment)
+                            segments++;
+                    }
+                    return -1;
+                }
+            }
+            return -1;
+        }
+
         private void AddPartNodes(Pattern<Word, ShapeNode> part, Match<Word, ShapeNode> match, Shape output)
         {
             int count;

src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisRealizationalAffixProcessRule.cs

@@ -59,7 +59,8 @@ public IEnumerable<Word> Apply(Word input)
                     outWord.Freeze();
                     if (_morpher.TraceManager.IsTracing)
                         _morpher.TraceManager.MorphologicalRuleUnapplied(_rule, i, input, outWord);
-                    output.Add(outWord);
+                    if (_morpher.MayContainRoot(input, outWord, nameof(AnalysisRealizationalAffixProcessRule)))
+                        output.Add(outWord);
                     unapplied = true;
                 }
 

src/SIL.Machine.Morphology.HermitCrab/RootAllomorphTrie.cs

@@ -13,6 +13,8 @@ internal class RootAllomorphTrie
         private readonly Fst<Shape, ShapeNode> _fsa;
         private readonly Func<Annotation<ShapeNode>, bool> _filter;
         private int _shapeCount;
+        private bool _hasEmptyAllomorph;
+        private readonly Dictionary<string, int> _rootLengths = new Dictionary<string, int>();
         private readonly Dictionary<string, RootAllomorph> _allomorphs;
 
         public RootAllomorphTrie(Func<Annotation<ShapeNode>, bool> filter)
@@ -26,11 +28,17 @@ public RootAllomorphTrie(Func<Annotation<ShapeNode>, bool> filter)
         public void Add(RootAllomorph allomorph)
         {
             _allomorphs[allomorph.ID] = allomorph;
+            _rootLengths[allomorph.ID] = allomorph.Segments.Shape.Count(n => _filter(n.Annotation));
             ShapeNode first = allomorph.Segments.Shape.GetFirst(n => _filter(n.Annotation));
             if (first == allomorph.Segments.Shape.End)
+            {
+                _hasEmptyAllomorph = true;
                 _fsa.StartState.Arcs.Add(_fsa.CreateAcceptingState(allomorph.ID, (s, match) => true, _shapeCount));
+            }
             else
+            {
                 AddNode(first, _fsa.StartState, allomorph.ID);
+            }
             _shapeCount++;
         }
 
@@ -67,6 +75,44 @@ private void AddNode(ShapeNode node, State<Shape, ShapeNode> state, string id)
                 AddNode(nextNode, nextState, id);
         }
 
+        public bool IsEmpty => _allomorphs.Count == 0;
+
+        /// <summary>
+        /// Whether any root occurs as a contiguous run of the shape's segments, starting anywhere and ending
+        /// anywhere. Stops at the first accepting path.
+        /// </summary>
+        public bool ContainsRootSubstring(Shape shape)
+        {
+            if (_hasEmptyAllomorph)
+                return true;
+            Annotation<ShapeNode> startAnn = shape.Annotations.GetFirst(_filter);
+            return _fsa.Transduce(shape, startAnn, null, false, false, false, out FstResult<Shape, ShapeNode> _);
+        }
+
+        /// <summary>
+        /// Finds the first root run by trying each start segment in turn, which costs the same arcs as
+        /// <see cref="ContainsRootSubstring"/> and also says where the run is, in segment indices.
+        /// </summary>
+        public bool TryFindRootRun(Shape shape, out RootRun run)
+        {
+            run = null;
+            if (_hasEmptyAllomorph)
+                return true;
+            int index = 0;
+            foreach (Annotation<ShapeNode> ann in shape.Annotations)
+            {
+                if (!_filter(ann))
+                    continue;
+                if (_fsa.Transduce(shape, ann, null, true, false, false, out FstResult<Shape, ShapeNode> result))
+                {
+                    run = new RootRun(index, index + _rootLengths[result.ID]);
+                    return true;
+                }
+                index++;
+            }
+            return false;
+        }
+
         public IEnumerable<RootAllomorph> Search(Shape shape)
         {
             Annotation<ShapeNode> startAnn = shape.Annotations.GetFirst(_filter);

src/SIL.Machine.Morphology.HermitCrab/RootRun.cs

@@ -0,0 +1,18 @@
+namespace SIL.Machine.Morphology.HermitCrab
+{
+    /// <summary>A root found as a run of segments [Start, End) of an analysis shape.</summary>
+    internal sealed class RootRun
+    {
+        public RootRun(int start, int end)
+        {
+            Start = start;
+            End = end;
+        }
+
+        /// <summary>Stands for no usable location: nothing lies inside it.</summary>
+        public static readonly RootRun None = new RootRun(-1, -1);
+
+        public int Start { get; }
+        public int End { get; }
+    }
+}