diff --git a/src/SIL.Machine.Morphology.HermitCrab/Morpher.cs b/src/SIL.Machine.Morphology.HermitCrab/Morpher.cs index d0675509e..14e2f3cde 100644 --- a/src/SIL.Machine.Morphology.HermitCrab/Morpher.cs +++ b/src/SIL.Machine.Morphology.HermitCrab/Morpher.cs @@ -25,6 +25,7 @@ public class Morpher : IMorphologicalAnalyzer, IMorphologicalGenerator private readonly ITraceManager _traceManager; private readonly ReadOnlyObservableCollection _morphemes; private readonly IList _lexicalPatterns = new List(); + private readonly Dictionary _rootSubstringPrune; public Morpher(ITraceManager traceManager, Language lang, int maxDegreeOfParallelism = 0) { @@ -52,6 +53,7 @@ public Morpher(ITraceManager traceManager, Language lang, int maxDegreeOfParalle morphemes.AddRange(stratum.MorphologicalRules.OfType()); morphemes.AddRange(stratum.AffixTemplates.SelectMany(t => t.Slots).SelectMany(s => s.Rules).Distinct()); } + _rootSubstringPrune = RootSubstringPruneStratum.Compute(lang, _allomorphTries, _lexicalPatterns.Count > 0); _analysisRule = lang.CompileAnalysisRule(this); _synthesisRule = lang.CompileSynthesisRule(this); MaxStemCount = 2; @@ -424,6 +426,113 @@ internal IEnumerable SearchRootAllomorphs(Stratum stratum, Shape return alloSearcher.Search(shape).Distinct(); } + /// + /// Controls the root-substring prune of . Shadow evaluates the predicate and + /// reports through without pruning. + /// + internal RootSubstringPruneMode RootSubstringPrune { get; set; } = RootSubstringPruneMode.On; + + /// Called in shadow mode with the rule output and the rule type that would have been pruned. + internal Action RootSubstringShadow { get; set; } + + /// + /// Diagnostic hook, called once per trie transduction. Null in production, so the hot path keeps no shared + /// counter. + /// + internal Action RootSubstringProbed { get; set; } + + internal IReadOnlyDictionary RootSubstringPruneStrata => + _rootSubstringPrune; + + /// + /// Whether an analysis rule output may still lead to a lexical hit. False only when the prune is on for + /// the output's stratum and no root occurs as a contiguous run of its segments. The probe is skipped when + /// the output kept at least as many segments as its input in the same stratum: the rule then removed no + /// segment-bearing material, so under the flag's copy conditions every root run of the input is still a + /// run of the output. Skipping only ever keeps a word, so it cannot lose a parse. + /// + internal bool MayContainRoot(Word input, Word word, string site) + { + if ( + RootSubstringPrune == RootSubstringPruneMode.Off + || word.Stratum == null + || !_rootSubstringPrune.TryGetValue(word.Stratum, out RootSubstringPruneStratum info) + || !info.Enabled + ) + { + return true; + } + + bool sameStratum = input.Stratum == word.Stratum; + int outputSegments = SegmentCount(word); + if (sameStratum && outputSegments >= SegmentCount(input)) + { + if (word.RootWindowStart == 0) + word.RootWitness = input.RootWitness; + return true; + } + + // Under the flag's copy conditions the output is a run of the input's segments starting at + // RootWindowStart, so a root run of the input inside that window is a root run of the output. + if (sameStratum && word.RootWindowStart >= 0) + { + RootRun witness = input.RootWitness; + if (witness == null) + { + TryFindRootRun(info, input, out RootRun found); + witness = input.RootWitness = found ?? RootRun.None; + } + int start = word.RootWindowStart; + if (witness.Start >= start && witness.End <= start + outputSegments) + { + word.RootWitness = new RootRun(witness.Start - start, witness.End - start); + return true; + } + } + + if (TryFindRootRun(info, word, out RootRun run)) + { + word.RootWitness = run ?? RootRun.None; + return true; + } + + if (RootSubstringPrune == RootSubstringPruneMode.Shadow) + { + RootSubstringShadow?.Invoke(word, site); + return true; + } + return false; + } + + internal bool RootSubstringPruneActive(Stratum stratum) => + RootSubstringPrune != RootSubstringPruneMode.Off + && stratum != null + && _rootSubstringPrune.TryGetValue(stratum, out RootSubstringPruneStratum info) + && info.Enabled; + + private bool TryFindRootRun(RootSubstringPruneStratum info, Word word, out RootRun run) + { + foreach (RootAllomorphTrie trie in info.Tries) + { + RootSubstringProbed?.Invoke(); + if (trie.TryFindRootRun(word.Shape, out run)) + return true; + } + run = null; + return false; + } + + private static int SegmentCount(Word word) + { + int count = 0; + foreach (ShapeNode node in word.Shape) + { + if (node.Type() == HCFeatureSystem.Segment) + count++; + } + return count; + } + internal IEnumerable LexicalLookup(Word input) { if (_traceManager.IsTracing) diff --git a/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisAffixProcessAllomorphRuleSpec.cs b/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisAffixProcessAllomorphRuleSpec.cs index 580b663fb..54c881f5f 100644 --- a/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisAffixProcessAllomorphRuleSpec.cs +++ b/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisAffixProcessAllomorphRuleSpec.cs @@ -20,6 +20,8 @@ public override Word ApplyRhs(PatternRule rule, Match rule, Match new ShapeNode( begin ? HCFeatureSystem.LeftSideAnchor : HCFeatureSystem.RightSideAnchor )); diff --git a/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisMorphologicalTransform.cs b/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisMorphologicalTransform.cs index ed77b3cb3..2d616f86f 100644 --- a/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisMorphologicalTransform.cs +++ b/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisMorphologicalTransform.cs @@ -178,6 +178,35 @@ public void GenerateShape(IList> lhs, Shape shape, Matc AddPartNodes(part, match, shape); } + /// + /// The number of segments of the match's input before the first part the output copies, or -1 if it + /// copies none. + /// + protected int CopiedWindowStart(IList> lhs, Match match) + { + foreach (Pattern part in lhs) + { + if (!_capturedParts.TryGetValue(part.Name, out int count)) + continue; + for (int i = 0; i < count; i++) + { + GroupCapture group = match.GroupCaptures[GetGroupName(part.Name, i)]; + if (!group.Success) + continue; + int segments = 0; + foreach (ShapeNode node in match.Input.Shape) + { + if (node == group.Range.Start) + return segments; + if (node.Type() == HCFeatureSystem.Segment) + segments++; + } + return -1; + } + } + return -1; + } + private void AddPartNodes(Pattern part, Match match, Shape output) { int count; diff --git a/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisRealizationalAffixProcessRule.cs b/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisRealizationalAffixProcessRule.cs index db1c99cb8..a094c35d6 100644 --- a/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisRealizationalAffixProcessRule.cs +++ b/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisRealizationalAffixProcessRule.cs @@ -59,7 +59,8 @@ public IEnumerable Apply(Word input) outWord.Freeze(); if (_morpher.TraceManager.IsTracing) _morpher.TraceManager.MorphologicalRuleUnapplied(_rule, i, input, outWord); - output.Add(outWord); + if (_morpher.MayContainRoot(input, outWord, nameof(AnalysisRealizationalAffixProcessRule))) + output.Add(outWord); unapplied = true; } diff --git a/src/SIL.Machine.Morphology.HermitCrab/RootAllomorphTrie.cs b/src/SIL.Machine.Morphology.HermitCrab/RootAllomorphTrie.cs index 7d34e81a5..58cd503df 100644 --- a/src/SIL.Machine.Morphology.HermitCrab/RootAllomorphTrie.cs +++ b/src/SIL.Machine.Morphology.HermitCrab/RootAllomorphTrie.cs @@ -13,6 +13,8 @@ internal class RootAllomorphTrie private readonly Fst _fsa; private readonly Func, bool> _filter; private int _shapeCount; + private bool _hasEmptyAllomorph; + private readonly Dictionary _rootLengths = new Dictionary(); private readonly Dictionary _allomorphs; public RootAllomorphTrie(Func, bool> filter) @@ -26,11 +28,17 @@ public RootAllomorphTrie(Func, bool> filter) public void Add(RootAllomorph allomorph) { _allomorphs[allomorph.ID] = allomorph; + _rootLengths[allomorph.ID] = allomorph.Segments.Shape.Count(n => _filter(n.Annotation)); ShapeNode first = allomorph.Segments.Shape.GetFirst(n => _filter(n.Annotation)); if (first == allomorph.Segments.Shape.End) + { + _hasEmptyAllomorph = true; _fsa.StartState.Arcs.Add(_fsa.CreateAcceptingState(allomorph.ID, (s, match) => true, _shapeCount)); + } else + { AddNode(first, _fsa.StartState, allomorph.ID); + } _shapeCount++; } @@ -67,6 +75,44 @@ private void AddNode(ShapeNode node, State state, string id) AddNode(nextNode, nextState, id); } + public bool IsEmpty => _allomorphs.Count == 0; + + /// + /// Whether any root occurs as a contiguous run of the shape's segments, starting anywhere and ending + /// anywhere. Stops at the first accepting path. + /// + public bool ContainsRootSubstring(Shape shape) + { + if (_hasEmptyAllomorph) + return true; + Annotation startAnn = shape.Annotations.GetFirst(_filter); + return _fsa.Transduce(shape, startAnn, null, false, false, false, out FstResult _); + } + + /// + /// Finds the first root run by trying each start segment in turn, which costs the same arcs as + /// and also says where the run is, in segment indices. + /// + public bool TryFindRootRun(Shape shape, out RootRun run) + { + run = null; + if (_hasEmptyAllomorph) + return true; + int index = 0; + foreach (Annotation ann in shape.Annotations) + { + if (!_filter(ann)) + continue; + if (_fsa.Transduce(shape, ann, null, true, false, false, out FstResult result)) + { + run = new RootRun(index, index + _rootLengths[result.ID]); + return true; + } + index++; + } + return false; + } + public IEnumerable Search(Shape shape) { Annotation startAnn = shape.Annotations.GetFirst(_filter); diff --git a/src/SIL.Machine.Morphology.HermitCrab/RootRun.cs b/src/SIL.Machine.Morphology.HermitCrab/RootRun.cs new file mode 100644 index 000000000..75cce321d --- /dev/null +++ b/src/SIL.Machine.Morphology.HermitCrab/RootRun.cs @@ -0,0 +1,18 @@ +namespace SIL.Machine.Morphology.HermitCrab +{ + /// A root found as a run of segments [Start, End) of an analysis shape. + internal sealed class RootRun + { + public RootRun(int start, int end) + { + Start = start; + End = end; + } + + /// Stands for no usable location: nothing lies inside it. + public static readonly RootRun None = new RootRun(-1, -1); + + public int Start { get; } + public int End { get; } + } +} diff --git a/src/SIL.Machine.Morphology.HermitCrab/RootSubstringPrune.cs b/src/SIL.Machine.Morphology.HermitCrab/RootSubstringPrune.cs new file mode 100644 index 000000000..9da243951 --- /dev/null +++ b/src/SIL.Machine.Morphology.HermitCrab/RootSubstringPrune.cs @@ -0,0 +1,189 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using SIL.Machine.Annotations; +using SIL.Machine.Matching; +using SIL.Machine.Morphology.HermitCrab.MorphologicalRules; + +namespace SIL.Machine.Morphology.HermitCrab +{ + internal enum RootSubstringPruneMode + { + Off, + Shadow, + On, + } + + /// + /// Decides, once per stratum, whether an analysis rule output that contains no root as a contiguous run of + /// segments can be dropped. Under the three conditions below, every word analysis can derive below such an + /// output is a contiguous run of that output's segment nodes, with feature structures at least as specific, + /// so an exact root match anywhere below it would be a substring match in the output itself: + /// 1. every morphological, template-slot and compounding rule of the stratum and of each deeper stratum + /// writes inserted material, then every LHS part copied unmodified, once, in LHS order and with nothing + /// between the copies, then inserted material (a compounding rule only needs this for its head parts); + /// 2. no deeper stratum has phonological rules (the stratum's own are unapplied before any rule output); + /// 3. the language has no lexical-pattern roots, which + /// matches against analyses when guessing roots. + /// + internal sealed class RootSubstringPruneStratum + { + public RootSubstringPruneStratum(IReadOnlyList tries, string disabledReason) + { + Tries = tries; + DisabledReason = disabledReason; + } + + public IReadOnlyList Tries { get; } + + /// The first rule or condition that disables the prune, or null if it is enabled. + public string DisabledReason { get; } + + public bool Enabled => DisabledReason == null; + + public static Dictionary Compute( + Language language, + IReadOnlyDictionary tries, + bool hasLexicalPatterns + ) + { + var result = new Dictionary(); + string firstReason = hasLexicalPatterns ? "the language has lexical-pattern roots" : null; + var coveredTries = new List(); + string pruleReason = null; + string ruleReason = null; + foreach (Stratum stratum in language.Strata) + { + // the phonological rules of the strata below this one run when analysis enters them + string reason = firstReason ?? pruleReason; + if (ruleReason == null) + ruleReason = FirstNonContiguousRemovingRule(stratum); + reason = reason ?? ruleReason; + if (!tries[stratum].IsEmpty) + coveredTries.Add(tries[stratum]); + result[stratum] = new RootSubstringPruneStratum(coveredTries.ToArray(), reason); + if (pruleReason == null && stratum.PhonologicalRules.Count > 0) + pruleReason = $"deeper stratum '{stratum.Name}' has phonological rules"; + } + return result; + } + + private static string FirstNonContiguousRemovingRule(Stratum stratum) + { + foreach (IMorphologicalRule rule in stratum.MorphologicalRules) + { + string reason = NonContiguousRemovingReason(rule); + if (reason != null) + return $"rule '{rule.Name}' of stratum '{stratum.Name}' {reason}"; + } + foreach ( + MorphemicMorphologicalRule rule in stratum + .AffixTemplates.SelectMany(t => t.Slots) + .SelectMany(s => s.Rules) + .Distinct() + ) + { + string reason = NonContiguousRemovingReason(rule); + if (reason != null) + return $"template rule '{rule.Name}' of stratum '{stratum.Name}' {reason}"; + } + return null; + } + + internal static string NonContiguousRemovingReason(IMorphologicalRule rule) + { + switch (rule) + { + case AffixProcessRule affixRule: + return AllomorphsReason(affixRule.Allomorphs); + case RealizationalAffixProcessRule realizationalRule: + return AllomorphsReason(realizationalRule.Allomorphs); + case CompoundingRule compoundingRule: + foreach (CompoundingSubrule subrule in compoundingRule.Subrules) + { + string reason = BlockReason( + subrule.HeadLhs, + subrule.Rhs, + new HashSet(subrule.HeadLhs.Select(p => p.Name)) + ); + if (reason != null) + return reason; + } + return null; + default: + return "has an unknown rule type"; + } + } + + private static string AllomorphsReason(IEnumerable allomorphs) + { + foreach (AffixProcessAllomorph allomorph in allomorphs) + { + string reason = BlockReason( + allomorph.Lhs, + allomorph.Rhs, + new HashSet(allomorph.Lhs.Select(p => p.Name)) + ); + if (reason != null) + return reason; + } + return null; + } + + private static string BlockReason( + IList> lhs, + IList rhs, + HashSet outputParts + ) + { + var copies = new List(); + int firstCopy = -1; + int lastCopy = -1; + for (int i = 0; i < rhs.Count; i++) + { + switch (rhs[i]) + { + case InsertSegments _: + case InsertSimpleContext _: + break; + case CopyFromInput copy: + if (!outputParts.Contains(copy.PartName)) + break; + if (copies.Contains(copy.PartName)) + return "copies a part more than once"; + copies.Add(copy.PartName); + if (firstCopy < 0) + firstCopy = i; + lastCopy = i; + break; + case ModifyFromInput _: + return "modifies a part"; + default: + return "has an unknown output action"; + } + } + + if (copies.Count > 0 && lastCopy - firstCopy + 1 != copies.Count) + return "inserts or copies other material between its output parts"; + + int next = 0; + foreach (Pattern part in lhs) + { + if (next < copies.Count && copies[next] == part.Name) + next++; + else if (copies.Contains(part.Name)) + return "copies its parts out of LHS order"; + else if (HasSegments(part)) + return $"drops the part '{part.Name}', which can hold segments, so analysis restores them"; + } + return null; + } + + private static bool HasSegments(PatternNode node) + { + if (node is Constraint constraint) + return constraint.Type() == HCFeatureSystem.Segment; + return !node.IsLeaf && node.Children.Any(HasSegments); + } + } +} diff --git a/src/SIL.Machine.Morphology.HermitCrab/Word.cs b/src/SIL.Machine.Morphology.HermitCrab/Word.cs index a1722a18a..f5fc5ac94 100644 --- a/src/SIL.Machine.Morphology.HermitCrab/Word.cs +++ b/src/SIL.Machine.Morphology.HermitCrab/Word.cs @@ -75,6 +75,15 @@ public Word(Stratum stratum, Shape shape) _disjunctiveAllomorphIndices = new Dictionary>(); } + /// + /// Where the shape lies in the segments of the word an analysis rule unapplied (or -1), and a root run + /// found in it. Both only let skip a probe, so a stale value can + /// keep a word but never drop one; they are not copied by Clone. + /// + internal int RootWindowStart { get; set; } = -1; + + internal RootRun RootWitness { get; set; } + protected Word(Word word) { _allomorphs = new Dictionary(word._allomorphs); diff --git a/tests/SIL.Machine.Morphology.HermitCrab.Tests/MorphologicalRules/AnalysisSyntacticFeatureStructTests.cs b/tests/SIL.Machine.Morphology.HermitCrab.Tests/MorphologicalRules/AnalysisSyntacticFeatureStructTests.cs index d73c85e69..68da0393b 100644 --- a/tests/SIL.Machine.Morphology.HermitCrab.Tests/MorphologicalRules/AnalysisSyntacticFeatureStructTests.cs +++ b/tests/SIL.Machine.Morphology.HermitCrab.Tests/MorphologicalRules/AnalysisSyntacticFeatureStructTests.cs @@ -28,7 +28,10 @@ public void AnalysisAffixProcessRule_CategoryChangeChain_RequiredOverridesAccumu AffixProcessRule v2n = MakeSuffixRule("v2n", v, n, Table3, "i"); AffixProcessRule third = MakeSuffixRule("third", n, v, Table3, "a"); - var morpher = new Morpher(TraceManager, Language, maxDegreeOfParallelism: 1); + var morpher = new Morpher(TraceManager, Language, maxDegreeOfParallelism: 1) + { + RootSubstringPrune = RootSubstringPruneMode.Off, + }; var analysisV2N = new AnalysisAffixProcessRule(morpher, v2n); var analysisN2V = new AnalysisAffixProcessRule(morpher, n2v); var analysisThird = new AnalysisAffixProcessRule(morpher, third); diff --git a/tests/SIL.Machine.Morphology.HermitCrab.Tests/MorphologicalRules/PruneDisagreeingCopiesTests.cs b/tests/SIL.Machine.Morphology.HermitCrab.Tests/MorphologicalRules/PruneDisagreeingCopiesTests.cs index ffd6fd9f8..77c16793d 100644 --- a/tests/SIL.Machine.Morphology.HermitCrab.Tests/MorphologicalRules/PruneDisagreeingCopiesTests.cs +++ b/tests/SIL.Machine.Morphology.HermitCrab.Tests/MorphologicalRules/PruneDisagreeingCopiesTests.cs @@ -548,7 +548,11 @@ private RewriteRule CreateBetweenVowelsRewrite(CharacterDefinitionTable table, s private Morpher CreateMorpher(bool pruneCopies, int maxDegreeOfParallelism = 0) { - return new Morpher(TraceManager, Language, maxDegreeOfParallelism) { PruneDisagreeingCopies = pruneCopies }; + return new Morpher(TraceManager, Language, maxDegreeOfParallelism) + { + PruneDisagreeingCopies = pruneCopies, + RootSubstringPrune = RootSubstringPruneMode.Off, + }; } private static string[] ParseGlosses(Morpher morpher, string word) diff --git a/tests/SIL.Machine.Morphology.HermitCrab.Tests/RootSubstringPruneTests.cs b/tests/SIL.Machine.Morphology.HermitCrab.Tests/RootSubstringPruneTests.cs new file mode 100644 index 000000000..98b3e413c --- /dev/null +++ b/tests/SIL.Machine.Morphology.HermitCrab.Tests/RootSubstringPruneTests.cs @@ -0,0 +1,424 @@ +using NUnit.Framework; +using SIL.Machine.Annotations; +using SIL.Machine.FeatureModel; +using SIL.Machine.Matching; +using SIL.Machine.Morphology.HermitCrab.MorphologicalRules; +using SIL.Machine.Morphology.HermitCrab.PhonologicalRules; + +namespace SIL.Machine.Morphology.HermitCrab; + +[TestFixture] +public class RootSubstringPruneTests : HermitCrabTestBase +{ + private static readonly FeatureStruct Any = FeatureStruct.New().Symbol(HCFeatureSystem.Segment).Value; + + private int _attempts; + + private AffixProcessRule Suffix(string name, string insert, string? requiredPos = null) + { + var rule = new AffixProcessRule + { + Id = name, + Name = name, + Gloss = name, + }; + if (requiredPos != null) + { + rule.RequiredSyntacticFeatureStruct = FeatureStruct + .New(Language.SyntacticFeatureSystem) + .Symbol(requiredPos) + .Value; + } + rule.Allomorphs.Add( + new AffixProcessAllomorph + { + Lhs = { Pattern.New("1").Annotation(Any).OneOrMore.Value }, + Rhs = { new CopyFromInput("1"), new InsertSegments(Table3, insert) }, + } + ); + return rule; + } + + private Morpher CreateMorpher(RootSubstringPruneMode mode) + { + var morpher = new Morpher(TraceManager, Language, maxDegreeOfParallelism: 1) { RootSubstringPrune = mode }; + morpher.RuleSelector = rule => + { + if (rule is IMorphologicalRule) + Interlocked.Increment(ref _attempts); + return true; + }; + return morpher; + } + + private static string[] Signature(IEnumerable words) => + words + .Select(w => string.Join(" ", w.AllomorphsInMorphOrder.Select(a => a.Morpheme.Gloss))) + .Distinct() + .OrderBy(s => s, StringComparer.Ordinal) + .ToArray(); + + private (string[] Parses, int Attempts) Parse(RootSubstringPruneMode mode, string word) + { + _attempts = 0; + Morpher morpher = CreateMorpher(mode); + string[] parses = Signature(morpher.ParseWord(word)); + return (parses, _attempts); + } + + private void AssertSameParses(string word, bool expectParse, bool expectFewerAttempts = false) + { + (string[] off, int offAttempts) = Parse(RootSubstringPruneMode.Off, word); + (string[] on, int onAttempts) = Parse(RootSubstringPruneMode.On, word); + Assert.That(on, Is.EqualTo(off)); + Assert.That(off.Length > 0, Is.EqualTo(expectParse)); + if (expectFewerAttempts) + Assert.That(onAttempts, Is.LessThan(offAttempts)); + else + Assert.That(onAttempts, Is.EqualTo(offAttempts)); + } + + [Test] + public void SuffixingGrammar_RootNotInLexicon_FewerAttemptsAndNoParse() + { + Morphophonemic.MorphologicalRules.Add(Suffix("dRule", "+d")); + Morphophonemic.MorphologicalRules.Add(Suffix("zRule", "+z")); + + Assert.That(CreateMorpher(RootSubstringPruneMode.On).RootSubstringPruneStrata[Morphophonemic].Enabled, Is.True); + AssertSameParses("kukdz", expectParse: false, expectFewerAttempts: true); + } + + [Test] + public void SuffixingGrammar_RootInLexicon_ParseSurvives() + { + Morphophonemic.MorphologicalRules.Add(Suffix("dRule", "+d", "V")); + Morphophonemic.MorphologicalRules.Add(Suffix("zRule", "+z", "V")); + + AssertSameParses("sagd", expectParse: true); + AssertSameParses("sagdz", expectParse: true); + } + + [Test] + public void Infix_FlagOffAndInfixedWordParses() + { + var cons = FeatureStruct + .New(Language.PhonologicalFeatureSystem) + .Symbol(HCFeatureSystem.Segment) + .Symbol("cons+") + .Value; + var infix = new AffixProcessRule + { + Id = "perf", + Name = "perf", + Gloss = "perf", + }; + infix.Allomorphs.Add( + new AffixProcessAllomorph + { + Lhs = + { + Pattern.New("1").Annotation(cons).Value, + Pattern.New("2").Annotation(cons).Value, + Pattern.New("3").Annotation(cons).Value, + }, + Rhs = + { + new CopyFromInput("1"), + new InsertSegments(Table3, "a"), + new CopyFromInput("2"), + new InsertSegments(Table3, "a"), + new CopyFromInput("3"), + }, + } + ); + Morphophonemic.MorphologicalRules.Add(infix); + Morphophonemic.MorphologicalRules.Add(Suffix("zRule", "+z")); + + Assert.That( + CreateMorpher(RootSubstringPruneMode.On).RootSubstringPruneStrata[Morphophonemic].Enabled, + Is.False + ); + AssertSameParses("katabz", expectParse: true); + } + + [Test] + public void DeeperStratumPhonologicalRule_FlagOffAndWordParses() + { + var p = Character(Table3, "p"); + var asp = FeatureStruct + .New(Language.PhonologicalFeatureSystem) + .Symbol(HCFeatureSystem.Segment) + .Symbol("asp+") + .Value; + var nonCons = FeatureStruct + .New(Language.PhonologicalFeatureSystem) + .Symbol(HCFeatureSystem.Segment) + .Symbol("cons-") + .Value; + var rule = new RewriteRule { Name = "asp", Lhs = Pattern.New().Annotation(p).Value }; + Morphophonemic.PhonologicalRules.Add(rule); + rule.Subrules.Add( + new RewriteSubrule + { + Rhs = Pattern.New().Annotation(asp).Value, + RightEnvironment = Pattern.New().Annotation(nonCons).Value, + } + ); + Allophonic.MorphologicalRules.Add(Suffix("dRule", "+d")); + + Morpher morpher = CreateMorpher(RootSubstringPruneMode.On); + Assert.That(morpher.RootSubstringPruneStrata[Morphophonemic].Enabled, Is.True); + Assert.That(morpher.RootSubstringPruneStrata[Allophonic].Enabled, Is.False); + Assert.That(morpher.RootSubstringPruneStrata[Allophonic].DisabledReason, Does.Contain("phonological")); + AssertSameParses("pʰud", expectParse: true); + } + + [Test] + public void ModifyFromInput_FlagOff() + { + var voiced = FeatureStruct + .New(Language.PhonologicalFeatureSystem) + .Symbol(HCFeatureSystem.Segment) + .Symbol("vd+") + .Value; + var p = Character(Table1, "p"); + var simulfix = new AffixProcessRule + { + Id = "SIMUL", + Name = "SIMUL", + Gloss = "SIMUL", + }; + simulfix.Allomorphs.Add( + new AffixProcessAllomorph + { + Lhs = + { + Pattern.New("1").Annotation(Any).OneOrMore.Value, + Pattern.New("2").Annotation(p).Value, + }, + Rhs = { new CopyFromInput("1"), new ModifyFromInput("2", voiced) }, + } + ); + Allophonic.MorphologicalRules.Add(simulfix); + + Morpher morpher = CreateMorpher(RootSubstringPruneMode.On); + Assert.That(morpher.RootSubstringPruneStrata[Allophonic].Enabled, Is.False); + Assert.That(morpher.RootSubstringPruneStrata[Allophonic].DisabledReason, Does.Contain("modifies")); + Assert.That(morpher.RootSubstringPruneStrata[Surface].Enabled, Is.False); + Assert.That(morpher.RootSubstringPruneStrata[Morphophonemic].Enabled, Is.True); + AssertSameParses("pib", expectParse: true); + } + + [Test] + public void TruncationRestoredLater_FlagOffAndWordParses() + { + var truncate = new AffixProcessRule + { + Id = "trunc", + Name = "trunc", + Gloss = "trunc", + }; + truncate.Allomorphs.Add( + new AffixProcessAllomorph + { + Lhs = + { + Pattern.New("1").Annotation(Any).OneOrMore.Value, + Pattern.New("2").Annotation(Character(Table3, "g")).Value, + }, + Rhs = { new CopyFromInput("1"), new InsertSegments(Table3, "+d") }, + } + ); + Morphophonemic.MorphologicalRules.Add(truncate); + Morphophonemic.MorphologicalRules.Add(Suffix("zRule", "+z")); + + Morpher morpher = CreateMorpher(RootSubstringPruneMode.On); + Assert.That(morpher.RootSubstringPruneStrata[Morphophonemic].Enabled, Is.False); + Assert.That(morpher.RootSubstringPruneStrata[Morphophonemic].DisabledReason, Does.Contain("drops the part")); + AssertSameParses("sadz", expectParse: true); + } + + [Test] + public void Compounding_TwoLexiconRootsParse_UnknownHeadIsPruned() + { + Allophonic.MorphologicalRules.Add( + new CompoundingRule + { + Name = "rule1", + Subrules = + { + new CompoundingSubrule + { + HeadLhs = { Pattern.New("head").Annotation(Any).OneOrMore.Value }, + NonHeadLhs = { Pattern.New("nonHead").Annotation(Any).OneOrMore.Value }, + Rhs = + { + new CopyFromInput("head"), + new InsertSegments(Table3, "+"), + new CopyFromInput("nonHead"), + }, + }, + }, + } + ); + + Assert.That(CreateMorpher(RootSubstringPruneMode.On).RootSubstringPruneStrata[Allophonic].Enabled, Is.True); + AssertSameParses("pʰutdat", expectParse: true); + AssertSameParses("zazdat", expectParse: false, expectFewerAttempts: true); + } + + [Test] + public void LexicalPatternRoot_FlagOffAndGuessedRootsUnchanged() + { + Morphophonemic.MorphologicalRules.Add(Suffix("ed_suffix", "+d", "V")); + Morphophonemic.CharacterDefinitionTable.AddNaturalClass(new NaturalClass(new FeatureStruct()) { Name = "Any" }); + LexEntry pattern = AddEntry("pattern", new FeatureStruct(), Morphophonemic, "[Any]*"); + try + { + var guesses = new List(); + foreach (RootSubstringPruneMode mode in new[] { RootSubstringPruneMode.Off, RootSubstringPruneMode.On }) + { + var morpher = new Morpher(TraceManager, Language) { RootSubstringPrune = mode }; + Assert.That(morpher.RootSubstringPruneStrata.Values, Has.None.Property("Enabled").True); + Assert.That(morpher.AnalyzeWord("gagd"), Is.Empty); + guesses.Add(morpher.AnalyzeWord("gagd", true).Select(a => a.ToString()).ToArray()); + } + Assert.That(guesses[1], Is.EqualTo(guesses[0])); + Assert.That(guesses[1], Does.Contain("[*gag ed_suffix]")); + } + finally + { + Morphophonemic.Entries.Remove(pattern); + } + } + + [Test] + public void Shadow_ParsesUnchangedAndReportsFiredNodes() + { + Morphophonemic.MorphologicalRules.Add(Suffix("dRule", "+d")); + Morphophonemic.MorphologicalRules.Add(Suffix("zRule", "+z")); + + (string[] off, int offAttempts) = Parse(RootSubstringPruneMode.Off, "kukdz"); + Morpher shadow = CreateMorpher(RootSubstringPruneMode.Shadow); + int fired = 0; + int probes = 0; + shadow.RootSubstringProbed = () => probes++; + shadow.RootSubstringShadow = (word, site) => fired++; + _attempts = 0; + string[] parses = Signature(shadow.ParseWord("kukdz")); + + Assert.That(parses, Is.EqualTo(off)); + Assert.That(_attempts, Is.EqualTo(offAttempts)); + Assert.That(fired, Is.GreaterThan(0)); + Assert.That(probes, Is.GreaterThan(0)); + } + + [Test] + public void RuleThatRemovesNoSegments_IsNotProbed() + { + var nullAffix = new AffixProcessRule + { + Id = "nullRule", + Name = "nullRule", + Gloss = "nullRule", + }; + nullAffix.Allomorphs.Add( + new AffixProcessAllomorph + { + Lhs = { Pattern.New("1").Annotation(Any).OneOrMore.Value }, + Rhs = { new CopyFromInput("1") }, + } + ); + Morphophonemic.MorphologicalRules.Add(nullAffix); + + (string[] off, _) = Parse(RootSubstringPruneMode.Off, "kuk"); + Morpher on = CreateMorpher(RootSubstringPruneMode.On); + int probes = 0; + on.RootSubstringProbed = () => probes++; + Assert.That(Signature(on.ParseWord("kuk")), Is.EqualTo(off)); + Assert.That(probes, Is.EqualTo(0)); + } + + [Test] + public void ReusedRootRun_SkipsProbesAndKeepsPruneDecisions() + { + Morphophonemic.MorphologicalRules.Add(Suffix("dRule", "+d", "V")); + Morphophonemic.MorphologicalRules.Add(Suffix("zRule", "+z", "V")); + Morphophonemic.MorphologicalRules.Add(Suffix("tRule", "+t", "V")); + + // Probes counted on the commit without root-run reuse: sagdzt 3. Prune decisions (shadow-fired outputs) + // and attempts must be identical to the probing implementation: sagdzt 0 fired, kukdzt 3 fired. + foreach ((string word, int maxProbes, int expectedFired) in new[] { ("sagdzt", 1, 0), ("kukdzt", 2, 3) }) + { + (string[] off, int offAttempts) = Parse(RootSubstringPruneMode.Off, word); + Morpher on = CreateMorpher(RootSubstringPruneMode.On); + int probes = 0; + on.RootSubstringProbed = () => probes++; + _attempts = 0; + Assert.That(Signature(on.ParseWord(word)), Is.EqualTo(off), word); + int onAttempts = _attempts; + + Morpher shadow = CreateMorpher(RootSubstringPruneMode.Shadow); + int fired = 0; + shadow.RootSubstringShadow = (w, site) => fired++; + shadow.ParseWord(word); + + Assert.That(probes, Is.LessThanOrEqualTo(maxProbes), word); + Assert.That(fired, Is.EqualTo(expectedFired), word); + Assert.That(onAttempts, expectedFired == 0 ? Is.EqualTo(offAttempts) : Is.LessThan(offAttempts), word); + } + } + + private Word Analysis(string shape, int windowStart = -1, RootRun? witness = null) => + new(Morphophonemic, Morphophonemic.CharacterDefinitionTable.Segment(shape)) + { + RootWindowStart = windowStart, + RootWitness = witness, + }; + + [Test] + public void RootWitness_FoundOnceInTheInputAndCopiedToOutputs() + { + Morphophonemic.MorphologicalRules.Add(Suffix("dRule", "+d", "V")); + Morpher morpher = CreateMorpher(RootSubstringPruneMode.On); + int probes = 0; + morpher.RootSubstringProbed = () => probes++; + + Word input = Analysis("sagdz"); + Word shorter = Analysis("sagd", windowStart: 0); + Word shortest = Analysis("sag", windowStart: 0); + + Assert.That(morpher.MayContainRoot(input, shorter, "test"), Is.True); + int afterFirst = probes; + Assert.That(morpher.MayContainRoot(input, shortest, "test"), Is.True); + + Assert.That(afterFirst, Is.EqualTo(1)); + Assert.That(probes, Is.EqualTo(afterFirst), "the second output reuses the witness found in the input"); + Assert.That((input.RootWitness!.Start, input.RootWitness.End), Is.EqualTo((0, 3))); + Assert.That((shortest.RootWitness!.Start, shortest.RootWitness.End), Is.EqualTo((0, 3))); + } + + [Test] + public void StaleWitness_OnlySkipsAProbeAndNeverPrunes() + { + Morphophonemic.MorphologicalRules.Add(Suffix("dRule", "+d", "V")); + Morpher morpher = CreateMorpher(RootSubstringPruneMode.On); + int probes = 0; + morpher.RootSubstringProbed = () => probes++; + + // The witness claims a root run that "kuk" does not hold: the output is kept, with no probe. + Word staleInput = Analysis("kukdz", witness: new RootRun(0, 3)); + Assert.That(morpher.MayContainRoot(staleInput, Analysis("kuk", windowStart: 0), "test"), Is.True); + Assert.That(probes, Is.EqualTo(0)); + + // A witness outside the output's window is ignored and the probe decides: no root, so pruned. + Word input = Analysis("sagkuk", witness: new RootRun(0, 3)); + Assert.That(morpher.MayContainRoot(input, Analysis("kuk", windowStart: 3), "test"), Is.False); + Assert.That(probes, Is.GreaterThan(0)); + + // A witness for an input with no root lies in no window, so the probe decides and finds sag. + probes = 0; + Word noRoot = Analysis("kukdz", witness: RootRun.None); + Assert.That(morpher.MayContainRoot(noRoot, Analysis("sag", windowStart: 0), "test"), Is.True); + } +}