diff --git a/src/SIL.Machine.Morphology.HermitCrab/LookupReplay.cs b/src/SIL.Machine.Morphology.HermitCrab/LookupReplay.cs new file mode 100644 index 000000000..86711e60e --- /dev/null +++ b/src/SIL.Machine.Morphology.HermitCrab/LookupReplay.cs @@ -0,0 +1,219 @@ +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using SIL.Machine.Morphology.HermitCrab.MorphologicalRules; + +namespace SIL.Machine.Morphology.HermitCrab +{ + internal enum LookupReplayMode + { + Off, + Shadow, + On, + } + + /// + /// Decides from a lexical-lookup candidate's root and trail alone whether synthesis could ever apply every trail + /// entry and end in a valid word. Returns false only when no synthesis exists, so rejecting never changes a + /// parse. Candidates with a lexical family root are never rejected, because stem choice and blocking can replace + /// the root. + /// + /// + /// The check is linear in the trail, stops at the first violated constraint, runs only when some rule carries a + /// stem-name or MPR constraint, and keeps nothing between candidates. + /// + internal sealed class LookupReplay + { + private readonly bool _checksMpr; + private readonly bool _overwriteReachable; + private readonly MprFeatureSet _realizationalOut = new MprFeatureSet(); + private readonly MprFeatureSet _compoundingOut = new MprFeatureSet(); + private long _mprWork; + + public LookupReplay(Language language) + { + var allOut = new MprFeatureSet(); + foreach (Stratum stratum in language.Strata) + { + var rules = new List(stratum.MorphologicalRules); + rules.AddRange(stratum.AffixTemplates.SelectMany(t => t.Slots).SelectMany(s => s.Rules)); + foreach (IMorphologicalRule rule in rules.Distinct()) + { + switch (rule) + { + case AffixProcessRule affixRule: + _checksMpr |= affixRule.RequiredStemName != null; + foreach (AffixProcessAllomorph allo in affixRule.Allomorphs) + { + _checksMpr |= allo.RequiredMprFeatures.Count > 0 || allo.ExcludedMprFeatures.Count > 0; + allOut.UnionWith(allo.OutMprFeatures); + } + break; + case RealizationalAffixProcessRule realRule: + foreach (AffixProcessAllomorph allo in realRule.Allomorphs) + { + _realizationalOut.UnionWith(allo.OutMprFeatures); + allOut.UnionWith(allo.OutMprFeatures); + } + break; + case CompoundingRule compRule: + _compoundingOut.UnionWith(compRule.OutputProdRestrictionsMprFeatures); + foreach (CompoundingSubrule subrule in compRule.Subrules) + _compoundingOut.UnionWith(subrule.OutMprFeatures); + allOut.UnionWith(_compoundingOut); + break; + } + } + } + _overwriteReachable = allOut.Any(f => f.Group != null && f.Group.Output == MprFeatureGroupOutput.Overwrite); + } + + /// The MPR pass's units of work: one per feature-set match, union or copy. + public long MprWork => Interlocked.Read(ref _mprWork); + + public long Work => MprWork; + + public bool IsFeasible(Word alternative) + { + if (!_checksMpr) + return true; + if (!(alternative.RootAllomorph?.Morpheme is LexEntry root) || root.Family != null) + return true; + + long work = 0; + bool feasible = IsMprFeasible(alternative, root, ref work); + if (work > 0) + Interlocked.Add(ref _mprWork, work); + return feasible; + } + + /// + /// A synthesis succeeds only if every trail entry applies, in order, with its recorded allomorph (or any + /// allomorph when none is recorded), so one entry that cannot apply is enough to reject. The checks + /// decided without shapes are the stem name and the MPR features. + /// + /// The MPR set a rule sees is the root's, plus the outputs of earlier trail entries, plus outputs of + /// realizational rules (which are not on the trail and may fire anywhere), minus features removed by + /// overwrite groups. The set used for a required check is a superset of that set: it ignores removals + /// and adds every output that might have been added. A required check that fails against a superset + /// fails against the actual set, for both All and Any groups. The set used for an excluded check is a + /// subset: the root's features plus outputs that certainly apply (those of recorded allomorphs). It is + /// used only when no output can reach an overwrite group and no compounding rule is on the trail, so + /// nothing is removed or added unseen. An excluded check that fails against a subset fails against the + /// actual set, for both All and Any groups, because both tests are monotone in the set. + /// + /// + /// Compounding rules and unknown trail entries only add their possible outputs, and they disable both the + /// excluded check and the stem-name check, since the head may differ. + /// + /// + private bool IsMprFeasible(Word alternative, LexEntry root, ref long work) + { + StemName rootStemName = alternative.RootAllomorph.StemName; + // The sets are cloned on first change, so a trail that adds no features copies nothing. + MprFeatureSet lower = root.MprFeatures; + MprFeatureSet extra = _realizationalOut; + bool lowerOwned = false; + bool extraOwned = false; + bool excludedOk = !_overwriteReachable; + bool stemNameOk = true; + IReadOnlyList rules = alternative.MorphologicalRuleApplications; + IReadOnlyList allomorphs = alternative.AllomorphApplications; + for (int i = rules.Count - 1; i >= 0; i--) + { + if (!(rules[i] is AffixProcessRule rule)) + { + if (_compoundingOut.Count > 0) + { + work++; + Own(ref extra, ref extraOwned).UnionWith(_compoundingOut); + } + excludedOk = false; + stemNameOk = false; + continue; + } + if (stemNameOk && rule.RequiredStemName != null && rule.RequiredStemName != rootStemName) + return false; + + var recorded = allomorphs[i] as AffixProcessAllomorph; + MprFeatureSet upper = null; + bool anyPossible = false; + if (recorded != null) + { + anyPossible = IsAllomorphPossible(recorded, lower, extra, excludedOk, ref upper, ref work); + } + else + { + foreach (AffixProcessAllomorph allo in rule.Allomorphs) + { + if (IsAllomorphPossible(allo, lower, extra, excludedOk, ref upper, ref work)) + { + anyPossible = true; + break; + } + } + } + if (!anyPossible) + return false; + + if (recorded != null) + { + if (recorded.OutMprFeatures.Count > 0) + { + work++; + Own(ref lower, ref lowerOwned).UnionWith(recorded.OutMprFeatures); + } + } + else + { + foreach (AffixProcessAllomorph allo in rule.Allomorphs) + { + if (allo.OutMprFeatures.Count > 0) + { + work++; + Own(ref extra, ref extraOwned).UnionWith(allo.OutMprFeatures); + } + } + } + } + return true; + } + + private static bool IsAllomorphPossible( + AffixProcessAllomorph allo, + MprFeatureSet lower, + MprFeatureSet extra, + bool excludedOk, + ref MprFeatureSet upper, + ref long work + ) + { + if (allo.RequiredMprFeatures.Count > 0) + { + if (upper == null) + { + work++; + upper = lower.Clone(); + upper.UnionWith(extra); + } + work++; + if (!allo.RequiredMprFeatures.IsMatchRequired(upper, out _)) + return false; + } + if (!excludedOk || allo.ExcludedMprFeatures.Count == 0) + return true; + work++; + return allo.ExcludedMprFeatures.IsMatchExcluded(lower, out _); + } + + private static MprFeatureSet Own(ref MprFeatureSet set, ref bool owned) + { + if (!owned) + { + set = set.Clone(); + owned = true; + } + return set; + } + } +} diff --git a/src/SIL.Machine.Morphology.HermitCrab/Morpher.cs b/src/SIL.Machine.Morphology.HermitCrab/Morpher.cs index d0675509e..eb2d6cc54 100644 --- a/src/SIL.Machine.Morphology.HermitCrab/Morpher.cs +++ b/src/SIL.Machine.Morphology.HermitCrab/Morpher.cs @@ -25,6 +25,7 @@ public class Morpher : IMorphologicalAnalyzer, IMorphologicalGenerator private readonly ITraceManager _traceManager; private readonly ReadOnlyObservableCollection _morphemes; private readonly IList _lexicalPatterns = new List(); + private readonly LookupReplay _lookupReplay; public Morpher(ITraceManager traceManager, Language lang, int maxDegreeOfParallelism = 0) { @@ -54,6 +55,7 @@ public Morpher(ITraceManager traceManager, Language lang, int maxDegreeOfParalle } _analysisRule = lang.CompileAnalysisRule(this); _synthesisRule = lang.CompileSynthesisRule(this); + _lookupReplay = new LookupReplay(lang); MaxStemCount = 2; MaxAlternatives = 0; MergeEquivalentAnalyses = true; @@ -114,6 +116,22 @@ public ITraceManager TraceManager /// public bool PruneDisagreeingCopies { get; set; } + internal LookupReplayMode LookupReplayMode { get; set; } = LookupReplayMode.On; + + /// Called with each alternative the replay judged, and whether it is feasible. + internal Action LookupReplayVerdict { get; set; } + + internal long LookupReplayWork => _lookupReplay.Work; + + private bool IsPrunedByLookupReplay(Word alternative) + { + if (LookupReplayMode == LookupReplayMode.Off) + return false; + bool feasible = _lookupReplay.IsFeasible(alternative); + LookupReplayVerdict?.Invoke(alternative, feasible); + return !feasible && LookupReplayMode == LookupReplayMode.On; + } + /// /// Caps concurrency within a parse or generation, including the analysis cascade, affix-template /// unapplication, and synthesis. A value of 1 runs the work sequentially; values below 1 leave @@ -355,6 +373,8 @@ private IEnumerable SynthesizeSequential(string word, IEnumerable an alternativeCount++; if (MaxAlternatives > 0 && alternativeCount > MaxAlternatives) throw new MaxAlternativesExceededException("MaxAlternatives exceeded"); + if (IsPrunedByLookupReplay(alternative)) + continue; foreach (Word validWord in _synthesisRule.Apply(alternative).Where(IsWordValid)) { if (IsMatch(word, validWord)) @@ -392,6 +412,8 @@ private IEnumerable SynthesizeParallel(string word, ConcurrentQueue { throw new MaxAlternativesExceededException("MaxAlternatives exceeded"); } + if (IsPrunedByLookupReplay(alternative)) + continue; foreach (Word validWord in _synthesisRule.Apply(alternative).Where(IsWordValid)) { if (IsMatch(word, validWord)) diff --git a/src/SIL.Machine.Morphology.HermitCrab/Word.cs b/src/SIL.Machine.Morphology.HermitCrab/Word.cs index a1722a18a..5fbe452c2 100644 --- a/src/SIL.Machine.Morphology.HermitCrab/Word.cs +++ b/src/SIL.Machine.Morphology.HermitCrab/Word.cs @@ -235,6 +235,10 @@ public IEnumerable MorphologicalRules get { return _mruleApps; } } + internal IReadOnlyList MorphologicalRuleApplications => _mruleApps; + + internal IReadOnlyList AllomorphApplications => _allomorphApps; + internal int MorphologicalRuleApplicationCount => _mruleAppCount; internal bool IsAllMorphologicalRulesApplied diff --git a/tests/SIL.Machine.Morphology.HermitCrab.Tests/LookupReplayMprTests.cs b/tests/SIL.Machine.Morphology.HermitCrab.Tests/LookupReplayMprTests.cs new file mode 100644 index 000000000..bbee7b2c4 --- /dev/null +++ b/tests/SIL.Machine.Morphology.HermitCrab.Tests/LookupReplayMprTests.cs @@ -0,0 +1,448 @@ +using NUnit.Framework; +using SIL.Machine.FeatureModel; +using SIL.Machine.Morphology.HermitCrab.MorphologicalRules; + +namespace SIL.Machine.Morphology.HermitCrab; + +[TestFixture] +public class LookupReplayMprTests +{ + private const string Letters = "aeioulmnrstkpd"; + + private string _dir = default!; + + [OneTimeSetUp] + public void SetUp() + { + _dir = Path.Combine(Path.GetTempPath(), "hc-lookup-replay-" + Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(_dir); + } + + [OneTimeTearDown] + public void TearDown() => Directory.Delete(_dir, true); + + [Test] + public void Default_IsOn() + { + Language language = Load(Grammar(rules: Rule("mrSuf", "SUF", Sub("a")), entries: Entry("eA", "tel", ""))); + var morpher = new Morpher(new TraceManager(), language); + Assert.That(morpher.LookupReplayMode, Is.EqualTo(LookupReplayMode.On)); + } + + [Test] + public void RequiredMpr_WrongClassRoot_IsPrunedAndParsesAreUnchanged() + { + Language language = Load( + Grammar( + rules: Rule("mrSuf", "SUF", Sub("a", required: "mprA"), Sub("o", required: "mprB")), + entries: Entry("eA", "tel", "mprA") + Entry("eB", "tel", "mprB") + ) + ); + + (string[] on, int syntheses) = ParseCounting(language, "tela", prune: true); + (string[] off, int synthesesOff) = ParseCounting(language, "tela", prune: false); + + Assert.That(on, Is.EqualTo(off)); + Assert.That(on, Is.EqualTo(new[] { "eA+SUF" })); + Assert.That(synthesesOff, Is.GreaterThan(syntheses)); + Assert.That(syntheses, Is.EqualTo(1)); + } + + [Test] + public void RequiredMpr_AnyGroup_IsSatisfiedByOneFeature() + { + Language language = Load( + Grammar( + groups: Group("mprA mprB", "any", "append"), + rules: Rule("mrSuf", "SUF", Sub("a", required: "mprA mprB")), + entries: Entry("eB", "tel", "mprB") + Entry("eC", "tel", "mprC") + ) + ); + + Assert.That(Verdicts(language, "tela"), Is.EqualTo(new[] { "eB:True", "eC:False" })); + } + + [Test] + public void ExcludedMpr_RootCarriesTheExcludedFeature_IsPruned() + { + Language language = Load( + Grammar( + rules: Rule("mrSuf", "SUF", Sub("a", excluded: "mprA")), + entries: Entry("eA", "tel", "mprA") + Entry("eB", "tel", "mprB") + ) + ); + + Assert.That(Verdicts(language, "tela"), Is.EqualTo(new[] { "eA:False", "eB:True" })); + Assert.That(ParseCounting(language, "tela", prune: true).Parses, Is.EqualTo(new[] { "eB+SUF" })); + } + + [Test] + public void ExcludedMpr_ReachableOverwriteGroup_IsNotUsedToPrune() + { + // mrTheme outputs mprB of an overwrite group, which can remove mprA, so the excluded test is unsafe. + Language language = Load( + Grammar( + groups: Group("mprA mprB", "all", "overwrite"), + rules: Rule("mrTheme", "THEME", Sub("u", outMpr: "mprB")) + + Rule("mrSuf", "SUF", Sub("a", excluded: "mprA")), + entries: Entry("eA", "tel", "mprA"), + order: "mrTheme mrSuf" + ) + ); + + Assert.That(Verdicts(language, "telua"), Is.EqualTo(new[] { "eA:True" })); + Assert.That(ParseCounting(language, "telua", prune: true).Parses, Is.EqualTo(new[] { "eA+THEME+SUF" })); + Assert.That(ParseCounting(language, "telua", prune: false).Parses, Is.EqualTo(new[] { "eA+THEME+SUF" })); + } + + [Test] + public void RequiredMpr_OutputOfAnEarlierRule_IsSeenByALaterRule() + { + Language language = Load( + Grammar( + rules: Rule("mrTheme", "THEME", Sub("u", outMpr: "mprB")) + + Rule("mrSuf", "SUF", Sub("a", required: "mprB")), + entries: Entry("eA", "tel", "mprA"), + order: "mrTheme mrSuf" + ) + ); + + Assert.That(Verdicts(language, "telua"), Is.EqualTo(new[] { "eA:True" })); + Assert.That(ParseCounting(language, "telua", prune: true).Parses, Is.EqualTo(new[] { "eA+THEME+SUF" })); + } + + [Test] + public void RequiredStemName_RootWithAnotherStemName_IsPruned() + { + Language language = Load( + Grammar( + header: HeadFeatures, + stemNames: StemNames, + rules: StemRule("mrPers", "PERS", null, PersOutput, Sub("m")) + + StemRule("mrSuf", "SUF", "snP1", "", Sub("o")), + entries: Entry("eX", "kap", "") + Entry("eY", "kap", "", stemName: "snP1"), + order: "mrPers mrSuf" + ) + ); + + Assert.That(Verdicts(language, "kapmo"), Is.EqualTo(new[] { "eX:False", "eY:True" })); + Assert.That(ParseCounting(language, "kapmo", prune: true).Parses, Is.EqualTo(new[] { "eY+PERS+SUF" })); + Assert.That(ParseCounting(language, "kapmo", prune: false).Parses, Is.EqualTo(new[] { "eY+PERS+SUF" })); + } + + [Test] + public void FamilyRoot_IsNeverPruned() + { + Language language = Load( + Grammar( + families: "f", + rules: Rule("mrSuf", "SUF", Sub("a", required: "mprA")), + entries: Entry("eB", "tel", "mprB", family: "fam") + Entry("eC", "tel", "mprB", family: "fam") + ) + ); + + Assert.That(Verdicts(language, "tela"), Is.EqualTo(new[] { "eB:True", "eC:True" })); + } + + [Test] + public void MergedAlternatives_AreJudgedPerAlternativeAndTheFeasibleOneSurvives() + { + // Both allomorphs insert "a", so with merging on the two analyses of "tela" are merged and later + // expanded. Only the alternative that records the class-A allomorph is feasible for the class-A root. + Language language = Load( + Grammar( + rules: Rule("mrSuf", "SUF", Sub("a", required: "mprB"), Sub("a", required: "mprA")), + entries: Entry("eA", "tel", "mprA") + ) + ); + + foreach (bool merge in new[] { true, false }) + { + var verdicts = new List(); + var morpher = new Morpher(new TraceManager(), language, maxDegreeOfParallelism: 1) + { + MergeEquivalentAnalyses = merge, + LookupReplayMode = LookupReplayMode.Shadow, + LookupReplayVerdict = (w, ok) => verdicts.Add(w.AllomorphApplications[0].Index + ":" + ok), + }; + string[] parses = Signatures(morpher.ParseWord("tela")); + + Assert.That(parses, Is.EqualTo(new[] { "eA+SUF" }), "merge=" + merge); + Assert.That( + verdicts.OrderBy(v => v, StringComparer.Ordinal), + Is.EqualTo(new[] { "0:False", "1:True" }), + "merge=" + merge + ); + + var pruning = new Morpher(new TraceManager(), language, maxDegreeOfParallelism: 1) + { + MergeEquivalentAnalyses = merge, + }; + Assert.That(Signatures(pruning.ParseWord("tela")), Is.EqualTo(parses), "merge=" + merge); + } + } + + [Test] + public void NullRecordedAllomorph_IsPrunedOnlyWhenEveryAllomorphFails() + { + // Guessed (null) allomorphs happen when the analysis came from a rule whose allomorph is unconstrained; + // the predicate must keep the candidate if any allomorph of the rule could still apply. + Language language = Load( + Grammar( + rules: Rule("mrSuf", "SUF", Sub("a", required: "mprA"), Sub("a", required: "mprB")), + entries: Entry("eA", "tel", "mprA") + Entry("eC", "tel", "mprC") + ) + ); + var rule = (AffixProcessRule)language.Strata[0].MorphologicalRules[0]; + + Assert.That(IsFeasible(language, "eA", rule, null), Is.True); + Assert.That(IsFeasible(language, "eC", rule, null), Is.False); + Assert.That(IsFeasible(language, "eA", rule, rule.Allomorphs[1]), Is.False); + } + + [Test] + public void ExcludedMpr_AnyGroup_RejectsOnlyWhenEveryFeatureIsPresent() + { + Language language = Load( + Grammar( + groups: Group("mprA mprB", "any", "append"), + rules: Rule("mrSuf", "SUF", Sub("a", excluded: "mprA mprB")), + entries: Entry("eA", "tel", "mprA") + Entry("eAB", "tel", "mprA mprB") + ) + ); + + Assert.That(Verdicts(language, "tela"), Is.EqualTo(new[] { "eA:True", "eAB:False" })); + Assert.That(ParseCounting(language, "tela", prune: true).Parses, Is.EqualTo(new[] { "eA+SUF" })); + } + + [Test] + public void RequiredMpr_OverwriteRemovesTheFeature_IsNotPrunedAndParsesAreUnchanged() + { + // mrTheme outputs mprB of an overwrite group, which removes mprA, so mrSuf can never apply. The replay + // ignores removals for required checks and keeps the candidate; synthesis then fails the same way. + Language language = Load( + Grammar( + groups: Group("mprA mprB", "all", "overwrite"), + rules: Rule("mrTheme", "THEME", Sub("u", outMpr: "mprB")) + + Rule("mrSuf", "SUF", Sub("a", required: "mprA")), + entries: Entry("eA", "tel", "mprA"), + order: "mrTheme mrSuf" + ) + ); + + Assert.That(Verdicts(language, "telua"), Is.EqualTo(new[] { "eA:True" })); + Assert.That(ParseCounting(language, "telua", prune: true).Parses, Is.Empty); + Assert.That(ParseCounting(language, "telua", prune: false).Parses, Is.Empty); + } + + [Test] + public void ParallelSynthesis_PrunesTheSameCandidatesAndParsesAreUnchanged() + { + Language language = Load( + Grammar( + rules: Rule("mrSuf", "SUF", Sub("a", required: "mprA"), Sub("o", required: "mprB")), + entries: Entry("eA", "tel", "mprA") + Entry("eB", "tel", "mprB") + ) + ); + var verdicts = new System.Collections.Concurrent.ConcurrentBag(); + var shadow = new Morpher(new TraceManager(), language, maxDegreeOfParallelism: 4) + { + LookupReplayMode = LookupReplayMode.Shadow, + LookupReplayVerdict = (w, ok) => verdicts.Add(((LexEntry)w.RootAllomorph.Morpheme).Id + ":" + ok), + }; + shadow.ParseWord("tela").ToList(); + var off = new Morpher(new TraceManager(), language, maxDegreeOfParallelism: 4) + { + LookupReplayMode = LookupReplayMode.Off, + }; + var on = new Morpher(new TraceManager(), language, maxDegreeOfParallelism: 4); + + Assert.That(verdicts.OrderBy(v => v, StringComparer.Ordinal), Is.EqualTo(new[] { "eA:True", "eB:False" })); + Assert.That(Signatures(on.ParseWord("tela")), Is.EqualTo(Signatures(off.ParseWord("tela")))); + Assert.That(Signatures(on.ParseWord("tela")), Is.EqualTo(new[] { "eA+SUF" })); + } + + [Test] + public void ClassGatedSlots_PruneMostWrongClassCandidatesAndParsesAreUnchanged() + { + // Three agreement slots, each with an allomorph per noun class, and one homophonous root per class: only + // the root whose class matches every slot survives, so the other candidates are doomed. + string[] classes = { "mprA", "mprB", "mprC" }; + string Slot(string id, string name, string insert) => + Rule(id, name, classes.Select((c, i) => Sub(insert + new string('a', i), required: c)).ToArray()); + Language language = Load( + Grammar( + rules: Slot("mrOne", "ONE", "t") + Slot("mrTwo", "TWO", "k") + Slot("mrThree", "THR", "p"), + entries: Entry("eA", "tel", "mprA") + Entry("eB", "tel", "mprB") + Entry("eC", "tel", "mprC") + ) + ); + + (string[] on, int syntheses) = ParseCounting(language, "teltkp", prune: true); + (string[] off, int synthesesOff) = ParseCounting(language, "teltkp", prune: false); + + Assert.That(on, Is.EqualTo(off)); + Assert.That(on, Is.Not.Empty); + Assert.That(synthesesOff, Is.GreaterThan(syntheses)); + } + + [Test] + public void FirstViolatedEntry_StopsTheReplay() + { + Language language = Load( + Grammar( + rules: Rule("mrX", "X", Sub("a", required: "mprA")) + Rule("mrY", "Y", Sub("o", required: "mprB")), + entries: Entry("eC", "tel", "mprC") + ) + ); + LexEntry entry = language.Strata[0].Entries.Single(); + var word = new Word(entry.PrimaryAllomorph, FeatureStruct.New().Value); + word.MorphologicalRuleUnapplied((AffixProcessRule)language.Strata[0].MorphologicalRules[0], null); + word.MorphologicalRuleUnapplied((AffixProcessRule)language.Strata[0].MorphologicalRules[1], null); + var replay = new LookupReplay(language); + + Assert.That(replay.IsFeasible(word), Is.False); + // One superset built and one required match, then the replay stops before the other rule. + Assert.That(replay.Work, Is.EqualTo(2)); + } + + [Test] + public void GrammarWithoutConstraints_DoesNoReplayWork() + { + Language language = Load(Grammar(rules: Rule("mrSuf", "SUF", Sub("a")), entries: Entry("eA", "tel", "mprA"))); + var morpher = new Morpher(new TraceManager(), language, maxDegreeOfParallelism: 1); + + Assert.That(Signatures(morpher.ParseWord("tela")), Is.EqualTo(new[] { "eA+SUF" })); + Assert.That(morpher.LookupReplayWork, Is.EqualTo(0)); + } + + private static bool IsFeasible(Language language, string entryId, AffixProcessRule rule, Allomorph? recorded) + { + LexEntry entry = language.Strata[0].Entries.Single(e => e.Id == entryId); + var word = new Word(entry.PrimaryAllomorph, FeatureStruct.New().Value); + word.MorphologicalRuleUnapplied(rule, recorded); + return new LookupReplay(language).IsFeasible(word); + } + + private static (string[] Parses, int Syntheses) ParseCounting(Language language, string word, bool prune) + { + var tm = new TraceManager { IsTracing = true }; + var morpher = new Morpher(tm, language, maxDegreeOfParallelism: 1) + { + LookupReplayMode = prune ? LookupReplayMode.On : LookupReplayMode.Off, + }; + string[] parses = Signatures(morpher.ParseWord(word, out object trace)); + return (parses, CountSynthesisRuleEvents((Trace)trace)); + } + + private static int CountSynthesisRuleEvents(Trace trace) => + (trace.Type == TraceType.MorphologicalRuleSynthesis ? 1 : 0) + trace.Children.Sum(CountSynthesisRuleEvents); + + private static List Verdicts(Language language, string word) + { + var verdicts = new List(); + var morpher = new Morpher(new TraceManager(), language, maxDegreeOfParallelism: 1) + { + LookupReplayMode = LookupReplayMode.Shadow, + LookupReplayVerdict = (w, ok) => verdicts.Add(((LexEntry)w.RootAllomorph.Morpheme).Id + ":" + ok), + }; + morpher.ParseWord(word).ToList(); + verdicts.Sort(StringComparer.Ordinal); + return verdicts; + } + + private static string[] Signatures(IEnumerable words) => + words + .Select(w => string.Join("+", w.AllomorphsInMorphOrder.Select(a => a.Morpheme.Id))) + .OrderBy(s => s, StringComparer.Ordinal) + .ToArray(); + + private Language Load(string xml) + { + string path = Path.Combine(_dir, Guid.NewGuid().ToString("N") + ".xml"); + File.WriteAllText(path, xml); + return XmlLanguageLoader.Load(path); + } + + private const string HeadFeatures = + "pers1"; + + private const string StemNames = + "P1" + + ""; + + private static string Group(string features, string matchType, string output) => + $"g{features.Length}"; + + private static string Sub(string shape, string? required = null, string? excluded = null, string? outMpr = null) + { + string id = "s" + Guid.NewGuid().ToString("N"); + string input = + (required != null ? $" requiredMPRFeatures=\"{required}\"" : "") + + (excluded != null ? $" excludedMPRFeatures=\"{excluded}\"" : ""); + string output = outMpr != null ? $" MPRFeatures=\"{outMpr}\"" : ""; + return $"{shape}"; + } + + private const string PersOutput = + ""; + + private static string Rule(string id, string morphemeId, params string[] subs) => + StemRule(id, morphemeId, null, "", subs); + + private static string StemRule( + string id, + string morphemeId, + string? stemName, + string extra, + params string[] subs + ) => + $"{id}{string.Concat(subs)}{extra}{morphemeId}"; + + private static string Entry( + string id, + string shape, + string features, + string? stemName = null, + string? family = null + ) => + $" 0 ? $" ruleFeatures=\"{features}\"" : "")}{(family != null ? $" family=\"{family}\"" : "")}>{shape}{id}{id}"; + + private static string Grammar( + string rules, + string entries, + string groups = "", + string header = "", + string stemNames = "", + string families = "", + string? order = null + ) + { + string segments = string.Concat( + Letters.Select(c => + $"{c}" + ) + ); + var ruleIds = System + .Text.RegularExpressions.Regex.Matches(rules, " m.Groups[1].Value); + string ruleOrder = order != null ? "linear" : "unordered"; + string ruleList = order ?? string.Join(" ", ruleIds); + return "T" + + "n" + + header + + "" + + string.Concat( + new[] { "mprA", "mprB", "mprC" }.Select(f => + $"{f}" + ) + ) + + (groups.Length > 0 ? groups : "") + + "" + + stemNames + + $"Main{segments}" + + "Any" + + families + + $"S" + + $"{rules}{entries}"; + } +} diff --git a/tests/SIL.Machine.Morphology.HermitCrab.Tests/LookupReplayOutputsTests.cs b/tests/SIL.Machine.Morphology.HermitCrab.Tests/LookupReplayOutputsTests.cs new file mode 100644 index 000000000..e283ef429 --- /dev/null +++ b/tests/SIL.Machine.Morphology.HermitCrab.Tests/LookupReplayOutputsTests.cs @@ -0,0 +1,92 @@ +using NUnit.Framework; +using SIL.Machine.Annotations; +using SIL.Machine.FeatureModel; +using SIL.Machine.Matching; +using SIL.Machine.Morphology.HermitCrab.MorphologicalRules; + +namespace SIL.Machine.Morphology.HermitCrab; + +// Realizational rules fire anywhere and a compounding rule on the trail adds outputs; a required check allows both. +[TestFixture] +public class LookupReplayOutputsTests : HermitCrabTestBase +{ + private static readonly FeatureStruct Any = FeatureStruct.New().Symbol(HCFeatureSystem.Segment).Value; + + private FeatureStruct Pos(string symbol) => FeatureStruct.New(Language.SyntacticFeatureSystem).Symbol(symbol).Value; + + private AffixProcessAllomorph Allomorph(string insert) => + new AffixProcessAllomorph + { + Lhs = { Pattern.New("1").Annotation(Any).OneOrMore.Value }, + Rhs = { new CopyFromInput("1"), new InsertSegments(Table3, insert) }, + }; + + private AffixProcessRule RequiringLatinate() + { + var rule = new AffixProcessRule + { + Id = "need", + Name = "need", + Gloss = "NEED", + RequiredSyntacticFeatureStruct = Pos("N"), + OutSyntacticFeatureStruct = Pos("N"), + }; + AffixProcessAllomorph allo = Allomorph("d"); + allo.RequiredMprFeatures.Add(Latinate); + rule.Allomorphs.Add(allo); + Morphophonemic.MorphologicalRules.Add(rule); + return rule; + } + + private bool IsFeasible(AffixProcessRule rule, IMorphologicalRule? earlier = null) + { + LexEntry entry = AddEntry("root", Pos("N"), Morphophonemic, "zud"); + var word = new Word(entry.PrimaryAllomorph, FeatureStruct.New().Value); + word.MorphologicalRuleUnapplied(rule, null); + if (earlier != null) + word.MorphologicalRuleUnapplied(earlier, null); + return new LookupReplay(Language).IsFeasible(word); + } + + [Test] + public void RequiredFeature_NoRuleCanSupplyIt_IsPruned() + { + Assert.That(IsFeasible(RequiringLatinate()), Is.False); + } + + [Test] + public void RequiredFeature_RealizationalRuleOutputsIt_IsNotPruned() + { + AffixProcessRule rule = RequiringLatinate(); + var real = new RealizationalAffixProcessRule + { + Name = "real", + Gloss = "REAL", + RealizationalFeatureStruct = Pos("N"), + }; + AffixProcessAllomorph realAllo = Allomorph("t"); + realAllo.OutMprFeatures.Add(Latinate); + real.Allomorphs.Add(realAllo); + Morphophonemic.MorphologicalRules.Add(real); + + Assert.That(IsFeasible(rule), Is.True); + } + + [Test] + public void RequiredFeature_CompoundingRuleOnTheTrailOutputsIt_IsNotPruned() + { + AffixProcessRule rule = RequiringLatinate(); + var compound = new CompoundingRule { Name = "compound" }; + var subrule = new CompoundingSubrule + { + HeadLhs = { Pattern.New("head").Annotation(Any).OneOrMore.Value }, + NonHeadLhs = { Pattern.New("nonHead").Annotation(Any).OneOrMore.Value }, + Rhs = { new CopyFromInput("head"), new InsertSegments(Table3, "+"), new CopyFromInput("nonHead") }, + }; + subrule.OutMprFeatures.Add(Latinate); + compound.Subrules.Add(subrule); + Morphophonemic.MorphologicalRules.Add(compound); + + Assert.That(IsFeasible(rule, compound), Is.True); + } +}