From 2e94abb5df284ccc37e3f2093c3de766e523add2 Mon Sep 17 00:00:00 2001 From: "claude[bot]" <41898282+claude[bot]@users.noreply.github.com> Date: Mon, 26 Oct 2026 13:15:25 +0000 Subject: [PATCH] Port 'Fix verse range marker placement bug' from machine.py PR #371 Co-Authored-By: Claude Sonnet 5.5 Co-authored-by: Eli C. Lowry <83078660+Enkidu93@users.noreply.github.com> --- .../PlaceMarkersUsfmUpdateBlockHandler.cs | 145 ++++++++-- .../Corpora/UpdateUsfmParserHandler.cs | 42 +-- src/SIL.Machine/Corpora/UsfmUpdateBlock.cs | 107 +++---- .../Corpora/UsfmUpdateBlockBuilder.cs | 90 ++++++ ...PlaceMarkersUsfmUpdateBlockHandlerTests.cs | 263 +++++++++++++++++- 5 files changed, 523 insertions(+), 124 deletions(-) create mode 100644 src/SIL.Machine/Corpora/UsfmUpdateBlockBuilder.cs diff --git a/src/SIL.Machine/Corpora/PlaceMarkersUsfmUpdateBlockHandler.cs b/src/SIL.Machine/Corpora/PlaceMarkersUsfmUpdateBlockHandler.cs index 5ee415bf1..8e155e1f3 100644 --- a/src/SIL.Machine/Corpora/PlaceMarkersUsfmUpdateBlockHandler.cs +++ b/src/SIL.Machine/Corpora/PlaceMarkersUsfmUpdateBlockHandler.cs @@ -42,23 +42,21 @@ public UsfmUpdateBlock ProcessBlock(UsfmUpdateBlock block) var elements = block.Elements.ToList(); // Nothing to do if there are no markers to place or no alignment to use - if (!block.Metadata.TryGetValue(PlaceMarkersAlignmentInfo.MetadataKey, out object alignmentObject)) - { + if (!elements.Any(e => e.IsPlaceable(UpdateUsfmMarkerBehavior.Preserve, UpdateUsfmMarkerBehavior.Preserve))) return block; - } - if (!(alignmentObject is PlaceMarkersAlignmentInfo alignmentInfo)) - { + + List rowAlignmentInfos = GetRowAlignmentInfos(block); + if (rowAlignmentInfos == null) return block; - } - if ( - elements.Count == 0 - || alignmentInfo.Alignment.RowCount == 0 - || alignmentInfo.Alignment.ColumnCount == 0 - || !elements.Any(e => e.IsPlaceable(alignmentInfo.ParagraphBehavior, alignmentInfo.StyleBehavior)) - ) - { + + // The same behaviors as the merged alignment info, which takes them from the first row + UpdateUsfmMarkerBehavior paragraphBehavior = rowAlignmentInfos[0].ParagraphBehavior; + UpdateUsfmMarkerBehavior styleBehavior = rowAlignmentInfos[0].StyleBehavior; + if (!elements.Any(e => e.IsPlaceable(paragraphBehavior, styleBehavior))) return block; - } + + // The alignment is checked only now, so that a block with nothing to place can't fail on it + PlaceMarkersAlignmentInfo alignmentInfo = GetBlockAlignmentInfo(block, rowAlignmentInfos); // Paragraph markers at the end of the block should stay there // Section headers should be ignored but re-inserted in the same position relative to other paragraph markers @@ -183,6 +181,7 @@ public UsfmUpdateBlock ProcessBlock(UsfmUpdateBlock block) } var toInsert = new List<(int Index, UsfmUpdateBlockElement Element)>(); + var openStyleTargetTokensByMarker = new Dictionary>(); foreach ( (UsfmUpdateBlockElement element, int adjacentSourceToken) in toPlace .Zip(adjacentSourceTokens) @@ -203,12 +202,41 @@ public UsfmUpdateBlock ProcessBlock(UsfmUpdateBlock block) targetTokens ); } + bool isEndStyle = + element.Type == UsfmUpdateBlockElementType.Style + && element.Tokens[0].Marker[element.Tokens[0].Marker.Length - 1] == '*'; + if (element.Type == UsfmUpdateBlockElementType.Style) + { + string marker = element.Tokens[0].Marker; + if (isEndStyle) + { + if ( + openStyleTargetTokensByMarker.TryGetValue( + marker.Substring(0, marker.Length - 1), + out List openingTargetTokens + ) + && openingTargetTokens.Count > 0 + ) + { + // Reordered words can predict a closing marker before its opening one + int openingTargetToken = openingTargetTokens[openingTargetTokens.Count - 1]; + openingTargetTokens.RemoveAt(openingTargetTokens.Count - 1); + adjacentTargetToken = Math.Max(adjacentTargetToken, openingTargetToken + 1); + } + } + else + { + if (!openStyleTargetTokensByMarker.TryGetValue(marker, out List openTokens)) + { + openTokens = new List(); + openStyleTargetTokensByMarker[marker] = openTokens; + } + openTokens.Add(adjacentTargetToken); + } + } + int targetStringIndex; - if ( - adjacentSourceToken > 0 - && element.Type == UsfmUpdateBlockElementType.Style - && element.Tokens[0].Marker[element.Tokens[0].Marker.Length - 1] == '*' - ) + if (isEndStyle && adjacentTargetToken > 0 && adjacentTargetToken <= targetTokens.Count) { targetStringIndex = targetTokenStarts[adjacentTargetToken - 1] + targetTokens[adjacentTargetToken - 1].Length; @@ -282,11 +310,82 @@ public UsfmUpdateBlock ProcessBlock(UsfmUpdateBlock block) headerElements.RemoveAt(0); } - var processedBlock = new UsfmUpdateBlock( - refs: block.Refs, - elements: placedElements.Concat(ignoredElements) + return new UsfmUpdateBlock(block.Refs, placedElements.Concat(ignoredElements), block.Rows); + } + + internal static List GetRowAlignmentInfos(UsfmUpdateBlock block) + { + var infos = new List(); + foreach (UsfmUpdateBlockRow row in block.Rows) + { + // Rows that contributed no text, such as the later verses of a verse range, have nothing to align + if (row.Text.Trim().Length == 0) + continue; + if ( + !row.Metadata.TryGetValue(PlaceMarkersAlignmentInfo.MetadataKey, out object infoObject) + || !(infoObject is PlaceMarkersAlignmentInfo info) + || info.TranslationTokens.Count == 0 + ) + { + return null; + } + infos.Add(info); + } + return infos.Count > 0 ? infos : null; + } + + internal static PlaceMarkersAlignmentInfo GetBlockAlignmentInfo( + UsfmUpdateBlock block, + IReadOnlyList rowAlignmentInfos + ) + { + // Checked before merging, which would shift a pair outside a row's tokens into the next row's tokens + foreach (PlaceMarkersAlignmentInfo info in rowAlignmentInfos) + CheckAlignedWordPairs(block, info); + return rowAlignmentInfos.Count == 1 ? rowAlignmentInfos[0] : MergeAlignmentInfos(rowAlignmentInfos); + } + + private static void CheckAlignedWordPairs(UsfmUpdateBlock block, PlaceMarkersAlignmentInfo info) + { + foreach (AlignedWordPair pair in info.Alignment.ToAlignedWordPairs()) + { + if (pair.SourceIndex >= info.SourceTokens.Count || pair.TargetIndex >= info.TranslationTokens.Count) + { + throw new UsfmUpdateBlockHandlerException( + $"Aligned word pair {pair.SourceIndex}-{pair.TargetIndex} is outside the tokens of its row for {FormatRefs(block)}.", + block + ); + } + } + } + + private static PlaceMarkersAlignmentInfo MergeAlignmentInfos(IReadOnlyList infos) + { + var sourceTokens = new List(); + var translationTokens = new List(); + var wordPairs = new List<(int, int)>(); + foreach (PlaceMarkersAlignmentInfo info in infos) + { + // Offset by token counts, not matrix size, since a matrix may omit unaligned trailing tokens + foreach (AlignedWordPair pair in info.Alignment.ToAlignedWordPairs()) + { + wordPairs.Add((sourceTokens.Count + pair.SourceIndex, translationTokens.Count + pair.TargetIndex)); + } + sourceTokens.AddRange(info.SourceTokens); + translationTokens.AddRange(info.TranslationTokens); + } + return new PlaceMarkersAlignmentInfo( + sourceTokens, + translationTokens, + new WordAlignmentMatrix(sourceTokens.Count, translationTokens.Count, wordPairs), + infos[0].ParagraphBehavior, + infos[0].StyleBehavior ); - return processedBlock; + } + + private static string FormatRefs(UsfmUpdateBlock block) + { + return string.Join(", ", block.Refs.Select(r => r.ToString())); } private int PredictMarkerLocation( diff --git a/src/SIL.Machine/Corpora/UpdateUsfmParserHandler.cs b/src/SIL.Machine/Corpora/UpdateUsfmParserHandler.cs index 1580b07f3..97f93586c 100644 --- a/src/SIL.Machine/Corpora/UpdateUsfmParserHandler.cs +++ b/src/SIL.Machine/Corpora/UpdateUsfmParserHandler.cs @@ -58,7 +58,7 @@ public class UpdateUsfmParserHandler : ScriptureRefUsfmParserHandlerBase private readonly UpdateUsfmMarkerBehavior _embedBehavior; private readonly UpdateUsfmMarkerBehavior _styleBehavior; private readonly HashSet _preserveParagraphStyles; - private readonly Stack _updateBlocks; + private readonly Stack _updateBlocks; private readonly Stack _updateBlockHandlers; private readonly List<(int, string)> _remarks; private readonly Stack _replace; @@ -89,7 +89,7 @@ public UpdateUsfmParserHandler( _updateRowsVersification = GetRowsVersification(rows); _tokens = new List(); _updatedText = new List(); - _updateBlocks = new Stack(); + _updateBlocks = new Stack(); _embedTokens = new List(); _idText = idText; _replace = new Stack(); @@ -137,7 +137,7 @@ public override void StartBook(UsfmParserState state, string marker, string code } CollectReadonlyTokens(state); - _updateBlocks.Push(new UsfmUpdateBlock()); + _updateBlocks.Push(new UsfmUpdateBlockBuilder()); var startBookTokens = new List(); if (_idText != null) startBookTokens.Add(new UsfmToken(_idText + " ")); @@ -150,7 +150,7 @@ public override void EndBook(UsfmParserState state, string marker) { UseUpdatedText(); PopNewTokens(); - UsfmUpdateBlock updateBlock = _updateBlocks.Pop(); + UsfmUpdateBlock updateBlock = _updateBlocks.Pop().Build(); _tokens.AddRange(updateBlock.GetTokens()); base.EndBook(state, marker); @@ -519,12 +519,9 @@ public string GetUsfm(UsfmStylesheet stylesheet) return tokenizer.Detokenize(tokens); } - private (IReadOnlyList RowTexts, Dictionary Metadata) AdvanceRows( - IReadOnlyList segScrRefs - ) + private List AdvanceRows(IReadOnlyList segScrRefs) { - var rowTexts = new List(); - Dictionary rowMetadata = null; + var blockRows = new List(); int sourceIndex = 0; // handle the special case of verse 0, which although first in the rows, @@ -558,8 +555,9 @@ IReadOnlyList segScrRefs { // source and row match // grab the text - both source and row will be incremented in due time... - rowTexts.Add(text); - rowMetadata = metadata.ToDictionary(kvp => kvp.Key, kvp => kvp.Value); + blockRows.Add( + new UsfmUpdateBlockRow(text, metadata.ToDictionary(kvp => kvp.Key, kvp => kvp.Value)) + ); break; } } @@ -569,7 +567,7 @@ IReadOnlyList segScrRefs _verseRowIndex++; } } - return (rowTexts, rowMetadata); + return blockRows; } private void CollectUpdatableTokens(UsfmParserState state) @@ -678,27 +676,29 @@ private bool HasNewText() private void StartUpdateBlock(IReadOnlyList scriptureRefs) { - (IReadOnlyList rowTexts, Dictionary metadata) = AdvanceRows(scriptureRefs); - _updateBlocks.Push( - new UsfmUpdateBlock(scriptureRefs, metadata: metadata ?? new Dictionary()) - ); - PushUpdatedText(rowTexts.Select(t => new UsfmToken(t + " "))); + List blockRows = AdvanceRows(scriptureRefs); + _updateBlocks.Push(new UsfmUpdateBlockBuilder(scriptureRefs, blockRows)); + PushUpdatedText(blockRows.Select(row => new UsfmToken(row.Text + " "))); } private void EndUpdateBlock(UsfmParserState state, IReadOnlyList scriptureRefs) { UseUpdatedText(); PopNewTokens(); - UsfmUpdateBlock updateBlock = _updateBlocks.Pop(); - updateBlock.UpdateRefs(scriptureRefs); + UsfmUpdateBlockBuilder updateBlockBuilder = _updateBlocks.Pop(); + updateBlockBuilder.UpdateRefs(scriptureRefs); // Strip off any non-verse paragraphs that are at the end of the update block var paraElems = new List(); - while (updateBlock.Elements.Count > 0 && IsNonverseParagraph(state, updateBlock.Elements.Last())) + while ( + updateBlockBuilder.Elements.Count > 0 && IsNonverseParagraph(state, updateBlockBuilder.Elements.Last()) + ) { - paraElems.Add(updateBlock.Pop()); + paraElems.Add(updateBlockBuilder.Pop()); } + UsfmUpdateBlock updateBlock = updateBlockBuilder.Build(); + foreach (IUsfmUpdateBlockHandler handler in _updateBlockHandlers) { try diff --git a/src/SIL.Machine/Corpora/UsfmUpdateBlock.cs b/src/SIL.Machine/Corpora/UsfmUpdateBlock.cs index 221407298..89d005f42 100644 --- a/src/SIL.Machine/Corpora/UsfmUpdateBlock.cs +++ b/src/SIL.Machine/Corpora/UsfmUpdateBlock.cs @@ -1,8 +1,36 @@ +using System; using System.Collections.Generic; using System.Linq; namespace SIL.Machine.Corpora { + public class UsfmUpdateBlockRow + { + public string Text { get; } + public IReadOnlyDictionary Metadata { get; } + + public UsfmUpdateBlockRow(string text, IReadOnlyDictionary metadata = null) + { + Text = text; + Metadata = metadata ?? new Dictionary(); + } + + public override bool Equals(object obj) + { + if (!(obj is UsfmUpdateBlockRow other)) + return false; + + return string.Equals(Text, other.Text, StringComparison.Ordinal) + && Metadata.Count == other.Metadata.Count + && !Metadata.Except(other.Metadata).Any(); + } + + public override int GetHashCode() + { + return Text?.GetHashCode() ?? 0; + } + } + public class UsfmUpdateBlock { public IReadOnlyList Refs @@ -13,86 +41,26 @@ public IReadOnlyList Elements { get => _elements; } - public IReadOnlyDictionary Metadata + + // Several rows can match a verse range, and this block's text is their texts concatenated in order + public IReadOnlyList Rows { - get => _metadata; + get => _rows; } private readonly List _refs; private readonly List _elements; - private readonly Dictionary _metadata; + private readonly List _rows; public UsfmUpdateBlock( IEnumerable refs = null, IEnumerable elements = null, - Dictionary metadata = null + IEnumerable rows = null ) { _refs = refs?.ToList() ?? new List(); _elements = elements?.ToList() ?? new List(); - _metadata = metadata ?? new Dictionary(); - } - - public void AddText(IEnumerable tokens) - { - _elements.Add(new UsfmUpdateBlockElement(UsfmUpdateBlockElementType.Text, tokens.ToList())); - } - - public void AddToken(UsfmToken token, bool markedForRemoval = false) - { - UsfmUpdateBlockElementType type; - switch (token.Type) - { - case UsfmTokenType.Text: - type = UsfmUpdateBlockElementType.Text; - break; - case UsfmTokenType.Paragraph: - type = UsfmUpdateBlockElementType.Paragraph; - break; - case UsfmTokenType.Character: - case UsfmTokenType.End: - type = UsfmUpdateBlockElementType.Style; - break; - default: - type = UsfmUpdateBlockElementType.Other; - break; - } - _elements.Add(new UsfmUpdateBlockElement(type, new List { token }, markedForRemoval)); - } - - public void AddEmbed(IEnumerable tokens, bool markedForRemoval = false) - { - _elements.Add( - new UsfmUpdateBlockElement(UsfmUpdateBlockElementType.Embed, tokens.ToList(), markedForRemoval) - ); - } - - public void ExtendLastElement(IEnumerable tokens) - { - _elements.Last().Tokens.AddRange(tokens); - } - - public void UpdateRefs(IEnumerable refs) - { - _refs.Clear(); - _refs.AddRange(refs); - } - - public UsfmUpdateBlockElement GetLastParagraph() - { - foreach (UsfmUpdateBlockElement element in Enumerable.Reverse(_elements)) - { - if (element.Type == UsfmUpdateBlockElementType.Paragraph) - return element; - } - return null; - } - - public UsfmUpdateBlockElement Pop() - { - UsfmUpdateBlockElement element = _elements.Last(); - _elements.RemoveAt(_elements.Count - 1); - return element; + _rows = rows?.ToList() ?? new List(); } public List GetTokens() @@ -109,8 +77,7 @@ public override bool Equals(object obj) return _refs.SequenceEqual(other._refs) && _elements.SequenceEqual(other._elements) - && _metadata.Count == other.Metadata.Count - && !_metadata.Except(other.Metadata).Any(); + && _rows.SequenceEqual(other._rows); } public override int GetHashCode() @@ -123,7 +90,7 @@ public override int GetHashCode() public UsfmUpdateBlock Clone() { - return new UsfmUpdateBlock(_refs, _elements); + return new UsfmUpdateBlock(_refs, _elements, _rows); } } } diff --git a/src/SIL.Machine/Corpora/UsfmUpdateBlockBuilder.cs b/src/SIL.Machine/Corpora/UsfmUpdateBlockBuilder.cs new file mode 100644 index 000000000..1dea8fd14 --- /dev/null +++ b/src/SIL.Machine/Corpora/UsfmUpdateBlockBuilder.cs @@ -0,0 +1,90 @@ +using System.Collections.Generic; +using System.Linq; + +namespace SIL.Machine.Corpora +{ + internal class UsfmUpdateBlockBuilder + { + private readonly List _refs; + private readonly List _elements = new List(); + private readonly List _rows; + + public UsfmUpdateBlockBuilder( + IEnumerable refs = null, + IEnumerable rows = null + ) + { + _refs = refs?.ToList() ?? new List(); + _rows = rows?.ToList() ?? new List(); + } + + public IReadOnlyList Elements => _elements; + + public void AddText(IEnumerable tokens) + { + _elements.Add(new UsfmUpdateBlockElement(UsfmUpdateBlockElementType.Text, tokens.ToList())); + } + + public void AddToken(UsfmToken token, bool markedForRemoval = false) + { + UsfmUpdateBlockElementType type; + switch (token.Type) + { + case UsfmTokenType.Text: + type = UsfmUpdateBlockElementType.Text; + break; + case UsfmTokenType.Paragraph: + type = UsfmUpdateBlockElementType.Paragraph; + break; + case UsfmTokenType.Character: + case UsfmTokenType.End: + type = UsfmUpdateBlockElementType.Style; + break; + default: + type = UsfmUpdateBlockElementType.Other; + break; + } + _elements.Add(new UsfmUpdateBlockElement(type, new List { token }, markedForRemoval)); + } + + public void AddEmbed(IEnumerable tokens, bool markedForRemoval = false) + { + _elements.Add( + new UsfmUpdateBlockElement(UsfmUpdateBlockElementType.Embed, tokens.ToList(), markedForRemoval) + ); + } + + public void ExtendLastElement(IEnumerable tokens) + { + _elements.Last().Tokens.AddRange(tokens); + } + + public void UpdateRefs(IEnumerable refs) + { + _refs.Clear(); + _refs.AddRange(refs); + } + + public UsfmUpdateBlockElement GetLastParagraph() + { + foreach (UsfmUpdateBlockElement element in Enumerable.Reverse(_elements)) + { + if (element.Type == UsfmUpdateBlockElementType.Paragraph) + return element; + } + return null; + } + + public UsfmUpdateBlockElement Pop() + { + UsfmUpdateBlockElement element = _elements.Last(); + _elements.RemoveAt(_elements.Count - 1); + return element; + } + + public UsfmUpdateBlock Build() + { + return new UsfmUpdateBlock(_refs, _elements, _rows); + } + } +} diff --git a/tests/SIL.Machine.Tests/Corpora/PlaceMarkersUsfmUpdateBlockHandlerTests.cs b/tests/SIL.Machine.Tests/Corpora/PlaceMarkersUsfmUpdateBlockHandlerTests.cs index c3218a2e5..ae6a7c5cb 100644 --- a/tests/SIL.Machine.Tests/Corpora/PlaceMarkersUsfmUpdateBlockHandlerTests.cs +++ b/tests/SIL.Machine.Tests/Corpora/PlaceMarkersUsfmUpdateBlockHandlerTests.cs @@ -1001,26 +1001,27 @@ public void UpdateUsfm_UnmatchedEndMarker() [Test] public void UpdateUsfm_MultipleTextRowsInVerseRangesAreUpdated() { - string source = "This is the first part. This is the second part."; - string pretranslation = "Esta es la primera parte. Esta es la segunda parte."; - PlaceMarkersAlignmentInfo alignInfo = new PlaceMarkersAlignmentInfo( - sourceTokens: Tokenizer.Tokenize(source).ToList(), - translationTokens: Tokenizer.Tokenize(pretranslation).ToList(), - alignment: ToWordAlignmentMatrix("0-0 1-1 2-2 3-3 4-4 5-5 6-6 7-7 8-8 9-9 10-10"), - paragraphBehavior: UpdateUsfmMarkerBehavior.Preserve, - styleBehavior: UpdateUsfmMarkerBehavior.Strip + PlaceMarkersAlignmentInfo firstAlignInfo = CreateAlignmentInfo( + "This is the first part.", + "Esta es la primera parte.", + "0-0 1-1 2-2 3-3 4-4 5-5" + ); + PlaceMarkersAlignmentInfo secondAlignInfo = CreateAlignmentInfo( + "This is the second part.", + "Esta es la segunda parte.", + "0-0 1-1 2-2 3-3 4-4 5-5" ); IReadOnlyList rows = [ new UpdateUsfmRow( ScrRef("MAT 1:1"), "Esta es la primera parte.", - new Dictionary { { "alignment_info", alignInfo } } + new Dictionary { { "alignment_info", firstAlignInfo } } ), new UpdateUsfmRow( ScrRef("MAT 1:2"), "Esta es la segunda parte.", - new Dictionary { { "alignment_info", alignInfo } } + new Dictionary { { "alignment_info", secondAlignInfo } } ), ]; string usfm = @@ -1178,6 +1179,248 @@ public void UpdateBlock_AnusvaraTokenization() AssertUsfmEquals(target, result); } + [Test] + public void UpdateUsfm_VerseRangeWithEmptyTrailingRow() + { + PlaceMarkersAlignmentInfo alignInfo = CreateAlignmentInfo( + "This is the first part. This is the second part.", + "Esta es la primera parte. Esta es la segunda parte.", + "0-0 1-1 2-2 3-3 4-4 5-5 6-6 7-7 8-8 9-9 10-10 11-11" + ); + PlaceMarkersAlignmentInfo emptyAlignInfo = new PlaceMarkersAlignmentInfo( + sourceTokens: [], + translationTokens: [], + alignment: ToWordAlignmentMatrix(""), + paragraphBehavior: UpdateUsfmMarkerBehavior.Preserve, + styleBehavior: UpdateUsfmMarkerBehavior.Strip + ); + string usfm = + @"\id MAT +\c 1 +\v 1-2 This is the first part. +\p This is the second part. +"; + string result = + @"\id MAT +\c 1 +\v 1-2 Esta es la primera parte. +\p Esta es la segunda parte. +"; + + foreach ( + UpdateUsfmRow emptyRow in new[] + { + new UpdateUsfmRow( + ScrRef("MAT 1:2"), + "", + new Dictionary { { "alignment_info", emptyAlignInfo } } + ), + new UpdateUsfmRow(ScrRef("MAT 1:2"), ""), + } + ) + { + IReadOnlyList rows = + [ + new UpdateUsfmRow( + ScrRef("MAT 1:1"), + "Esta es la primera parte. Esta es la segunda parte.", + new Dictionary { { "alignment_info", alignInfo } } + ), + emptyRow, + ]; + string target = UpdateUsfm(rows, usfm, usfmUpdateBlockHandlers: [new PlaceMarkersUsfmUpdateBlockHandler()]); + AssertUsfmEquals(target, result); + } + } + + [Test] + public void UpdateUsfm_RowsWithTheSameAlignmentInfoAreEachMerged() + { + PlaceMarkersAlignmentInfo alignInfo = CreateAlignmentInfo("Amen.", "Amén.", "0-0 1-1"); + IReadOnlyList rows = + [ + new UpdateUsfmRow( + ScrRef("MAT 1:1"), + "Amén.", + new Dictionary { { "alignment_info", alignInfo } } + ), + new UpdateUsfmRow( + ScrRef("MAT 1:2"), + "Amén.", + new Dictionary { { "alignment_info", alignInfo } } + ), + ]; + string usfm = + @"\id MAT +\c 1 +\v 1-2 Amen. +\p Amen. +"; + + string target = UpdateUsfm(rows, usfm, usfmUpdateBlockHandlers: [new PlaceMarkersUsfmUpdateBlockHandler()]); + + string result = + @"\id MAT +\c 1 +\v 1-2 Amén. +\p Amén. +"; + AssertUsfmEquals(target, result); + } + + [Test] + public void UpdateUsfm_RowWithoutAlignmentInfoIsNotPlaced() + { + IReadOnlyList rows = + [ + new UpdateUsfmRow( + ScrRef("MAT 1:1"), + "Esta es la primera parte.", + new Dictionary + { + { + "alignment_info", + CreateAlignmentInfo( + "This is the first part.", + "Esta es la primera parte.", + "0-0 1-1 2-2 3-3 4-4 5-5" + ) + }, + } + ), + new UpdateUsfmRow(ScrRef("MAT 1:2"), "Esta es la segunda parte."), + ]; + string usfm = + @"\id MAT +\c 1 +\v 1-2 This is the first part. +\p This is the second part. +"; + + string target = UpdateUsfm(rows, usfm, usfmUpdateBlockHandlers: [new PlaceMarkersUsfmUpdateBlockHandler()]); + + string result = + @"\id MAT +\c 1 +\v 1-2 Esta es la primera parte. Esta es la segunda parte. +\p +"; + AssertUsfmEquals(target, result); + } + + [Test] + public void UpdateUsfm_EndStyleMarkerFollowsStartMarker() + { + PlaceMarkersAlignmentInfo alignInfo = new PlaceMarkersAlignmentInfo( + sourceTokens: Tokenizer.Tokenize("This is the big dog.").ToList(), + translationTokens: Tokenizer.Tokenize("Este es el perro grande.").ToList(), + alignment: ToWordAlignmentMatrix("0-0 1-1 2-2 3-4 4-3 5-5"), + paragraphBehavior: UpdateUsfmMarkerBehavior.Preserve, + styleBehavior: UpdateUsfmMarkerBehavior.Preserve + ); + IReadOnlyList rows = + [ + new UpdateUsfmRow( + ScrRef("MAT 1:1"), + "Este es el perro grande.", + new Dictionary { { "alignment_info", alignInfo } } + ), + ]; + string usfm = + @"\id MAT +\c 1 +\v 1 This is the \w big\w* dog. +"; + + string target = UpdateUsfm( + rows, + usfm, + styleBehavior: UpdateUsfmMarkerBehavior.Preserve, + usfmUpdateBlockHandlers: [new PlaceMarkersUsfmUpdateBlockHandler()] + ); + + string result = + @"\id MAT +\c 1 +\v 1 Este es el \w perro\w* grande. +"; + AssertUsfmEquals(target, result); + } + + [Test] + public void GetBlockAlignmentInfo_ShiftsLaterRowsByTokenCounts() + { + var block = new UsfmUpdateBlock( + ScrRef("MAT 1:1", "MAT 1:2", "MAT 1:3"), + rows: + [ + new UsfmUpdateBlockRow( + "x y .", + new Dictionary + { + { "alignment_info", CreateAlignmentInfo("a b", "x y .", "0-0 1-1") }, + } + ), + new UsfmUpdateBlockRow(""), + new UsfmUpdateBlockRow( + "z w v", + new Dictionary + { + { "alignment_info", CreateAlignmentInfo("c d", "z w v", "0-0 1-2") }, + } + ), + ] + ); + + PlaceMarkersAlignmentInfo info = PlaceMarkersUsfmUpdateBlockHandler.GetBlockAlignmentInfo( + block, + PlaceMarkersUsfmUpdateBlockHandler.GetRowAlignmentInfos(block) + ); + + Assert.That(info.SourceTokens, Is.EqualTo(new[] { "a", "b", "c", "d" })); + Assert.That(info.TranslationTokens, Is.EqualTo(new[] { "x", "y", ".", "z", "w", "v" })); + Assert.That( + info.Alignment.ValueEquals(new WordAlignmentMatrix(4, 6, new[] { (0, 0), (1, 1), (2, 3), (3, 5) })), + Is.True + ); + } + + [Test] + public void GetBlockAlignmentInfo_RejectsAlignedWordPairOutsideRowTokens() + { + var block = new UsfmUpdateBlock( + ScrRef("MAT 1:1"), + rows: + [ + new UsfmUpdateBlockRow( + "x y", + new Dictionary + { + { "alignment_info", CreateAlignmentInfo("a b", "x y", "0-0 1-5") }, + } + ), + ] + ); + + Assert.Throws(() => + PlaceMarkersUsfmUpdateBlockHandler.GetBlockAlignmentInfo( + block, + PlaceMarkersUsfmUpdateBlockHandler.GetRowAlignmentInfos(block) + ) + ); + } + + private static PlaceMarkersAlignmentInfo CreateAlignmentInfo(string source, string translation, string alignment) + { + return new PlaceMarkersAlignmentInfo( + sourceTokens: Tokenizer.Tokenize(source).ToList(), + translationTokens: Tokenizer.Tokenize(translation).ToList(), + alignment: ToWordAlignmentMatrix(alignment), + paragraphBehavior: UpdateUsfmMarkerBehavior.Preserve, + styleBehavior: UpdateUsfmMarkerBehavior.Strip + ); + } + private static ScriptureRef[] ScrRef(params string[] refs) { return refs.Select(r => ScriptureRef.Parse(r)).ToArray();