From eacdd00a1d6d57ecf2929de6a945fa1cd91342ca Mon Sep 17 00:00:00 2001 From: Tim Haasdyk Date: Mon, 26 Oct 2026 10:34:27 +0100 Subject: [PATCH 1/3] Trace sync planning through a public SIL.Harmony ActivitySource GetCommitsMissingFromRemote now runs in a span from the SIL.Harmony source, so a sync server can export it by adding HarmonyActivitySource.ActivitySourceName to its tracer provider. The span gets a harmony.sync.divergence event for each client whose commit sets diverged, tagged with this side's plan. The check is symmetric, so a server records it whenever the client falls back to recovery. The span is tagged harmony.sync.remote_heads_only when the peer sends only ClientHeads and planning compares timestamps alone. Co-Authored-By: Claude Opus 5.5 --- src/SIL.Harmony.Core/HarmonyActivitySource.cs | 10 +++++ src/SIL.Harmony.Core/QueryHelpers.cs | 43 ++++++++++++++++++- 2 files changed, 51 insertions(+), 2 deletions(-) create mode 100644 src/SIL.Harmony.Core/HarmonyActivitySource.cs diff --git a/src/SIL.Harmony.Core/HarmonyActivitySource.cs b/src/SIL.Harmony.Core/HarmonyActivitySource.cs new file mode 100644 index 0000000..5e659cb --- /dev/null +++ b/src/SIL.Harmony.Core/HarmonyActivitySource.cs @@ -0,0 +1,10 @@ +using System.Diagnostics; + +namespace SIL.Harmony.Core; + +public static class HarmonyActivitySource +{ + public const string ActivitySourceName = "SIL.Harmony"; + internal static readonly ActivitySource Source = + new(ActivitySourceName, typeof(HarmonyActivitySource).Assembly.GetName().Version?.ToString()); +} diff --git a/src/SIL.Harmony.Core/QueryHelpers.cs b/src/SIL.Harmony.Core/QueryHelpers.cs index d4294c4..fe7549b 100644 --- a/src/SIL.Harmony.Core/QueryHelpers.cs +++ b/src/SIL.Harmony.Core/QueryHelpers.cs @@ -1,3 +1,4 @@ +using System.Diagnostics; using System.IO.Hashing; using Microsoft.EntityFrameworkCore; @@ -46,11 +47,12 @@ public static async IAsyncEnumerable GetCommitsMissingFromRemote c.ChangeEntities); + using var activity = StartPlanActivity(remoteState); foreach (var localClientState in localState.ClientStates) { var clientCommits = commits.Where(c => c.ClientId == localClientState.ClientId).DefaultOrder(); var remoteClientState = remoteState.GetClientState(localClientState.ClientId); - switch (PlanFor(localClientState, remoteClientState)) + switch (PlanAndRecordDivergence(activity, localClientState, remoteClientState)) { case SyncPlan.SendNothing: break; @@ -98,6 +100,42 @@ private static SyncPlan PlanFor(ClientState local, ClientState? remote) return local.CommitCount >= remote.CommitCount ? SyncPlan.SendAll : SyncPlan.SendNothing; } + private static Activity? StartPlanActivity(SyncState remoteState) + { + var activity = HarmonyActivitySource.Source.StartActivity(nameof(GetCommitsMissingFromRemote)); + if (activity is { IsAllDataRequested: true } && remoteState.ClientStates.Any(c => c.OnlyHasTimestamp)) + activity.SetTag("harmony.sync.remote_heads_only", true); + return activity; + } + + private static SyncPlan PlanAndRecordDivergence(Activity? activity, ClientState local, ClientState? remote) + { + var plan = PlanFor(local, remote); + if (activity is { IsAllDataRequested: true } && remote is { OnlyHasTimestamp: false } && local != remote && HaveDiverged(local, remote)) + RecordDivergence(activity, local, remote, plan); + return plan; + } + + //a side that is just behind has an older head and fewer commits; anything else means the commit sets diverged + private static bool HaveDiverged(ClientState local, ClientState remote) + { + var (ahead, behind) = local.MaxTimestamp >= remote.MaxTimestamp ? (local, remote) : (remote, local); + return ahead.MaxTimestamp == behind.MaxTimestamp || behind.CommitCount >= ahead.CommitCount; + } + + private static void RecordDivergence(Activity activity, ClientState local, ClientState remote, SyncPlan plan) + { + activity.AddEvent(new ActivityEvent("harmony.sync.divergence", tags: new ActivityTagsCollection + { + ["harmony.client_id"] = local.ClientId.ToString(), + ["harmony.plan"] = plan.ToString(), + ["harmony.local.commit_count"] = local.CommitCount, + ["harmony.local.max_timestamp"] = local.MaxTimestamp, + ["harmony.remote.commit_count"] = remote.CommitCount, + ["harmony.remote.max_timestamp"] = remote.MaxTimestamp + })); + } + //the db keeps sub-millisecond precision, so re-check against the millisecond value the remote reported private static bool IsAfter(CommitBase commit, long afterMs) => commit.DateTime.ToUnixTimeMilliseconds() > afterMs; @@ -122,11 +160,12 @@ public static IEnumerable GetCommitsMissingFromRemote SyncState localState, SyncState remoteState) where TCommit : CommitBase { + using var activity = StartPlanActivity(remoteState); foreach (var localClientState in localState.ClientStates) { ClientState? remoteClientState = remoteState.GetClientState(localClientState.ClientId); var clientCommits = commits.Where(c => c.ClientId == localClientState.ClientId).DefaultOrder(); - switch (PlanFor(localClientState, remoteClientState)) + switch (PlanAndRecordDivergence(activity, localClientState, remoteClientState)) { case SyncPlan.SendNothing: break; From 7d0311b8566158a983ae7c814b5633b5a19f03a9 Mon Sep 17 00:00:00 2001 From: Tim Haasdyk Date: Wed, 28 Oct 2026 12:19:50 +0700 Subject: [PATCH 2/3] Record sync recovery from PlanFor itself, with both sides' hashes IsRecovering flags a client when either side's PlanFor falls back to sending its whole history, replacing a separate divergence rule that reached the same answer only together with its caller's check. The event is now harmony.sync.recovery and carries both hashes. Co-Authored-By: Claude Opus 5.5 --- src/SIL.Harmony.Core/QueryHelpers.cs | 29 ++++++++++++++-------------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/src/SIL.Harmony.Core/QueryHelpers.cs b/src/SIL.Harmony.Core/QueryHelpers.cs index fe7549b..593de20 100644 --- a/src/SIL.Harmony.Core/QueryHelpers.cs +++ b/src/SIL.Harmony.Core/QueryHelpers.cs @@ -52,7 +52,7 @@ public static async IAsyncEnumerable GetCommitsMissingFromRemote c.ClientId == localClientState.ClientId).DefaultOrder(); var remoteClientState = remoteState.GetClientState(localClientState.ClientId); - switch (PlanAndRecordDivergence(activity, localClientState, remoteClientState)) + switch (PlanAndRecordRecovery(activity, localClientState, remoteClientState)) { case SyncPlan.SendNothing: break; @@ -108,31 +108,32 @@ private static SyncPlan PlanFor(ClientState local, ClientState? remote) return activity; } - private static SyncPlan PlanAndRecordDivergence(Activity? activity, ClientState local, ClientState? remote) + private static SyncPlan PlanAndRecordRecovery(Activity? activity, ClientState local, ClientState? remote) { var plan = PlanFor(local, remote); - if (activity is { IsAllDataRequested: true } && remote is { OnlyHasTimestamp: false } && local != remote && HaveDiverged(local, remote)) - RecordDivergence(activity, local, remote, plan); + if (activity is { IsAllDataRequested: true } && remote is { OnlyHasTimestamp: false } && IsRecovering(local, remote)) + RecordRecovery(activity, local, remote, plan); return plan; } - //a side that is just behind has an older head and fewer commits; anything else means the commit sets diverged - private static bool HaveDiverged(ClientState local, ClientState remote) - { - var (ahead, behind) = local.MaxTimestamp >= remote.MaxTimestamp ? (local, remote) : (remote, local); - return ahead.MaxTimestamp == behind.MaxTimestamp || behind.CommitCount >= ahead.CommitCount; - } + //a side sends its whole history for this client only when PlanFor falls back to recovery + private static bool IsRecovering(ClientState local, ClientState remote) => + local != remote + && (PlanFor(local, remote) == SyncPlan.SendAll || PlanFor(remote, local) == SyncPlan.SendAll); - private static void RecordDivergence(Activity activity, ClientState local, ClientState remote, SyncPlan plan) + private static void RecordRecovery(Activity activity, ClientState local, ClientState remote, SyncPlan plan) { - activity.AddEvent(new ActivityEvent("harmony.sync.divergence", tags: new ActivityTagsCollection + activity.AddEvent(new ActivityEvent("harmony.sync.recovery", tags: new ActivityTagsCollection { ["harmony.client_id"] = local.ClientId.ToString(), ["harmony.plan"] = plan.ToString(), ["harmony.local.commit_count"] = local.CommitCount, ["harmony.local.max_timestamp"] = local.MaxTimestamp, + //OTLP has no unsigned 64-bit attribute type + ["harmony.local.hash"] = local.Hash.ToString("x16"), ["harmony.remote.commit_count"] = remote.CommitCount, - ["harmony.remote.max_timestamp"] = remote.MaxTimestamp + ["harmony.remote.max_timestamp"] = remote.MaxTimestamp, + ["harmony.remote.hash"] = remote.Hash.ToString("x16") })); } @@ -165,7 +166,7 @@ public static IEnumerable GetCommitsMissingFromRemote { ClientState? remoteClientState = remoteState.GetClientState(localClientState.ClientId); var clientCommits = commits.Where(c => c.ClientId == localClientState.ClientId).DefaultOrder(); - switch (PlanAndRecordDivergence(activity, localClientState, remoteClientState)) + switch (PlanAndRecordRecovery(activity, localClientState, remoteClientState)) { case SyncPlan.SendNothing: break; From 8d31c427f0cf0702520dea73751257cda08d4215 Mon Sep 17 00:00:00 2001 From: Tim Haasdyk Date: Wed, 28 Oct 2026 12:36:08 +0700 Subject: [PATCH 3/3] Plan every client inside the span and end it before yielding commits Planning only reads the two sync states, so it now runs up front in a PlanClients span that ends before either iterator yields. Activity.Current no longer leaks to a caller between items or drops out after the first yield, and the span measures planning rather than streaming. Co-Authored-By: Claude Opus 5.5 --- src/SIL.Harmony.Core/QueryHelpers.cs | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/src/SIL.Harmony.Core/QueryHelpers.cs b/src/SIL.Harmony.Core/QueryHelpers.cs index 593de20..4aa78ba 100644 --- a/src/SIL.Harmony.Core/QueryHelpers.cs +++ b/src/SIL.Harmony.Core/QueryHelpers.cs @@ -47,12 +47,10 @@ public static async IAsyncEnumerable GetCommitsMissingFromRemote c.ChangeEntities); - using var activity = StartPlanActivity(remoteState); - foreach (var localClientState in localState.ClientStates) + foreach (var (localClientState, remoteClientState, plan) in PlanClients(localState, remoteState)) { var clientCommits = commits.Where(c => c.ClientId == localClientState.ClientId).DefaultOrder(); - var remoteClientState = remoteState.GetClientState(localClientState.ClientId); - switch (PlanAndRecordRecovery(activity, localClientState, remoteClientState)) + switch (plan) { case SyncPlan.SendNothing: break; @@ -100,9 +98,20 @@ private static SyncPlan PlanFor(ClientState local, ClientState? remote) return local.CommitCount >= remote.CommitCount ? SyncPlan.SendAll : SyncPlan.SendNothing; } + //planning only reads the two states, so the span ends before either iterator yields and Activity.Current never crosses a yield + private static (ClientState Local, ClientState? Remote, SyncPlan Plan)[] PlanClients(SyncState localState, SyncState remoteState) + { + using var activity = StartPlanActivity(remoteState); + return [.. localState.ClientStates.Select(local => + { + var remote = remoteState.GetClientState(local.ClientId); + return (local, remote, PlanAndRecordRecovery(activity, local, remote)); + })]; + } + private static Activity? StartPlanActivity(SyncState remoteState) { - var activity = HarmonyActivitySource.Source.StartActivity(nameof(GetCommitsMissingFromRemote)); + var activity = HarmonyActivitySource.Source.StartActivity(nameof(PlanClients)); if (activity is { IsAllDataRequested: true } && remoteState.ClientStates.Any(c => c.OnlyHasTimestamp)) activity.SetTag("harmony.sync.remote_heads_only", true); return activity; @@ -161,12 +170,10 @@ public static IEnumerable GetCommitsMissingFromRemote SyncState localState, SyncState remoteState) where TCommit : CommitBase { - using var activity = StartPlanActivity(remoteState); - foreach (var localClientState in localState.ClientStates) + foreach (var (localClientState, remoteClientState, plan) in PlanClients(localState, remoteState)) { - ClientState? remoteClientState = remoteState.GetClientState(localClientState.ClientId); var clientCommits = commits.Where(c => c.ClientId == localClientState.ClientId).DefaultOrder(); - switch (PlanAndRecordRecovery(activity, localClientState, remoteClientState)) + switch (plan) { case SyncPlan.SendNothing: break;