diff --git a/GVFS/GVFS.Common/GitStatusCache.cs b/GVFS/GVFS.Common/GitStatusCache.cs index 369adda87..3ac34f287 100644 --- a/GVFS/GVFS.Common/GitStatusCache.cs +++ b/GVFS/GVFS.Common/GitStatusCache.cs @@ -2,6 +2,7 @@ using GVFS.Common.NamedPipes; using GVFS.Common.Tracing; using System; +using System.Collections.Generic; using System.ComponentModel; using System.IO; @@ -57,6 +58,7 @@ public class GitStatusCache : IDisposable private readonly Lock hydrationStatisticsLock = new Lock(); private Func projectedFolderCountProvider; + private Func> modifiedPathsProvider; private volatile CacheState cacheState = CacheState.Dirty; @@ -97,6 +99,12 @@ public void SetProjectedFolderCountProvider(Func provider) this.projectedFolderCountProvider = provider; } + public void SetModifiedPathsProvider(Func> provider) + { + ArgumentNullException.ThrowIfNull(provider); + this.modifiedPathsProvider = provider; + } + public virtual void Initialize() { this.isInitialized = true; @@ -491,9 +499,10 @@ internal void UpdateHydrationSummary() EnlistmentHydrationSummary hydrationSummary = this.CreateHydrationSummary(diagnostics); NamedPipeMessages.HydrationStatus.CacheState state = hydrationSummary.IsValid - ? diagnostics.CountsComplete ? NamedPipeMessages.HydrationStatus.CacheState.Available : NamedPipeMessages.HydrationStatus.CacheState.IncompleteData + ? NamedPipeMessages.HydrationStatus.CacheState.Available : diagnostics.Outcome == HydrationSummaryOutcome.Cancelled ? NamedPipeMessages.HydrationStatus.CacheState.Cancelled : diagnostics.Outcome == HydrationSummaryOutcome.Failed ? NamedPipeMessages.HydrationStatus.CacheState.Failed + : diagnostics.Outcome == HydrationSummaryOutcome.IncompleteData ? NamedPipeMessages.HydrationStatus.CacheState.IncompleteData : NamedPipeMessages.HydrationStatus.CacheState.InvalidCounts; this.hydrationCache = new HydrationCacheSnapshot(hydrationSummary, state, invalidationVersion); EventMetadata metadata = new EventMetadata(); @@ -557,7 +566,7 @@ internal virtual HydrationStatusCircuitBreaker CreateHydrationCircuitBreaker() internal virtual EnlistmentHydrationSummary CreateHydrationSummary(HydrationSummaryDiagnostics diagnostics) { return EnlistmentHydrationSummary.CreateSummary(this.context.Enlistment, this.context.FileSystem, this.context.Tracer, - this.projectedFolderCountProvider, this.shutdownTokenSource.Token, diagnostics); + this.projectedFolderCountProvider, new EnlistmentPathData(this.modifiedPathsProvider), this.shutdownTokenSource.Token, diagnostics); } private void RecordHydrationSkip(NamedPipeMessages.HydrationStatus.CacheState state) diff --git a/GVFS/GVFS.Common/HealthCalculator/EnlistmentHydrationSummary.cs b/GVFS/GVFS.Common/HealthCalculator/EnlistmentHydrationSummary.cs index 9c605af1d..5d3ea16ea 100644 --- a/GVFS/GVFS.Common/HealthCalculator/EnlistmentHydrationSummary.cs +++ b/GVFS/GVFS.Common/HealthCalculator/EnlistmentHydrationSummary.cs @@ -1,4 +1,5 @@ using GVFS.Common.FileSystem; +using GVFS.Common.NamedPipes; using GVFS.Common.Tracing; using System; using System.Diagnostics; @@ -18,33 +19,32 @@ public class EnlistmentHydrationSummary public Exception Error { get; private set; } = null; public HydrationSummaryDiagnostics Diagnostics { get; private set; } - public int HydratedFileCount => PlaceholderFileCount + ModifiedFileCount; - public int HydratedFolderCount => PlaceholderFolderCount + ModifiedFolderCount; + public long HydratedFileCount => (long)this.PlaceholderFileCount + this.ModifiedFileCount; + public long HydratedFolderCount => (long)this.PlaceholderFolderCount + this.ModifiedFolderCount; + public bool IsValid => this.Diagnostics?.CountsComplete == true && this.ToResponse().IsValid; - public bool IsValid + public string ToMessage() { - get + if (!this.IsValid) { - return PlaceholderFileCount >= 0 - && PlaceholderFolderCount >= 0 - && ModifiedFileCount >= 0 - && ModifiedFolderCount >= 0 - && TotalFileCount >= HydratedFileCount - && TotalFolderCount >= HydratedFolderCount; + return "Error calculating hydration summary. Run 'gvfs health' at the repository root for hydration status details."; } + + return this.ToResponse().ToDisplayMessage(); } - public string ToMessage() + public NamedPipeMessages.HydrationStatus.Response ToResponse() { - if (!IsValid) + return new NamedPipeMessages.HydrationStatus.Response { - return "Error calculating hydration summary. Run 'gvfs health' at the repository root for hydration status details."; - } - - int fileHydrationPercent = TotalFileCount == 0 ? 0 : (int)((100L * HydratedFileCount) / TotalFileCount); - int folderHydrationPercent = TotalFolderCount == 0 ? 0 : (int)((100L * HydratedFolderCount) / TotalFolderCount); - return $"{fileHydrationPercent}% of files and {folderHydrationPercent}% of folders hydrated. Run 'gvfs health' at the repository root for details."; + PlaceholderFileCount = this.PlaceholderFileCount, + PlaceholderFolderCount = this.PlaceholderFolderCount, + ModifiedFileCount = this.ModifiedFileCount, + ModifiedFolderCount = this.ModifiedFolderCount, + TotalFileCount = this.TotalFileCount, + TotalFolderCount = this.TotalFolderCount, + }; } public static EnlistmentHydrationSummary CreateSummary( @@ -96,14 +96,10 @@ internal static EnlistmentHydrationSummary CreateSummary( diagnostics.FinishPhase(diagnostics.ModifiedPathsState == HydrationDataState.NotStarted ? HydrationDataState.Complete : diagnostics.ModifiedPathsState); cancellationToken.ThrowIfCancellationRequested(); - // Preserve the early-exit check without reporting its temporary folder total as an observation. - summary.TotalFolderCount = summary.HydratedFolderCount + 1; - if (!summary.IsValid) + if (summary.TotalFileCount < 0) { - summary.TotalFolderCount = 0; diagnostics.EarlyExit = true; - diagnostics.Outcome = diagnostics.IndexState == HydrationDataState.Failed - ? HydrationSummaryOutcome.IncompleteData : HydrationSummaryOutcome.InvalidCounts; + diagnostics.Outcome = HydrationSummaryOutcome.IncompleteData; return summary; } @@ -112,8 +108,8 @@ internal static EnlistmentHydrationSummary CreateSummary( summary.TotalFolderCount = projectedFolderCountProvider(); diagnostics.TotalFolderCount = summary.TotalFolderCount; diagnostics.FinishPhase(HydrationDataState.Complete); - diagnostics.Outcome = !summary.IsValid || (diagnostics.GetValidationFailures() & ~HydrationValidationFailure.MissingCounts) != HydrationValidationFailure.None ? HydrationSummaryOutcome.InvalidCounts - : diagnostics.CountsComplete ? HydrationSummaryOutcome.Success : HydrationSummaryOutcome.IncompleteData; + diagnostics.Outcome = !diagnostics.CountsComplete ? HydrationSummaryOutcome.IncompleteData + : summary.IsValid ? HydrationSummaryOutcome.Success : HydrationSummaryOutcome.InvalidCounts; return summary; } catch (OperationCanceledException e) diff --git a/GVFS/GVFS.Common/HealthCalculator/EnlistmentPathData.cs b/GVFS/GVFS.Common/HealthCalculator/EnlistmentPathData.cs index ed76e811d..5d8f30bca 100644 --- a/GVFS/GVFS.Common/HealthCalculator/EnlistmentPathData.cs +++ b/GVFS/GVFS.Common/HealthCalculator/EnlistmentPathData.cs @@ -12,6 +12,8 @@ namespace GVFS.Common { public class EnlistmentPathData { + private readonly Func> modifiedPathsProvider; + public List GitFolderPaths; public List GitFilePaths; public List PlaceholderFolderPaths; @@ -21,7 +23,13 @@ public class EnlistmentPathData public List GitTrackingPaths; public EnlistmentPathData() + : this(null) + { + } + + public EnlistmentPathData(Func> modifiedPathsProvider) { + this.modifiedPathsProvider = modifiedPathsProvider; this.GitFolderPaths = new List(); this.GitFilePaths = new List(); this.PlaceholderFolderPaths = new List(); @@ -115,6 +123,18 @@ public void LoadPathsFromGitIndex(GVFSEnlistment enlistment) public virtual void LoadModifiedPaths(GVFSEnlistment enlistment, ITracer tracer, HydrationSummaryDiagnostics diagnostics = null) { + if (this.modifiedPathsProvider != null) + { + // Match the pipe response's empty-entry filtering. + this.AddModifiedPaths(this.modifiedPathsProvider().Where(path => !string.IsNullOrEmpty(path))); + if (diagnostics != null) + { + diagnostics.ModifiedPathsState = HydrationDataState.Complete; + } + + return; + } + if (TryLoadModifiedPathsFromPipe(enlistment, tracer, diagnostics)) { if (diagnostics != null) diff --git a/GVFS/GVFS.Common/HealthCalculator/HydrationSummaryDiagnostics.cs b/GVFS/GVFS.Common/HealthCalculator/HydrationSummaryDiagnostics.cs index eecbb7c0c..491b52ea8 100644 --- a/GVFS/GVFS.Common/HealthCalculator/HydrationSummaryDiagnostics.cs +++ b/GVFS/GVFS.Common/HealthCalculator/HydrationSummaryDiagnostics.cs @@ -168,6 +168,8 @@ public HydrationValidationFailure GetValidationFailures() public EventMetadata GetMetadata() { + long? fileCountDelta = (long?)this.PlaceholderFileCount + this.ModifiedFileCount - this.TotalFileCount; + long? folderCountDelta = (long?)this.PlaceholderFolderCount + this.ModifiedFolderCount - this.TotalFolderCount; return new EventMetadata { ["SchemaVersion"] = 1, @@ -187,6 +189,11 @@ public EventMetadata GetMetadata() [nameof(this.TotalFolderCount)] = this.TotalFolderCount, ["HydratedFileCount"] = (long?)this.PlaceholderFileCount + this.ModifiedFileCount, ["HydratedFolderCount"] = (long?)this.PlaceholderFolderCount + this.ModifiedFolderCount, + ["FileCountSource"] = "IndexHeader", + ["FileCountDelta"] = fileCountDelta, + ["FolderCountDelta"] = folderCountDelta, + ["FileCountExcess"] = fileCountDelta.HasValue ? Math.Max(0, fileCountDelta.Value) : null, + ["FolderCountExcess"] = folderCountDelta.HasValue ? Math.Max(0, folderCountDelta.Value) : null, [nameof(this.IndexState)] = this.IndexState, [nameof(this.PlaceholderState)] = this.PlaceholderState, [nameof(this.ModifiedPathsState)] = this.ModifiedPathsState, diff --git a/GVFS/GVFS.Common/NamedPipes/HydrationStatusNamedPipeMessages.cs b/GVFS/GVFS.Common/NamedPipes/HydrationStatusNamedPipeMessages.cs index 53259fc2c..9e43b0848 100644 --- a/GVFS/GVFS.Common/NamedPipes/HydrationStatusNamedPipeMessages.cs +++ b/GVFS/GVFS.Common/NamedPipes/HydrationStatusNamedPipeMessages.cs @@ -31,6 +31,7 @@ public enum ParseOutcome TooFewFields, InvalidNumber, InvalidCounts, + IncompleteData, } public sealed class CacheInfo @@ -95,16 +96,20 @@ public class Response public int TotalFileCount { get; set; } public int TotalFolderCount { get; set; } - public int HydratedFileCount => this.PlaceholderFileCount + this.ModifiedFileCount; - public int HydratedFolderCount => this.PlaceholderFolderCount + this.ModifiedFolderCount; + public long HydratedFileCount => (long)this.PlaceholderFileCount + this.ModifiedFileCount; + public long HydratedFolderCount => (long)this.PlaceholderFolderCount + this.ModifiedFolderCount; + public long FileCountDelta => this.HydratedFileCount - this.TotalFileCount; + public long FolderCountDelta => this.HydratedFolderCount - this.TotalFolderCount; + public long FileCountExcess => Math.Max(0, this.FileCountDelta); + public long FolderCountExcess => Math.Max(0, this.FolderCountDelta); public bool IsValid => this.PlaceholderFileCount >= 0 && this.PlaceholderFolderCount >= 0 && this.ModifiedFileCount >= 0 && this.ModifiedFolderCount >= 0 - && this.TotalFileCount >= this.HydratedFileCount - && this.TotalFolderCount >= this.HydratedFolderCount; + && this.TotalFileCount >= 0 + && this.TotalFolderCount >= 0; public string ToDisplayMessage() { @@ -113,9 +118,9 @@ public string ToDisplayMessage() return null; } - int filePercent = this.TotalFileCount == 0 ? 0 : (int)((100L * this.HydratedFileCount) / this.TotalFileCount); - int folderPercent = this.TotalFolderCount == 0 ? 0 : (int)((100L * this.HydratedFolderCount) / this.TotalFolderCount); - return $"{filePercent}% of files and {folderPercent}% of folders hydrated. Run 'gvfs health' at the repository root for details."; + return $"Estimated hydration: files {FormatEstimate(this.HydratedFileCount, this.TotalFileCount)}; " + + $"folders {FormatEstimate(this.HydratedFolderCount, this.TotalFolderCount)}. " + + "Run 'gvfs health' at the repository root for details."; } public string ToBody(CacheInfo info = null) @@ -172,8 +177,32 @@ public static bool TryParse(string body, out Response response, out ParseOutcome TotalFolderCount = totalFolderCount, }; - outcome = response.IsValid ? ParseOutcome.Success : ParseOutcome.InvalidCounts; - return response.IsValid; + if (!response.IsValid) + { + outcome = ParseOutcome.InvalidCounts; + return false; + } + + if (CacheInfo.TryParse(body, out CacheInfo info, hasCounts: true) && !info.CountsComplete) + { + outcome = ParseOutcome.IncompleteData; + return false; + } + + outcome = ParseOutcome.Success; + return true; + } + + private static string FormatEstimate(long hydratedCount, int totalCount) + { + if (totalCount == 0) + { + return hydratedCount == 0 ? "0%" : "n/a (no current denominator)"; + } + + return hydratedCount > totalCount + ? "100% (counts exceed current total)" + : ((100L * hydratedCount) / totalCount).ToString(CultureInfo.InvariantCulture) + "%"; } } } diff --git a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/HealthTests.cs b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/HealthTests.cs index f8c122861..befb2238e 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/HealthTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/HealthTests.cs @@ -9,6 +9,7 @@ using System.IO; using System.Text.RegularExpressions; using System.Linq; +using System.Text; using System.Text.Json; using System.Threading; @@ -154,7 +155,7 @@ public void StatusRecordsTheDisplayedResultAndFallbackDiagnostics(int backoffSec Assert.IsTrue(cache.GetProperty("DiagnosticsAvailable").GetBoolean()); cache.GetProperty("State").GetString().ShouldEqual("FeatureDisabled"); string displayOutcome = root.GetProperty("DisplayOutcome").GetString(); - output.ShouldContain(displayOutcome == "Percentage" ? "% of files and " : "Error calculating hydration summary."); + output.ShouldContain(displayOutcome == "Percentage" ? "Estimated hydration: " : "Error calculating hydration summary."); root.GetProperty("ResultSource").GetString().ShouldEqual("Fallback"); string calculation = lines.Single(line => line.Contains("HydrationSummaryDuration ")); using (JsonDocument calculationDocument = JsonDocument.Parse(calculation.Substring(calculation.IndexOf('{')))) @@ -197,12 +198,197 @@ public void CachedStatusRecordsThePublishedCalculationIdentity() root.GetProperty("CalculationId").GetString().ShouldEqual(cache.GetProperty("CalculationId").GetString()); root.GetProperty("Outcome").GetString().ShouldEqual("Success"); root.GetProperty("CalculationOutcome").GetString().ShouldEqual("Success"); - output.ShouldContain("% of files and "); + output.ShouldContain("Estimated hydration: "); + } + + [TestCase(true), Order(8)] + [TestCase(false)] + public void StatusToleratesRetainedFoldersAndNewFilesAfterBranchSwitch(bool cached) + { + string originalCommit = this.RunGit("rev-parse HEAD").Trim(); + string prefix = cached ? "hydration-cached" : "hydration-fallback"; + string retainedFolderName = prefix + "-old-retained"; + string removedFolderName = prefix + "-old-clean"; + string currentFolderName = prefix + "-current"; + string retainedFolder = Path.Combine(this.Enlistment.RepoRoot, retainedFolderName); + string removedFolder = Path.Combine(this.Enlistment.RepoRoot, removedFolderName); + string newFolder = Path.Combine(this.Enlistment.RepoRoot, prefix + "-untracked"); + string untrackedFile = Path.Combine(retainedFolder, "untracked.txt"); + string[] newFiles = Enumerable.Range(0, 3).Select(i => Path.Combine(this.Enlistment.RepoRoot, prefix + "-new-" + i + ".txt")).ToArray(); + bool oldBranchCreated = false; + bool currentBranchCreated = false; + bool testCompleted = false; + + void Cleanup(Action action) + { + try + { + action(); + } + catch (Exception exception) when (!testCompleted + && (exception is IOException || exception is UnauthorizedAccessException || exception is AssertionException)) + { + TestContext.Error.WriteLine("Hydration test cleanup failed: " + exception); + } + } + + try + { + string blob = this.RunGitWithInput("hash-object -w --stdin", "tracked\n"); + string subtree = this.RunGitWithInput("mktree", "100644 blob " + blob + "\ttracked.txt\n"); + string oldTree = this.RunGitWithInput("mktree", + "040000 tree " + subtree + "\t" + removedFolderName + "\n040000 tree " + subtree + "\t" + retainedFolderName + "\n"); + string newTree = this.RunGitWithInput("mktree", "040000 tree " + subtree + "\t" + currentFolderName + "\n"); + string oldCommit = this.RunGitWithInput("commit-tree " + oldTree + " -p " + originalCommit, "Old hydration population\n"); + string newCommit = this.RunGitWithInput("commit-tree " + newTree + " -p " + oldCommit, "Current hydration population\n"); + this.RunGit("branch hydration-count-old " + oldCommit); + oldBranchCreated = true; + this.RunGit("branch hydration-count-current " + newCommit); + currentBranchCreated = true; + this.RunGit("checkout -f hydration-count-old"); + File.ReadAllText(Path.Combine(retainedFolder, "tracked.txt")).ShouldEqual("tracked\n"); + File.ReadAllText(Path.Combine(removedFolder, "tracked.txt")).ShouldEqual("tracked\n"); + File.WriteAllText(untrackedFile, "keep this folder"); + this.Enlistment.WaitForBackgroundOperations(); + + this.RunGit("checkout hydration-count-current"); + File.ReadAllText(Path.Combine(this.Enlistment.RepoRoot, currentFolderName, "tracked.txt")).ShouldEqual("tracked\n"); + Directory.CreateDirectory(newFolder); + File.WriteAllText(Path.Combine(newFolder, "new.txt"), "new untracked folder"); + foreach (string file in newFiles) + { + File.WriteAllText(file, "new untracked file"); + } + + this.Enlistment.WaitForBackgroundOperations(); + File.ReadAllText(untrackedFile).ShouldEqual("keep this folder"); + Assert.IsFalse(Directory.Exists(removedFolder), "A clean obsolete folder must be removed, unlike the retained folder."); + string database = Path.Combine(this.Enlistment.DotGVFSRoot, "databases", "VFSForGit.sqlite"); + string placeholders = GVFSHelpers.GetAllSQLitePlaceholdersAsString(database); + placeholders.ShouldContain(retainedFolderName + GVFSHelpers.PlaceholderFieldDelimiter); + Assert.That(placeholders, Does.Not.Contain(removedFolderName + GVFSHelpers.PlaceholderFieldDelimiter)); + + this.RemountWithHydrationStatus(cached, backoffSeconds: 0); + JsonElement completion = this.WaitForHydrationCacheState(cached ? "Available" : "FeatureDisabled", out string output, out string[] lines); + completion.GetProperty("Outcome").GetString().ShouldEqual("Success"); + completion.GetProperty("CalculationOutcome").GetString().ShouldEqual("Success"); + completion.GetProperty("ResultSource").GetString().ShouldEqual(cached ? "Cache" : "Fallback"); + output.ShouldContain("Estimated hydration: files 100% (counts exceed current total); folders 100% (counts exceed current total)"); + JsonElement result = completion.GetProperty(cached ? "Cache" : "Summary"); + JsonElement counts = cached ? result.GetProperty("Counts") : result; + counts.GetProperty("TotalFileCount").GetInt32().ShouldEqual(1); + counts.GetProperty("TotalFolderCount").GetInt32().ShouldEqual(1); + Assert.That(result.GetProperty("FileCountExcess").GetInt64(), Is.GreaterThan(0)); + Assert.That(result.GetProperty("FolderCountExcess").GetInt64(), Is.GreaterThan(0)); + result.GetProperty("FolderCountSource").GetString().ShouldEqual(cached ? "Projection" : "IndexParser"); + Assert.IsTrue(result.GetProperty("CountsComplete").GetBoolean()); + if (cached) + { + int calculations = this.CountMountHydrationCalculations(); + string calculationId = completion.GetProperty("CalculationId").GetString(); + for (int i = 0; i < 3; i++) + { + HashSet previous = new HashSet(Directory.GetFiles(this.Enlistment.GVFSLogsRoot, "gvfs_health_*.log")); + this.Enlistment.Health(statusOnly: true).ShouldEqual(output); + using (JsonDocument repeated = this.ReadHealthCompletion(previous, out string[] repeatedLines)) + { + repeated.RootElement.GetProperty("ResultSource").GetString().ShouldEqual("Cache"); + repeated.RootElement.GetProperty("CalculationId").GetString().ShouldEqual(calculationId); + Assert.IsFalse(repeatedLines.Any(line => line.Contains("HydrationSummaryDuration "))); + } + } + + this.CountMountHydrationCalculations().ShouldEqual(calculations); + Stopwatch hookDeadline = Stopwatch.StartNew(); + string hookOutput; + do + { + ProcessResult status = GitProcess.InvokeProcess(this.Enlistment.RepoRoot, "status"); + status.ExitCode.ShouldEqual(0); + hookOutput = status.Output + status.Errors; + } + while (!hookOutput.Contains("Estimated hydration: ") && hookDeadline.Elapsed < TimeSpan.FromSeconds(30)); + + hookOutput.ShouldContain("Estimated hydration: "); + } + else + { + string calculation = lines.Single(line => line.Contains("HydrationSummaryDuration ")); + using (JsonDocument record = JsonDocument.Parse(calculation.Substring(calculation.IndexOf('{')))) + { + record.RootElement.GetProperty("Outcome").GetString().ShouldEqual("Success"); + record.RootElement.GetProperty("ValidationFailures").GetString().ShouldContain("HydratedFilesExceedTotal"); + record.RootElement.GetProperty("ValidationFailures").GetString().ShouldContain("HydratedFoldersExceedTotal"); + } + } + + testCompleted = true; + } + finally + { + foreach (string file in newFiles) + { + Cleanup(() => File.Delete(file)); + } + + if (File.Exists(untrackedFile)) + { + Cleanup(() => File.Delete(untrackedFile)); + } + + if (Directory.Exists(newFolder)) + { + Cleanup(() => Directory.Delete(newFolder, recursive: true)); + } + + if (Directory.Exists(retainedFolder)) + { + Cleanup(() => Directory.Delete(retainedFolder, recursive: true)); + } + + Cleanup(() => this.RunGit("checkout -f " + originalCommit)); + if (oldBranchCreated) + { + Cleanup(() => this.RunGit("branch -D hydration-count-old")); + } + + if (currentBranchCreated) + { + Cleanup(() => this.RunGit("branch -D hydration-count-current")); + } + + Cleanup(() => this.Enlistment.WaitForBackgroundOperations()); + } + } + + private int CountMountHydrationCalculations() + { + return Directory.GetFiles(this.Enlistment.GVFSLogsRoot, "gvfs_mount_*.log") + .Sum(path => GVFSHelpers.ReadAllTextFromWriteLockedFile(path).Split('\n').Count(line => line.Contains("HydrationSummaryDuration "))); + } + + private string RunGitWithInput(string command, string input) + { + using (MemoryStream stream = new MemoryStream(Encoding.UTF8.GetBytes(input))) + { + ProcessResult result = GitProcess.InvokeProcess(this.Enlistment.RepoRoot, command, inputStream: stream); + result.ExitCode.ShouldEqual(0, result.Errors); + return result.Output.Trim(); + } + } + + private string RunGit(string command) + { + ProcessResult result = GitProcess.InvokeProcess(this.Enlistment.RepoRoot, command); + result.ExitCode.ShouldEqual(0, result.Errors); + return result.Output; } private JsonElement WaitForHydrationCacheState(string expectedState, out string output, out string[] lines) { Stopwatch deadline = Stopwatch.StartNew(); + Stopwatch stable = new Stopwatch(); + string calculationId = null; string lastCompletion = string.Empty; output = string.Empty; lines = Array.Empty(); @@ -214,16 +400,43 @@ private JsonElement WaitForHydrationCacheState(string expectedState, out string { JsonElement root = completion.RootElement; lastCompletion = root.GetRawText(); - if (root.GetProperty("Cache").GetProperty("State").GetString() == expectedState) + JsonElement cache = root.GetProperty("Cache"); + if (cache.GetProperty("State").GetString() == expectedState + && (expectedState != "Available" || !cache.GetProperty("Invalidated").GetBoolean())) + { + if (expectedState != "Available") + { + return root.Clone(); + } + + string currentId = cache.GetProperty("CalculationId").GetString(); + if (currentId != calculationId) + { + calculationId = currentId; + stable.Restart(); + } + + if (stable.Elapsed >= TimeSpan.FromSeconds(2)) + { + return root.Clone(); + } + } + else { - return root.Clone(); + calculationId = null; + stable.Reset(); } } Thread.Sleep(100); } - throw new AssertionException("The hydration cache did not reach " + expectedState + ". Last completion: " + lastCompletion); + string lastCalculation = Directory.GetFiles(this.Enlistment.GVFSLogsRoot, "gvfs_mount_*.log") + .OrderBy(path => path) + .SelectMany(path => GVFSHelpers.ReadAllTextFromWriteLockedFile(path).Split('\n')) + .LastOrDefault(line => line.Contains("HydrationSummaryDuration ")); + throw new AssertionException("The hydration cache did not reach " + expectedState + ". Last completion: " + lastCompletion + + ". Last mount calculation: " + lastCalculation); } private void RemountWithHydrationStatus(bool enabled, int backoffSeconds = 2) diff --git a/GVFS/GVFS.FunctionalTests/Tools/GitHelpers.cs b/GVFS/GVFS.FunctionalTests/Tools/GitHelpers.cs index 3d75ec817..67c55d193 100644 --- a/GVFS/GVFS.FunctionalTests/Tools/GitHelpers.cs +++ b/GVFS/GVFS.FunctionalTests/Tools/GitHelpers.cs @@ -104,7 +104,7 @@ private static string FilterMessages( bool removeFSMonitorMessages, bool removeGvfsHealthMessages) { - if (!string.IsNullOrEmpty(input) && (removeWaitingMessages || removeUpgradeMessages || removePartialHydrationMessages || removeFSMonitorMessages)) + if (!string.IsNullOrEmpty(input) && (removeWaitingMessages || removeUpgradeMessages || removePartialHydrationMessages || removeFSMonitorMessages || removeGvfsHealthMessages)) { IEnumerable lines = SplitLinesKeepingNewlines(input); IEnumerable filteredLines = lines.Where(line => @@ -113,7 +113,7 @@ private static string FilterMessages( (removeUpgradeMessages && line.StartsWith("A new version of VFS for Git is available.")) || (removeWaitingMessages && line.StartsWith("Waiting for ")) || (removePartialHydrationMessages && line.StartsWith("You are in a partially-hydrated checkout with ")) || - (removeGvfsHealthMessages && line.TrimEnd().EndsWith("Run 'gvfs health' for details.")) || + (removeGvfsHealthMessages && (line.StartsWith("Estimated hydration: ") || line.TrimEnd().EndsWith("Run 'gvfs health' for details."))) || (removeFSMonitorMessages && line.TrimEnd().EndsWith(" is incompatible with fsmonitor"))) { return false; diff --git a/GVFS/GVFS.Mount/InProcessMount.cs b/GVFS/GVFS.Mount/InProcessMount.cs index 215207ece..975f24dac 100644 --- a/GVFS/GVFS.Mount/InProcessMount.cs +++ b/GVFS/GVFS.Mount/InProcessMount.cs @@ -896,15 +896,7 @@ private void HandleGetHydrationStatusRequest(NamedPipeServer.Connection connecti return; } - NamedPipeMessages.HydrationStatus.Response response = new NamedPipeMessages.HydrationStatus.Response - { - PlaceholderFileCount = summary.PlaceholderFileCount, - PlaceholderFolderCount = summary.PlaceholderFolderCount, - ModifiedFileCount = summary.ModifiedFileCount, - ModifiedFolderCount = summary.ModifiedFolderCount, - TotalFileCount = summary.TotalFileCount, - TotalFolderCount = summary.TotalFolderCount, - }; + NamedPipeMessages.HydrationStatus.Response response = summary.ToResponse(); connection.TrySendResponse( new NamedPipeMessages.Message(NamedPipeMessages.HydrationStatus.SuccessResult, response.ToBody(info))); diff --git a/GVFS/GVFS.UnitTests/Common/EnlistmentHydrationSummaryTests.cs b/GVFS/GVFS.UnitTests/Common/EnlistmentHydrationSummaryTests.cs index 99ead1c81..058da18a5 100644 --- a/GVFS/GVFS.UnitTests/Common/EnlistmentHydrationSummaryTests.cs +++ b/GVFS/GVFS.UnitTests/Common/EnlistmentHydrationSummaryTests.cs @@ -1,5 +1,6 @@ using GVFS.Common; using GVFS.Common.Git; +using GVFS.Common.NamedPipes; using GVFS.Common.Tracing; using GVFS.UnitTests.Mock.Common; using GVFS.UnitTests.Mock.FileSystem; @@ -326,11 +327,12 @@ public void CreateSummary_CancelledToken_ReturnsInvalidSummary() Assert.IsNull(result.Error); } - [TestCase(20, "Success", "None")] - [TestCase(1, "InvalidCounts", "HydratedFoldersExceedTotal")] - public void CalculationEmitsCountsAndFinalValidity(int folderCount, string outcome, string validationFailures) + [TestCase(100, 20, "None")] + [TestCase(100, 1, "HydratedFoldersExceedTotal")] + [TestCase(4, 1, "HydratedFilesExceedTotal, HydratedFoldersExceedTotal")] + public void CalculationEmitsCountsAndFinalValidity(int fileCount, int folderCount, string validationFailures) { - this.AddIndex(100); + this.AddIndex(fileCount); using (JsonTracer tracer = new JsonTracer("test", "test", disableTelemetry: true)) { RecordingListener listener = new RecordingListener(Keywords.Telemetry); @@ -339,16 +341,21 @@ public void CalculationEmitsCountsAndFinalValidity(int folderCount, string outco this.context.Enlistment, this.fileSystem, tracer, () => folderCount, new TestPathData()); TraceEventMessage message = listener.Events.Single(e => e.EventName == "HydrationSummaryDuration"); - Assert.AreEqual(outcome == "Success" ? EventLevel.Informational : EventLevel.Warning, message.Level); + Assert.AreEqual(EventLevel.Informational, message.Level); using (JsonDocument json = JsonDocument.Parse(message.Payload)) { JsonElement root = json.RootElement; - Assert.AreEqual(outcome, root.GetProperty("Outcome").GetString()); + Assert.AreEqual("Success", root.GetProperty("Outcome").GetString()); Assert.AreEqual(validationFailures, root.GetProperty("ValidationFailures").GetString()); - Assert.AreEqual(100, root.GetProperty("TotalFileCount").GetInt32()); + Assert.AreEqual(fileCount, root.GetProperty("TotalFileCount").GetInt32()); Assert.AreEqual(2, root.GetProperty("PlaceholderFileCount").GetInt32()); Assert.AreEqual(3, root.GetProperty("ModifiedFileCount").GetInt32()); Assert.AreEqual(folderCount, root.GetProperty("TotalFolderCount").GetInt32()); + Assert.AreEqual(5L - fileCount, root.GetProperty("FileCountDelta").GetInt64()); + Assert.AreEqual(2L - folderCount, root.GetProperty("FolderCountDelta").GetInt64()); + Assert.AreEqual(Math.Max(0, 5L - fileCount), root.GetProperty("FileCountExcess").GetInt64()); + Assert.AreEqual(Math.Max(0, 2L - folderCount), root.GetProperty("FolderCountExcess").GetInt64()); + Assert.AreEqual("IndexHeader", root.GetProperty("FileCountSource").GetString()); Assert.AreEqual(summary.IsValid, root.GetProperty("IsValid").GetBoolean()); Assert.IsTrue(root.GetProperty("CountsComplete").GetBoolean()); Assert.AreNotEqual(Guid.Empty, Guid.Parse(root.GetProperty("CalculationId").GetString())); @@ -401,18 +408,21 @@ public void FailureRetainsEarlierAndPartialCounts(HydrationSummaryPhase phase) } } - [TestCase] - public void FailedSourceDoesNotReportZeroAsAvailable() + [TestCase(HydrationDataState.Failed, true)] + [TestCase(HydrationDataState.Partial, false)] + public void IncompleteSourceDoesNotProduceUsableSummary(HydrationDataState state, bool empty) { this.AddIndex(100); - TestPathData paths = new TestPathData { ModifiedState = HydrationDataState.Failed, EmptyModifiedPaths = true }; + TestPathData paths = new TestPathData { ModifiedState = state, EmptyModifiedPaths = empty }; EnlistmentHydrationSummary summary = EnlistmentHydrationSummary.CreateSummary( this.context.Enlistment, this.fileSystem, this.tracer, () => 20, paths); - Assert.IsNull(summary.Diagnostics.ModifiedFileCount); - Assert.IsNull(summary.Diagnostics.ModifiedFolderCount); - Assert.AreEqual(HydrationDataState.Failed, summary.Diagnostics.ModifiedPathsState); + Assert.AreEqual(empty ? (int?)null : 3, summary.Diagnostics.ModifiedFileCount); + Assert.AreEqual(empty ? (int?)null : 1, summary.Diagnostics.ModifiedFolderCount); + Assert.AreEqual(state, summary.Diagnostics.ModifiedPathsState); Assert.IsFalse(summary.Diagnostics.CountsComplete); Assert.AreEqual(HydrationSummaryOutcome.IncompleteData, summary.Diagnostics.Outcome); + Assert.IsFalse(summary.IsValid); + Assert.That(summary.ToMessage(), Does.StartWith("Error calculating hydration summary.")); } [TestCase] @@ -429,18 +439,37 @@ public void EmptySourceReportsCollectedZero() } [TestCase] - public void EarlyExitLeavesFolderCountAndTimingUnavailable() + public void EmptyCompleteSummaryDisplaysZeroRatherThanUnavailable() { this.AddIndex(0); EnlistmentHydrationSummary summary = EnlistmentHydrationSummary.CreateSummary( - this.context.Enlistment, this.fileSystem, this.tracer, () => throw new AssertionException("Folder provider must not run."), new TestPathData()); - Assert.IsTrue(summary.Diagnostics.EarlyExit); - Assert.IsFalse(summary.IsValid); - Assert.AreEqual(HydrationSummaryOutcome.InvalidCounts, summary.Diagnostics.Outcome); - Assert.IsNull(summary.Diagnostics.TotalFolderCount); - Assert.IsNull(summary.Diagnostics.TreeCountMs); - Assert.AreEqual(HydrationDataState.NotStarted, summary.Diagnostics.FolderCountState); - Assert.IsTrue(summary.Diagnostics.GetValidationFailures().HasFlag(HydrationValidationFailure.HydratedFilesExceedTotal)); + this.context.Enlistment, this.fileSystem, this.tracer, () => 0, + new TestPathData { EmptyPlaceholders = true, EmptyModifiedPaths = true }); + Assert.IsTrue(summary.IsValid); + Assert.That(summary.ToMessage(), Does.Contain("files 0%; folders 0%")); + } + + [TestCase(4, 1, "files 100% (counts exceed current total); folders 100% (counts exceed current total)")] + [TestCase(0, 20, "files n/a (no current denominator); folders 10%")] + [TestCase(100, 0, "files 5%; folders n/a (no current denominator)")] + public void CountDriftCollectsBothDenominatorsAndSharesTheWireDisplay(int files, int folders, string estimate) + { + this.AddIndex(files); + EnlistmentHydrationSummary summary = EnlistmentHydrationSummary.CreateSummary( + this.context.Enlistment, this.fileSystem, this.tracer, () => folders, new TestPathData()); + Assert.IsFalse(summary.Diagnostics.EarlyExit); + Assert.IsTrue(summary.IsValid); + Assert.AreEqual(HydrationSummaryOutcome.Success, summary.Diagnostics.Outcome); + Assert.AreEqual(folders, summary.Diagnostics.TotalFolderCount); + Assert.IsNotNull(summary.Diagnostics.TreeCountMs); + Assert.AreEqual(HydrationDataState.Complete, summary.Diagnostics.FolderCountState); + Assert.AreEqual("Estimated hydration: " + estimate + ". Run 'gvfs health' at the repository root for details.", summary.ToMessage()); + Assert.IsTrue(NamedPipeMessages.HydrationStatus.Response.TryParse(summary.ToResponse().ToBody(), out NamedPipeMessages.HydrationStatus.Response response)); + Assert.AreEqual(summary.ToMessage(), response.ToDisplayMessage()); + Assert.AreEqual(files, response.TotalFileCount); + Assert.AreEqual(folders, response.TotalFolderCount); + Assert.AreEqual(5, response.HydratedFileCount); + Assert.AreEqual(2, response.HydratedFolderCount); } [TestCase] @@ -454,6 +483,7 @@ public void CancellationRetainsSuccessfulObservations() EnlistmentHydrationSummary summary = EnlistmentHydrationSummary.CreateSummary( this.context.Enlistment, this.fileSystem, this.tracer, () => 20, paths, cts.Token); Assert.AreEqual(HydrationSummaryOutcome.Cancelled, summary.Diagnostics.Outcome); + Assert.IsFalse(summary.IsValid); Assert.IsTrue(summary.Diagnostics.CancellationRequested); Assert.AreEqual(HydrationSummaryPhase.Placeholders, summary.Diagnostics.Phase); Assert.AreEqual(HydrationDataState.Complete, summary.Diagnostics.PlaceholderState); @@ -588,13 +618,17 @@ private sealed class TestPathData : EnlistmentPathData public HydrationSummaryPhase FailurePhase { get; set; } public HydrationDataState ModifiedState { get; set; } = HydrationDataState.Complete; public bool EmptyModifiedPaths { get; set; } + public bool EmptyPlaceholders { get; set; } public Action AfterPlaceholders { get; set; } public Action DuringModifiedPaths { get; set; } public override void LoadPlaceholdersFromDatabase(GVFSEnlistment enlistment, HydrationSummaryDiagnostics diagnostics = null) { - this.PlaceholderFilePaths.AddRange(new[] { "a", "b" }); - this.PlaceholderFolderPaths.Add("folder"); + if (!this.EmptyPlaceholders) + { + this.PlaceholderFilePaths.AddRange(new[] { "a", "b" }); + this.PlaceholderFolderPaths.Add("folder"); + } if (this.FailurePhase == HydrationSummaryPhase.Placeholders) { throw new IOException("mock:\\private\\placeholder {secret}"); diff --git a/GVFS/GVFS.UnitTests/Common/EnlistmentPathDiagnosticsTests.cs b/GVFS/GVFS.UnitTests/Common/EnlistmentPathDiagnosticsTests.cs index 4c2aed4a6..31b8848a2 100644 --- a/GVFS/GVFS.UnitTests/Common/EnlistmentPathDiagnosticsTests.cs +++ b/GVFS/GVFS.UnitTests/Common/EnlistmentPathDiagnosticsTests.cs @@ -14,6 +14,33 @@ namespace GVFS.UnitTests.Common [TestFixture] public class EnlistmentPathDiagnosticsTests { + [TestCase(false)] + [TestCase(true)] + public void InMemoryModifiedPathsDoNotUsePipeOrDisk(bool empty) + { + string[] entries = empty ? Array.Empty() : new[] { string.Empty, "file", "folder/", string.Empty }; + int calls = 0; + SourcePathData paths = new SourcePathData(() => + { + calls++; + return entries; + }); + HydrationSummaryDiagnostics diagnostics = new HydrationSummaryDiagnostics(); + using (MockTracer tracer = new MockTracer()) + { + paths.LoadModifiedPaths(CreateEnlistment(), tracer, diagnostics); + } + + Assert.AreEqual(1, calls); + CollectionAssert.AreEqual(empty ? Array.Empty() : new[] { "file" }, paths.ModifiedFilePaths); + CollectionAssert.AreEqual(empty ? Array.Empty() : new[] { "folder/" }, paths.ModifiedFolderPaths); + Assert.AreEqual(HydrationDataState.Complete, diagnostics.ModifiedPathsState); + Assert.AreEqual(HydrationPathSourceOutcome.NotAttempted, diagnostics.ModifiedPathsPipeOutcome); + Assert.AreEqual(HydrationPathSourceOutcome.NotAttempted, diagnostics.ModifiedPathsDiskOutcome); + Assert.AreEqual(0, paths.Pipe.ConnectTimeoutMs); + Assert.IsFalse(paths.Pipe.Disposed); + } + [TestCase("S", "Success")] [TestCase("NotReady", "MountNotReady")] [TestCase("InvalidVersion", "InvalidVersion")] @@ -126,6 +153,11 @@ private static GVFSEnlistment CreateEnlistment() private sealed class SourcePathData : EnlistmentPathData { + public SourcePathData(Func> modifiedPathsProvider = null) + : base(modifiedPathsProvider) + { + } + public MockNamedPipeClient Pipe { get; } = new MockNamedPipeClient { Response = new NamedPipeMessages.Message("unknown", null), diff --git a/GVFS/GVFS.UnitTests/Common/GitStatusCacheTests.cs b/GVFS/GVFS.UnitTests/Common/GitStatusCacheTests.cs index aa82ec0c8..43bbc0b4a 100644 --- a/GVFS/GVFS.UnitTests/Common/GitStatusCacheTests.cs +++ b/GVFS/GVFS.UnitTests/Common/GitStatusCacheTests.cs @@ -109,6 +109,7 @@ public void DisabledHydrationFeatureExplainsTheCacheMiss() using (GitStatusCache cache = new GitStatusCache(this.context, TimeSpan.Zero)) { cache.SetProjectedFolderCountProvider(() => throw new AssertionException("A disabled feature must not count folders.")); + cache.SetModifiedPathsProvider(() => throw new AssertionException("A disabled feature must not read modified paths.")); cache.UpdateHydrationSummary(); Assert.IsNull(cache.GetCachedHydrationSummary(out NamedPipeMessages.HydrationStatus.CacheInfo info)); Assert.AreEqual(NamedPipeMessages.HydrationStatus.CacheState.FeatureDisabled, info.State); @@ -149,7 +150,7 @@ public void HydrationSnapshotPublishesCountsIdentityAgeAndInvalidationTogether() } [TestCase("InvalidCounts", false)] - [TestCase("IncompleteData", true)] + [TestCase("IncompleteData", false)] [TestCase("Cancelled", false)] [TestCase("Failed", false)] [Category(CategoryConstants.ExceptionExpected)] @@ -178,6 +179,38 @@ public void HydrationRefreshReportsEachCalculationState(string state, bool avail } } + [TestCase] + public void CountDriftRemainsCachedWithoutRepeatedCalculationOrBreakerFailures() + { + this.AddHydrationIndex(); + using (DiagnosticStatusCache cache = new DiagnosticStatusCache(this.context)) + { + int calculations = 0; + cache.Calculate = diagnostics => + { + calculations++; + return EnlistmentHydrationSummary.CreateSummary(this.context.Enlistment, this.fileSystem, this.context.Tracer, + () => 1, new EmptyHydrationPaths { CountDrift = true }, diagnostics: diagnostics); + }; + cache.UpdateHydrationSummary(); + EnlistmentHydrationSummary summary = cache.GetCachedHydrationSummary(out NamedPipeMessages.HydrationStatus.CacheInfo info); + Assert.IsNotNull(summary); + Assert.AreEqual(NamedPipeMessages.HydrationStatus.CacheState.Available, info.State); + Assert.IsTrue(info.CountsComplete); + Assert.AreEqual(2, summary.HydratedFolderCount); + Assert.AreEqual(1, summary.TotalFolderCount); + Assert.IsTrue(NamedPipeMessages.HydrationStatus.Response.TryParse(summary.ToResponse().ToBody(info), out _)); + for (int i = 0; i < 3; i++) + { + Assert.AreSame(summary, cache.GetCachedHydrationSummary(out NamedPipeMessages.HydrationStatus.CacheInfo repeated)); + Assert.AreEqual(info.CalculationId, repeated.CalculationId); + } + + Assert.AreEqual(1, calculations); + Assert.AreEqual(0, cache.Breaker.Failures); + } + } + [TestCase] [Category(CategoryConstants.ExceptionExpected)] public void ConfigurationFailurePreservesSummaryAndDoesNotTripBreaker() @@ -269,9 +302,14 @@ private EnlistmentHydrationSummary CalculateHydration(HydrationSummaryDiagnostic private sealed class EmptyHydrationPaths : EnlistmentPathData { public bool FailModified { get; set; } + public bool CountDrift { get; set; } public override void LoadPlaceholdersFromDatabase(GVFSEnlistment enlistment, HydrationSummaryDiagnostics diagnostics = null) { + if (this.CountDrift) + { + this.PlaceholderFolderPaths.AddRange(new[] { "old", "new" }); + } } public override void LoadModifiedPaths(GVFSEnlistment enlistment, ITracer tracer, HydrationSummaryDiagnostics diagnostics = null) diff --git a/GVFS/GVFS.UnitTests/Common/HydrationStatusDiagnosticsTests.cs b/GVFS/GVFS.UnitTests/Common/HydrationStatusDiagnosticsTests.cs index 3416b97ca..d1ed003b5 100644 --- a/GVFS/GVFS.UnitTests/Common/HydrationStatusDiagnosticsTests.cs +++ b/GVFS/GVFS.UnitTests/Common/HydrationStatusDiagnosticsTests.cs @@ -64,7 +64,8 @@ public void CacheExtensionRejectsUnknownOrMalformedFields(string body) [TestCase(null, "MalformedResponse")] [TestCase("1,2", "MalformedResponse")] [TestCase("1,2,secret,4,100,50", "MalformedResponse")] - [TestCase("1,2,3,4,100,1", "InvalidCounts")] + [TestCase("1,2,3,4,100,-1", "InvalidCounts")] + [TestCase("1,2,3,4,100,1", "Success")] [TestCase("1,2,3,4,100,50", "Success")] public void CacheResultDistinguishesMalformedAndInvalidCounts(string body, string outcome) { @@ -112,7 +113,7 @@ public void CachedDisplayRecordsCountsWithoutFallback() EventMetadata cache = (EventMetadata)metadata["Cache"]; Assert.AreEqual(HealthVerb.CacheLookupOutcome.Success, cache["Outcome"]); Assert.AreEqual(100, ((EventMetadata)cache["Counts"])["TotalFileCount"]); - Assert.That(output.ToString(), Does.Contain("4% of files and 12% of folders hydrated.")); + Assert.That(output.ToString(), Does.Contain("Estimated hydration: files 4%; folders 12%.")); } } @@ -175,17 +176,16 @@ public void AdversarialResponsesCannotCopyRawTextIntoTelemetry() } } - [TestCase(true)] - [TestCase(false)] - public void CacheHitKeepsCommandOutcomeSeparateFromCompleteness(bool complete) + [TestCase] + public void CacheHitKeepsCommandOutcomeSeparateFromCompleteness() { using (StringWriter output = new StringWriter()) using (MockTracer tracer = new MockTracer()) { NamedPipeMessages.HydrationStatus.CacheInfo info = new NamedPipeMessages.HydrationStatus.CacheInfo { - CountsComplete = complete, - State = complete ? NamedPipeMessages.HydrationStatus.CacheState.Available : NamedPipeMessages.HydrationStatus.CacheState.IncompleteData, + CountsComplete = true, + State = NamedPipeMessages.HydrationStatus.CacheState.Available, CalculationId = Guid.NewGuid(), }; WorkerHealthVerb verb = new WorkerHealthVerb { Output = output }; @@ -193,13 +193,63 @@ public void CacheHitKeepsCommandOutcomeSeparateFromCompleteness(bool complete) EventMetadata metadata = new EventMetadata(); verb.OutputHydrationPercent(null, tracer, metadata, Guid.NewGuid()); Assert.AreEqual(HealthVerb.HealthCommandOutcome.Success, metadata["Outcome"]); - Assert.AreEqual(complete ? HydrationSummaryOutcome.Success : HydrationSummaryOutcome.IncompleteData, metadata["CalculationOutcome"]); + Assert.AreEqual(HydrationSummaryOutcome.Success, metadata["CalculationOutcome"]); Assert.AreEqual(info.CalculationId.Value.ToString("N"), metadata["CalculationId"]); Assert.AreEqual("Cache", metadata["ResultSource"]); Assert.IsTrue(verb.Pipe.Disposed); } } + [TestCase] + public void PartialCacheResponseRemainsDistinctFromCountDrift() + { + HealthVerb.CacheLookupResult result = HealthVerb.ParseCachedResponse( + new NamedPipeMessages.Message("S", "1,2,3,4,100,50,D1,6,-,0,0,0")); + Assert.AreEqual(HealthVerb.CacheLookupOutcome.IncompleteData, result.Outcome); + Assert.AreEqual(NamedPipeMessages.HydrationStatus.ParseOutcome.IncompleteData, result.ParseOutcome); + Assert.AreEqual(100, result.Counts["TotalFileCount"]); + } + + [TestCase("50,50,50,50,100,100", "files 100%; folders 100%")] + [TestCase("1,1,1,1,3,3", "files 66%; folders 66%")] + public void CachedStatusFormatsExactAndFractionalHydration(string body, string estimate) + { + using (StringWriter output = new StringWriter()) + using (MockTracer tracer = new MockTracer()) + { + WorkerHealthVerb verb = new WorkerHealthVerb { Output = output }; + verb.Pipe.Response = new NamedPipeMessages.Message("S", body); + EventMetadata metadata = new EventMetadata(); + verb.OutputHydrationPercent(null, tracer, metadata, Guid.NewGuid()); + Assert.AreEqual("Cache", metadata["ResultSource"]); + Assert.That(output.ToString(), Does.Contain("Estimated hydration: " + estimate + ".")); + } + } + + [TestCase] + public void CachedDriftPreservesRawCountsAndExcessWithoutFallback() + { + using (StringWriter output = new StringWriter()) + using (MockTracer tracer = new MockTracer()) + { + WorkerHealthVerb verb = new WorkerHealthVerb { Output = output }; + verb.Pipe.Response = new NamedPipeMessages.Message("S", "2147483647,2,2147483647,3,1,0,D1,1,-,0,0,1"); + EventMetadata metadata = new EventMetadata(); + verb.OutputHydrationPercent(null, tracer, metadata, Guid.NewGuid()); + Assert.AreEqual("Cache", metadata["ResultSource"]); + Assert.AreEqual(HealthVerb.HealthCommandOutcome.Success, metadata["Outcome"]); + Assert.AreEqual(HydrationSummaryOutcome.Success, metadata["CalculationOutcome"]); + EventMetadata cache = (EventMetadata)metadata["Cache"]; + Assert.AreEqual(1, ((EventMetadata)cache["Counts"])["TotalFileCount"]); + Assert.AreEqual(4294967293L, cache["FileCountDelta"]); + Assert.AreEqual(4294967293L, cache["FileCountExcess"]); + Assert.AreEqual(5L, cache["FolderCountExcess"]); + Assert.AreEqual("IndexHeader", cache["FileCountSource"]); + Assert.AreEqual(HydrationFolderCountSource.Projection, cache["FolderCountSource"]); + Assert.That(output.ToString(), Does.Contain("files 100% (counts exceed current total); folders n/a (no current denominator)")); + } + } + [TestCase("ConnectTimeout", "Connect")] [TestCase("ConnectFailed", "Connect")] [TestCase("SendFailed", "Send")] @@ -333,6 +383,9 @@ public void DiagnosticArithmeticDoesNotOverflowOrHideMultipleFailures() Assert.IsTrue(failures.HasFlag(HydrationValidationFailure.HydratedFilesExceedTotal)); Assert.IsTrue(failures.HasFlag(HydrationValidationFailure.HydratedFoldersExceedTotal)); Assert.AreEqual(4294967294L, diagnostics.GetMetadata()["HydratedFileCount"]); + Assert.AreEqual(4294967293L, diagnostics.GetMetadata()["FileCountDelta"]); + Assert.AreEqual(4294967293L, diagnostics.GetMetadata()["FileCountExcess"]); + Assert.AreEqual(4L, diagnostics.GetMetadata()["FolderCountExcess"]); } [TestCase] diff --git a/GVFS/GVFS.UnitTests/Common/HydrationStatusErrorPathTests.cs b/GVFS/GVFS.UnitTests/Common/HydrationStatusErrorPathTests.cs index 77d9b7ebf..8fb916419 100644 --- a/GVFS/GVFS.UnitTests/Common/HydrationStatusErrorPathTests.cs +++ b/GVFS/GVFS.UnitTests/Common/HydrationStatusErrorPathTests.cs @@ -1,3 +1,5 @@ +extern alias Hooks; + using GVFS.Common; using GVFS.Common.Git; using GVFS.Common.NamedPipes; @@ -10,6 +12,7 @@ using System.IO; using System.Linq; using System.Threading; +using HookResponse = Hooks::GVFS.Common.NamedPipes.NamedPipeMessages.HydrationStatus.Response; namespace GVFS.UnitTests.Common { @@ -111,6 +114,8 @@ public void TryParse_NonIntegerValues_ReturnsFalse(string body) [TestCase("0,-1,0,0,10,5")] [TestCase("0,0,-1,0,10,5")] [TestCase("0,0,0,-1,10,5")] + [TestCase("0,0,0,0,-1,5")] + [TestCase("0,0,0,0,10,-1")] public void TryParse_NegativeCounts_ReturnsFalse(string body) { bool result = NamedPipeMessages.HydrationStatus.Response.TryParse(body, out NamedPipeMessages.HydrationStatus.Response response); @@ -119,10 +124,41 @@ public void TryParse_NegativeCounts_ReturnsFalse(string body) [TestCase("100,0,100,0,50,5")] [TestCase("0,100,0,100,10,5")] - public void TryParse_HydratedExceedsTotal_ReturnsFalse(string body) + public void TryParse_HydratedExceedsTotal_PreservesRawCounts(string body) { bool result = NamedPipeMessages.HydrationStatus.Response.TryParse(body, out NamedPipeMessages.HydrationStatus.Response response); - Assert.IsFalse(result); + Assert.IsTrue(result); + Assert.AreEqual(body, response.ToBody()); + Assert.That(response.ToDisplayMessage(), Does.Contain("100% (counts exceed current total)")); + } + + [TestCase("0,0,0,0,0,0", "files 0%; folders 0%")] + [TestCase("40,10,10,5,100,50", "files 50%; folders 30%")] + [TestCase("50,50,50,50,100,100", "files 100%; folders 100%")] + [TestCase("1,1,1,1,3,3", "files 66%; folders 66%")] + [TestCase("100,5,1,1,100,5", "files 100% (counts exceed current total); folders 100% (counts exceed current total)")] + [TestCase("1,1,0,0,0,2", "files n/a (no current denominator); folders 50%")] + [TestCase("1,1,0,0,2,0", "files 50%; folders n/a (no current denominator)")] + [TestCase("2147483647,2147483647,2147483647,2147483647,2147483647,2147483647", + "files 100% (counts exceed current total); folders 100% (counts exceed current total)")] + public void SummaryProtocolAndHookUseTheSameEstimates(string body, string estimate) + { + Assert.IsTrue(NamedPipeMessages.HydrationStatus.Response.TryParse(body, out NamedPipeMessages.HydrationStatus.Response response)); + Assert.IsTrue(HookResponse.TryParse(body, out HookResponse hookResponse)); + Assert.AreEqual(body, response.ToBody()); + Assert.AreEqual(response.ToBody(), hookResponse.ToBody()); + Assert.AreEqual("Estimated hydration: " + estimate + ". Run 'gvfs health' at the repository root for details.", response.ToDisplayMessage()); + Assert.AreEqual(response.ToDisplayMessage(), hookResponse.ToDisplayMessage()); + Assert.AreEqual((long)response.PlaceholderFileCount + response.ModifiedFileCount, response.HydratedFileCount); + Assert.AreEqual((long)response.PlaceholderFolderCount + response.ModifiedFolderCount, response.HydratedFolderCount); + } + + [TestCase("1,2,3,4,100,50,D1,6,-,0,0,0")] + [TestCase("-1,2,3,4,100,50")] + public void SummaryProtocolAndHookRejectUnavailableObservations(string body) + { + Assert.IsFalse(NamedPipeMessages.HydrationStatus.Response.TryParse(body, out _)); + Assert.IsFalse(HookResponse.TryParse(body, out _)); } [TestCase] diff --git a/GVFS/GVFS.UnitTests/GVFS.UnitTests.csproj b/GVFS/GVFS.UnitTests/GVFS.UnitTests.csproj index 906a6955e..42ed65d1d 100644 --- a/GVFS/GVFS.UnitTests/GVFS.UnitTests.csproj +++ b/GVFS/GVFS.UnitTests/GVFS.UnitTests.csproj @@ -13,6 +13,7 @@ + diff --git a/GVFS/GVFS.Virtualization/FileSystemCallbacks.cs b/GVFS/GVFS.Virtualization/FileSystemCallbacks.cs index 68358e8dd..c55c5e61b 100644 --- a/GVFS/GVFS.Virtualization/FileSystemCallbacks.cs +++ b/GVFS/GVFS.Virtualization/FileSystemCallbacks.cs @@ -117,6 +117,7 @@ public FileSystemCallbacks( this.gitStatusCache = gitStatusCache ?? new GitStatusCache(context, TimeSpan.Zero); this.gitStatusCache.SetProjectedFolderCountProvider( () => this.GitIndexProjection.GetProjectedFolderCount()); + this.gitStatusCache.SetModifiedPathsProvider(this.GetAllModifiedPaths); this.logsHeadPath = Path.Combine(this.context.Enlistment.DotGitRoot, GVFSConstants.DotGit.Logs.HeadRelativePath); diff --git a/GVFS/GVFS/CommandLine/HealthVerb.cs b/GVFS/GVFS/CommandLine/HealthVerb.cs index a6128dd8e..0675df433 100644 --- a/GVFS/GVFS/CommandLine/HealthVerb.cs +++ b/GVFS/GVFS/CommandLine/HealthVerb.cs @@ -280,6 +280,7 @@ internal static CacheLookupResult ParseCachedResponse(NamedPipeMessages.Message result.Response = response; result.ParseOutcome = outcome; result.Outcome = valid ? CacheLookupOutcome.Success + : outcome == NamedPipeMessages.HydrationStatus.ParseOutcome.IncompleteData ? CacheLookupOutcome.IncompleteData : outcome == NamedPipeMessages.HydrationStatus.ParseOutcome.InvalidCounts ? CacheLookupOutcome.InvalidCounts : CacheLookupOutcome.MalformedResponse; NamedPipeMessages.HydrationStatus.CacheInfo.TryParse(message.Body, out NamedPipeMessages.HydrationStatus.CacheInfo cacheInfo, hasCounts: true); result.Info = cacheInfo; @@ -305,6 +306,7 @@ internal enum CacheLookupOutcome UnexpectedResponse, MalformedResponse, InvalidCounts, + IncompleteData, Failed, } @@ -339,6 +341,12 @@ public EventMetadata GetMetadata() ["ExceptionHResult"] = this.Exception?.HResult, [nameof(this.ParseOutcome)] = this.ParseOutcome, [nameof(this.Counts)] = this.Counts, + ["FileCountSource"] = this.Response == null ? null : "IndexHeader", + ["FolderCountSource"] = this.Response == null ? null : HydrationFolderCountSource.Projection, + ["FileCountDelta"] = this.Response?.FileCountDelta, + ["FolderCountDelta"] = this.Response?.FolderCountDelta, + ["FileCountExcess"] = this.Response?.FileCountExcess, + ["FolderCountExcess"] = this.Response?.FolderCountExcess, ["DiagnosticsAvailable"] = this.Info != null, [nameof(this.DiagnosticsStatus)] = this.DiagnosticsStatus, ["State"] = this.Info?.State, diff --git a/docs/hydration-status-diagnostics.md b/docs/hydration-status-diagnostics.md index 4423e459b..19799c576 100644 --- a/docs/hydration-status-diagnostics.md +++ b/docs/hydration-status-diagnostics.md @@ -19,9 +19,40 @@ Partial counts remain available, but their source state is not `Complete`. `IndexState`, `PlaceholderState`, `ModifiedPathsState`, and `FolderCountState` describe source completeness. `CountsComplete` requires all four states to equal `Complete`. -`IsValid` records the existing display calculation. +`IsValid` requires complete sources and six nonnegative counts. `ValidationFailures` independently records missing counts, negative counts, overflow, and hydrated counts that exceed their totals. Its arithmetic uses 64-bit sums. +Excess counts and sums larger than a 32-bit integer do not make a complete summary invalid. +`FileCountDelta` and `FolderCountDelta` record the signed difference between the observed sum and its denominator. +`FileCountExcess` and `FolderCountExcess` record the positive part of each difference. +An unavailable input leaves its difference and excess null. + +`FileCountSource` identifies the index header. +`FolderCountSource` identifies the projection or the index parser. +The folder denominator excludes the root and folders without projection-eligible entries. +Placeholder and modified-path records describe a different population. +They can include new paths, deletion records, overlap, and obsolete folders that still contain untracked files. +The summary does not intersect or deduplicate these populations. + +## Estimated display + +The CLI and the status hook use the same estimate: + +| Observation | Component display | +| --- | --- | +| Positive denominator, sum within range | Integer percentage | +| Positive denominator, sum exceeds denominator | `100% (counts exceed current total)` | +| Both counts zero | `0%` | +| Positive sum, zero denominator | `n/a (no current denominator)` | + +Every usable message starts with `Estimated hydration`. +This format also applies to `gvfs health --status` when the status-hook feature is disabled. +The display cap does not change raw counts, denominators, wire values, or telemetry. +A component without a denominator does not hide the other component. +Incomplete sources, negative counts, cancellation, and exceptions produce unavailable results. +Count drift alone does not increment the circuit breaker. + +## Calculation phases `Phase` identifies the last started phase, which can already be complete when cancellation occurs between phases. `CancellationRequested` distinguishes caller-requested cancellation from other cancellation exceptions. @@ -37,6 +68,9 @@ They do not prove that all sources came from one atomic repository snapshot. The placeholder fields distinguish partial, expanded, and possible-tombstone folders. The modified-path fields describe the pipe attempt and the disk fallback separately. +Background hydration reads the mount's existing in-memory modified paths without a pipe or disk fallback. +After a successful read, both attempt outcomes remain `NotAttempted`, and the source state is `Complete`. +External health calculations retain the pipe attempt and disk fallback. The disk fields count add, delete, and other records. These fields describe the existing reader. They do not change its replay semantics or the recursive meaning of modified-folder entries. @@ -85,6 +119,10 @@ The age is nonnegative, or `-1` when unavailable. The last two fields use `0` or `1`. Older clients ignore the extension. New clients accept responses without it. +New clients also accept complete nonnegative counts that exceed a denominator. +They reject a recognized extension that reports incomplete counts. +Older clients can reject excess counts safely. +A new client can use the fallback when an older mount rejects its own summary. The mount publishes the summary and its diagnostic identity together. Cache state describes the latest refresh or skip decision. @@ -112,4 +150,6 @@ The listener can drop events when its queue is full or its collector is unavaila Process termination can also prevent a terminal record. Mount telemetry cannot observe a hook connection failure that never reaches the mount. -Diagnostic collection must preserve count sets, disk-log replay, percentages, cache acceptance, and circuit-breaker policy. +This contract does not change count populations, disk-log replay, placeholder cleanup, or the circuit-breaker failure policy. +The disk fallback retains its pre-existing replay limitations. +The estimate does not make that reader an exact inventory.