Retry credential fill after transient Git config access failures

GVFS/GVFS.Common/Git/GitProcess.cs

@@ -40,6 +40,8 @@ public class GitProcess : ICredentialStore
         private const string MissingPasswordErrorMessage = "Git credential fill returned no password";
         private const int MaxCredentialOutputKeys = 32;
         private const int MaxCredentialOutputKeyLength = 32;
+        private const int MaxCredentialConfigReadAttempts = 5;
+        private const int CredentialConfigReadRetryDelayMs = 20;
 
         private static readonly Encoding UTF8NoBOM = new UTF8Encoding(false);
 
@@ -319,13 +321,11 @@ public virtual bool TryGetCertificatePassword(
 
                 // See GetFromConfig for why pre-command hook is disabled
                 // for bootstrap-time git operations.
-                Result gitCredentialOutput = this.InvokeGitAgainstDotGitFolder(
+                Result gitCredentialOutput = this.InvokeCredentialFill(
+                    tracer,
                     "credential fill",
-                    stdin => stdin.Write("protocol=cert\npath=" + certificatePath + "\nusername=\n\n"),
-                    parseStdOutLine: null,
-                    usePreCommandHook: false,
-                    mayRequireAuth: true,
-                    hasVisibleConsoleWindow: hasVisibleConsoleWindow);
+                    "protocol=cert\npath=" + certificatePath + "\nusername=\n\n",
+                    hasVisibleConsoleWindow);
 
                 if (gitCredentialOutput.ExitCodeIsFailure)
                 {
@@ -386,14 +386,12 @@ public virtual bool TryGetCredential(
 
                 // See GetFromConfig for why pre-command hook is disabled
                 // for bootstrap-time git operations.
-                Result gitCredentialOutput = this.InvokeGitAgainstDotGitFolder(
+                Result gitCredentialOutput = this.InvokeCredentialFill(
+                    tracer,
                     GenerateCredentialVerbCommand("fill"),
-                    stdin => stdin.Write($"url={repoUrl}\n\n"),
-                    parseStdOutLine: null,
-                    usePreCommandHook: false,
-                    timeoutMs: timeoutMs,
-                    mayRequireAuth: true,
-                    hasVisibleConsoleWindow: hasVisibleConsoleWindow);
+                    $"url={repoUrl}\n\n",
+                    hasVisibleConsoleWindow,
+                    timeoutMs);
 
                 if (gitCredentialOutput.ExitCodeIsFailure)
                 {
@@ -1147,6 +1145,9 @@ protected virtual Result InvokeGitImpl(
 
                             return new Result(output.ToString(), "Operation timed out: " + errors.ToString(), Result.GenericFailureCode, output.Truncated, errors.Truncated);
                         }
+
+                        // A timed wait does not wait for redirected-stream callbacks to finish.
+                        this.executingProcess.WaitForExit();
                     }
 
                     return new Result(output.ToString(), errors.ToString(), this.executingProcess.ExitCode, output.Truncated, errors.Truncated);
@@ -1181,6 +1182,108 @@ private static bool CurrentProcessHasVisibleConsoleWindowSafe()
             }
         }
 
+        private Result InvokeCredentialFill(
+            ITracer tracer,
+            string command,
+            string input,
+            bool hasVisibleConsoleWindow,
+            int timeoutMs = Timeout.Infinite)
+        {
+            Stopwatch timer = Stopwatch.StartNew();
+            int remainingTimeoutMs = timeoutMs;
+            for (int attempt = 1; ; ++attempt)
+            {
+                Result result = this.InvokeGitAgainstDotGitFolder(
+                    command,
+                    stdin => stdin.Write(input),
+                    parseStdOutLine: null,
+                    usePreCommandHook: false,
+                    timeoutMs: remainingTimeoutMs,
+                    mayRequireAuth: true,
+                    hasVisibleConsoleWindow: hasVisibleConsoleWindow);
+
+                if (!IsCredentialConfigReadFailure(result))
+                {
+                    return result;
+                }
+
+                int delayMs = CredentialConfigReadRetryDelayMs << (attempt - 1);
+                bool willRetry = attempt < MaxCredentialConfigReadAttempts &&
+                    (timeoutMs == Timeout.Infinite || timeoutMs - timer.ElapsedMilliseconds > delayMs);
+                tracer.RelatedEvent(
+                    EventLevel.Warning,
+                    "CredentialConfigReadFailure",
+                    new EventMetadata
+                    {
+                        { "AttemptNumber", attempt },
+                        { "MaxAttempts", MaxCredentialConfigReadAttempts },
+                        { "WillRetry", willRetry },
+                        { "RetryDelayMilliseconds", willRetry ? delayMs : 0 },
+                    },
+                    Keywords.Network | Keywords.Telemetry);
+
+                if (!willRetry)
+                {
+                    return result;
+                }
+
+                TraceCredentialFillErrorOutput(tracer, result.Errors);
+                Thread.Sleep(delayMs);
+                if (timeoutMs != Timeout.Infinite)
+                {
+                    remainingTimeoutMs = (int)Math.Max(0, timeoutMs - timer.ElapsedMilliseconds);
+                }
+            }
+        }
+
+        private static bool IsCredentialConfigReadFailure(Result result)
+        {
+            if (!result.ExitCodeIsFailure || result.ErrorsTruncated || result.OutputTruncated ||
+                !string.IsNullOrEmpty(result.Output))
+            {
+                return false;
+            }
+
+            // Only retry Git's config-read failure, not a credential helper or prompt failure.
+            bool configReadFailed = false;
+            bool transientAccessFailure = false;
+            foreach (string errorLine in result.Errors.Split('\n'))
+            {
+                string line = errorLine.TrimEnd('\r');
+                if (line.Length == 0)
+                {
+                    continue;
+                }
+
+                if (line == "fatal: unknown error occurred while reading the configuration files")
+                {
+                    configReadFailed = true;
+                }
+                else if (line.StartsWith("warning: unable to access '", StringComparison.Ordinal))
+                {
+                    int separator = line.LastIndexOf("': ", StringComparison.Ordinal);
+                    if (separator < 0)
+                    {
+                        return false;
+                    }
+
+                    string reason = line.Substring(separator + 3);
+                    if (reason != "Permission denied" && reason != "Device or resource busy")
+                    {
+                        return false;
+                    }
+
+                    transientAccessFailure = true;
+                }
+                else
+                {
+                    return false;
+                }
+            }
+
+            return configReadFailed && transientAccessFailure;
+        }
+
         private static void AddCredentialFillResult(EventMetadata data, Result result)
         {
             // Describes the failure without the text of stderr, which can contain a secret.