From f16f63c666782f71e8aab7e8b19daf937d8efc10 Mon Sep 17 00:00:00 2001 From: Dylan Murphy Date: Tue, 17 Nov 2026 18:01:05 -0500 Subject: [PATCH 01/10] Android: clear a backoff wait when the app starts, resumes or comes to the front After a long offline time every Android upload could wait up to 2 hours, and opening the app did not wake it. The library now clears that wait by itself. - QueueController.retryNow(keys?): a queued entry that waits for a backoff runs now and keeps at most 5 backoff steps. A queued entry with no wait is scheduled when no row of it can start now (one WorkManager query). The boot sweep and resume() do the same; a return to the foreground calls retryNow on the queue executor. - resume() starts the backoff of a resumed entry over. - schedule() replaces a main-chain row that WorkManager holds back after a system stop, when no run of the entry is in progress. Before, it added no row, and the entry waited for WorkManager's own backoff (up to 5 h). - A chunked part uses the stored streak only until a part is accepted in the run. - A transport failure waits at most 15 minutes. - After a time-limit stop, no fresh wake row when WorkManager starts the stopped row again at once, so the entry does not get two rows. Co-Authored-By: Claude Opus 5.5 --- .../backgroundupload/ChunkedUploadWorker.kt | 17 +- .../ai/openspace/backgroundupload/EntryRun.kt | 35 ++- .../backgroundupload/EntryTransitions.kt | 22 +- .../openspace/backgroundupload/EntryWorker.kt | 7 + .../backgroundupload/ForegroundTask.kt | 23 ++ .../backgroundupload/QueueController.kt | 69 ++++- .../backgroundupload/RetryClassifier.kt | 19 ++ .../openspace/backgroundupload/Scheduler.kt | 244 ++++++++++++++---- .../backgroundupload/TransferHost.kt | 6 + .../backgroundupload/UploaderModule.kt | 12 +- .../openspace/backgroundupload/WorkerOps.kt | 18 +- .../backgroundupload/EntryRunTest.kt | 113 ++++++++ .../backgroundupload/EntryTransitionsTest.kt | 8 + .../backgroundupload/ForegroundTaskTest.kt | 65 +++++ .../backgroundupload/MainChainTest.kt | 196 ++++++++++++++ .../backgroundupload/QueueControllerTest.kt | 133 +++++++++- .../openspace/backgroundupload/TestSupport.kt | 104 ++++++++ 17 files changed, 1006 insertions(+), 85 deletions(-) create mode 100644 android/src/main/java/ai/openspace/backgroundupload/ForegroundTask.kt create mode 100644 android/src/test/java/ai/openspace/backgroundupload/ForegroundTaskTest.kt create mode 100644 android/src/test/java/ai/openspace/backgroundupload/MainChainTest.kt diff --git a/android/src/main/java/ai/openspace/backgroundupload/ChunkedUploadWorker.kt b/android/src/main/java/ai/openspace/backgroundupload/ChunkedUploadWorker.kt index 74d19acd..b1880834 100644 --- a/android/src/main/java/ai/openspace/backgroundupload/ChunkedUploadWorker.kt +++ b/android/src/main/java/ai/openspace/backgroundupload/ChunkedUploadWorker.kt @@ -68,8 +68,15 @@ internal class ChunkedTransfer(private val run: EntryRun) { return Settlement.Completed(null, lastAcceptedUrl ?: d0.reportUrl, d0.method) } + /** + * [initialStreak] is the entry's stored streak. A part uses it only until + * a part is accepted in this run. After that the server answers, so the + * part counts its own failures from 0. Without that, one failed part after + * a long backoff would release the whole entry for the next, longer wait. + */ internal suspend fun executePart(index: Int, blob: File, initialStreak: Int) { - var streak = initialStreak + var base = initialStreak + var failures = 0 while (true) { if (index in acceptedHere) return val latest = run.ops.latest(run.entryId, run.generation) @@ -104,12 +111,14 @@ internal class ChunkedTransfer(private val run: EntryRun) { } is EntryRun.AttemptResult.Auth -> { if (!r.reissue) throw EntryRun.ParkException(r.headerGeneration) - streak = 0 + base = 0 + failures = 0 } EntryRun.AttemptResult.Transient -> { onPartProgress(index, 0L) - streak++ - run.backoffOrRelease(a.policy, streak, a.entry.expiresAt) + if (acceptedHere.isNotEmpty()) base = 0 + failures++ + run.backoffOrRelease(a.policy, base + failures, a.entry.expiresAt) } is EntryRun.AttemptResult.Terminal -> { onPartProgress(index, 0L) diff --git a/android/src/main/java/ai/openspace/backgroundupload/EntryRun.kt b/android/src/main/java/ai/openspace/backgroundupload/EntryRun.kt index fbb6c2c1..3d721a89 100644 --- a/android/src/main/java/ai/openspace/backgroundupload/EntryRun.kt +++ b/android/src/main/java/ai/openspace/backgroundupload/EntryRun.kt @@ -202,38 +202,49 @@ internal class EntryRun( * attempt. A run that only waited (for the network, a slot, or a short * backoff) sent nothing, so it goes back to queued with no backoff step. * - * After any time-limit stop a fresh wake row starts the entry again. - * WorkManager also runs the stopped row again, but only after its own - * backoff, which grows with every start of that row. The other stop - * reasons say nothing about the server, and the system decides when their - * row runs again. + * After a time-limit stop a fresh wake row starts the entry again, unless + * WorkManager starts the stopped row again at once. WorkManager runs the + * stopped row again after its own backoff, which grows with every start + * of that row ([WorkManagerBackoff]). For the first few starts that wait + * is already over at the stop; a wake then would be a second row for the + * entry, and one of the two would wait at the per-id gate for the whole + * upload. The other stop reasons say nothing about the server, and the + * system decides when their row runs again. * * The state change is saved first, then the diagnostic record. */ private fun afterStop(kind: StopKind) { when { - kind == StopKind.TIME_LIMIT && stopCutRequest -> releaseAfterTimeout() - kind == StopKind.TIME_LIMIT -> ops.stopped(entryId, generation, wake = true) - else -> ops.stopped(entryId, generation) + kind != StopKind.TIME_LIMIT -> ops.stopped(entryId, generation) + stopCutRequest -> releaseAfterTimeout(wake = !stoppedRowRestartsNow()) + else -> ops.stopped(entryId, generation, wake = !stoppedRowRestartsNow()) } if (kind.logged) Diag.countStop(kind.name, entryId) } + /** WorkManager starts the stopped row again at once. False when unknown. */ + private fun stoppedRowRestartsNow(): Boolean { + val by = host.rowRestartsBy() ?: return false + return by <= host.now() + } + /** * The system stopped the run at its time limit while a request was on its * way. That happens to a run with no foreground mode after about 10 * minutes, or to a foreground run whose service type used up its time. * Starting again at once would send the body from byte 0 on every run, so * this is one more transient failure: the next backoff step, then a wake. + * With [wake] false the stopped row, which WorkManager starts again at + * once, sleeps out a short wait or makes the wake itself ([waitUntilDue]). */ - private fun releaseAfterTimeout() { + private fun releaseAfterTimeout(wake: Boolean) { runCatching { val e = store.load(entryId) if (!EntryTransitions.isOwnedRun(e, generation)) return val streak = e!!.backoffStreak + 1 val now = host.now() val backoff = RetryClassifier.backoffMs(policy(e), streak) - ops.release(entryId, generation, RetryClassifier.nextAttemptAt(now, backoff, e.expiresAt), streak) + ops.release(entryId, generation, RetryClassifier.nextAttemptAt(now, backoff, e.expiresAt), streak, wake) }.onFailure { if (it is WorkerOps.WakeNotScheduledException) { Diag.error("released '$entryId' after a timeout stop, but could not schedule its wake", it.cause) @@ -348,7 +359,9 @@ internal class EntryRun( val result = if (verdict is RetryClassifier.Verdict.Terminal) { AttemptResult.Terminal(verdict.errorKind, verdict.message, null) } else AttemptResult.Transient - return Attempt(entry, url, policy, result) + // The attempt started on a validated network (waitForNetwork), so a + // transport failure waits at most 15 minutes. + return Attempt(entry, url, RetryClassifier.transportFailurePolicy(policy), result) } // A part counts on its own, so each part gets a rule's full maxRetries. diff --git a/android/src/main/java/ai/openspace/backgroundupload/EntryTransitions.kt b/android/src/main/java/ai/openspace/backgroundupload/EntryTransitions.kt index 60a7d423..63556d08 100644 --- a/android/src/main/java/ai/openspace/backgroundupload/EntryTransitions.kt +++ b/android/src/main/java/ai/openspace/backgroundupload/EntryTransitions.kt @@ -1,6 +1,7 @@ package ai.openspace.backgroundupload import kotlin.math.max +import kotlin.math.min /** * The state changes of one entry, as pure functions. [QueueController] and @@ -69,15 +70,32 @@ object EntryTransitions { * resume(): back to awaiting-auth only when no updateHeaders() came in * between. It compares the entry's own headerGeneration, which * updateHeaders() saves with the new headers. The settings value can be - * older after a settings write that never landed. + * older after a settings write that never landed. Back to queued, the + * backoff starts over: the app asked for the entry to run. */ fun toResumed(e: QueueEntry, now: Long): QueueEntry = if (e.parkedGeneration != null && e.parkedGeneration == e.headerGeneration) { e.copy(state = EntryState.AWAITING_AUTH, updatedAt = now) } else { - e.copy(state = EntryState.QUEUED, parkedGeneration = null, updatedAt = now) + e.copy(state = EntryState.QUEUED, parkedGeneration = null, backoffStreak = 0, updatedAt = now) } + /** Whether [e] is queued and waits for a backoff to end. */ + fun waitsOnBackoff(e: QueueEntry) = e.state == EntryState.QUEUED && e.nextAttemptAt != null + + /** + * The app started, resumed, or came to the foreground: a queued entry + * stops waiting for its backoff and runs now. The streak keeps at most + * [RetryClassifier.RETRY_NOW_MAX_STREAK] steps, so the entry sends once + * now, and a server that is still down is not asked again every few + * seconds. + */ + fun toRetryNow(e: QueueEntry, now: Long) = e.copy( + nextAttemptAt = null, + backoffStreak = min(e.backoffStreak, RetryClassifier.RETRY_NOW_MAX_STREAK), + updatedAt = now, + ) + /** * A system stop of a running worker. No outcome. A nextAttemptAt that is * already past is cleared: a queued row does not show a due time behind it. diff --git a/android/src/main/java/ai/openspace/backgroundupload/EntryWorker.kt b/android/src/main/java/ai/openspace/backgroundupload/EntryWorker.kt index f74fae4f..23cedff8 100644 --- a/android/src/main/java/ai/openspace/backgroundupload/EntryWorker.kt +++ b/android/src/main/java/ai/openspace/backgroundupload/EntryWorker.kt @@ -50,7 +50,11 @@ open class EntryWorker(protected val context: Context, params: WorkerParameters) @Volatile private var showsNotification = false + /** When this run began. WorkManager set the row RUNNING before it. */ + private var startedAt = 0L + final override suspend fun doWork(): Result = withContext(Dispatchers.IO) { + startedAt = System.currentTimeMillis() val id = inputData.getString(WorkManagerScheduler.ENTRY_ID_KEY) ?: return@withContext Result.success() // A headless run has no module, so it opens the log that its Diag calls write to. DiagnosticLog.get(context) @@ -117,6 +121,9 @@ open class EntryWorker(protected val context: Context, params: WorkerParameters) // every stop takes the plain path: back to queued, no backoff step. override fun stopKind() = if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.S) StopKind.of(stopReason) else StopKind.OTHER + + // runAttemptCount is the row's count before this start. + override fun rowRestartsBy() = WorkManagerBackoff.restartBy(startedAt, runAttemptCount) } // MARK: - notification diff --git a/android/src/main/java/ai/openspace/backgroundupload/ForegroundTask.kt b/android/src/main/java/ai/openspace/backgroundupload/ForegroundTask.kt new file mode 100644 index 00000000..d1392b4d --- /dev/null +++ b/android/src/main/java/ai/openspace/backgroundupload/ForegroundTask.kt @@ -0,0 +1,23 @@ +package ai.openspace.backgroundupload + +import java.util.concurrent.Executor + +/** + * What the module does when the app comes to the foreground, off the UI + * thread, on [executor] (the queue executor, so it never overlaps another + * queue call). First a settings write that failed earlier is tried again, + * then queued entries stop waiting for their backoff and run now + * ([QueueController.retryNow]). Neither step throws on the executor. + */ +internal class ForegroundTask( + private val executor: Executor, + private val saveSettings: () -> Unit, + private val retryNow: () -> Unit, +) { + fun onForeground() { + executor.execute { + runCatching(saveSettings).onFailure { Diag.error("could not save the settings on return to the foreground", it) } + runCatching(retryNow).onFailure { Diag.error("retryNow on return to the foreground failed", it) } + } + } +} diff --git a/android/src/main/java/ai/openspace/backgroundupload/QueueController.kt b/android/src/main/java/ai/openspace/backgroundupload/QueueController.kt index a1e251f3..d5217ddb 100644 --- a/android/src/main/java/ai/openspace/backgroundupload/QueueController.kt +++ b/android/src/main/java/ai/openspace/backgroundupload/QueueController.kt @@ -210,6 +210,8 @@ class QueueController( * Undoes the pause of the same scope: [keys] null turns the gate off, a * list removes those keys from the set. A row comes back only when no * scope still pauses it, so the gate on + its key resumed stays paused. + * A resumed entry starts its backoff over. A queued entry of the same + * scope that waits for a backoff runs now, as with [retryNow]. * * A failed entry save throws E_STORAGE after every other entry is moved * and every queued entry is scheduled. The entry that failed stays paused. @@ -221,7 +223,11 @@ class QueueController( } val now = clock() val result = transformAll { e -> - if (e.state == EntryState.PAUSED && !s.isPaused(e.key)) EntryTransitions.toResumed(e, now) else e + when { + e.state == EntryState.PAUSED && !s.isPaused(e.key) -> EntryTransitions.toResumed(e, now) + inScope(e, keys) && EntryTransitions.waitsOnBackoff(e) -> EntryTransitions.toRetryNow(e, now) + else -> e + } } result.changed.forEach { events.state(it.toRow()) } // Every queued entry, not only the resumed ones: a run is idempotent. @@ -229,6 +235,59 @@ class QueueController( result.throwIfFailed("resume") } + /** + * The app started, came to the foreground, or resumed: queued entries in + * scope run now. [keys] null is every key; an empty list changes nothing. + * + * An entry that waits for a backoff stops waiting ([EntryTransitions.toRetryNow]) + * and is scheduled. A queued entry with no wait is scheduled only when no + * row of it can start now ([WorkScheduler.startableIds], one query): a row + * that WorkManager holds back after a stop is replaced ([WorkScheduler.schedule]). + * Running, paused, parked and settled entries are left alone. A queued + * entry sends nothing, and the per-id gate holds any new row while a + * worker of the entry still winds down, so this never starts a second + * transfer next to one in progress. + * + * It reads the queue once, and saves only the entries that wait. Never + * throws: a failed entry save keeps that entry's wait and writes an + * error; the other entries go on. + */ + fun retryNow(keys: List? = null) { + if (keys != null && keys.isEmpty()) return + val now = clock() + val queued = store.all().filter { !it.legacy && it.state == EntryState.QUEUED && inScope(it, keys) } + val cleared = mutableListOf() + var failures = 0 + var first: IOException? = null + store.locked { + queued.filter { EntryTransitions.waitsOnBackoff(it) }.forEach { e -> + try { + var next: QueueEntry? = null + // The snapshot can be stale: the store decides under the lock. + store.compute(e.id) { cur -> + if (cur != null && EntryTransitions.waitsOnBackoff(cur)) EntryTransitions.toRetryNow(cur, now).also { next = it } else cur + } + next?.let { cleared += it } + } catch (error: IOException) { + failures++ + if (first == null) first = error + } + } + } + cleared.forEach { scheduleRun(it) } + val idle = queued.filter { it.nextAttemptAt == null } + if (idle.isNotEmpty()) { + val startable = runCatching { scheduler.startableIds() } + .onFailure { Diag.error("retryNow could not read WorkManager; it schedules every queued entry", it) } + .getOrNull() + idle.filter { startable == null || it.id !in startable }.forEach { scheduleRun(it) } + } + cleared.forEach { events.state(it.toRow()) } + first?.let { Diag.error("retryNow could not save $failures entries; they keep their wait", it) } + } + + private fun inScope(e: QueueEntry, keys: List?) = keys == null || e.key in keys + /** * Live: journal a 'cancelled' (user) outcome, then forget after its ack. * Settled (or legacy): forget now, row, bytes, and its unacknowledged @@ -428,11 +487,15 @@ class QueueController( * 2. A running entry with no worker: a process death mid-run. Queue it. * A live row that disagrees with the paused settings, the gate or its * key (a death partway through pause() or resume()): make it agree. + * A queued entry that waits for a backoff runs now, with at most 5 + * backoff steps kept ([EntryTransitions.toRetryNow]). * 3. A settled entry with records of its generation other than its own: * orphans from a cancel race. Ack them. * 4. A completed or cancelled entry whose own record is gone: the ack * landed but the forget did not. Forget it. * 5. Schedule every queued entry, and the expiry wake of every parked one. + * A row that WorkManager holds back after a stop is replaced + * ([WorkScheduler.schedule]), so each queued entry can start now. * * 6. Delete the orphan directories that hold only copied bodies and are * older than [QueueStore.ORPHAN_MIN_AGE_MS] ([QueueStore.removeOrphans]). @@ -486,6 +549,10 @@ class QueueController( } else if (!paused && cur.state == EntryState.PAUSED) { cur = EntryTransitions.toResumed(cur, now) } + // The app started: a queued entry no longer waits for its backoff. + if (EntryTransitions.waitsOnBackoff(cur)) { + cur = EntryTransitions.toRetryNow(cur, now) + } if (cur !== e) { if (trySave(cur)) changed += cur else continue } diff --git a/android/src/main/java/ai/openspace/backgroundupload/RetryClassifier.kt b/android/src/main/java/ai/openspace/backgroundupload/RetryClassifier.kt index edbe4e15..19491aeb 100644 --- a/android/src/main/java/ai/openspace/backgroundupload/RetryClassifier.kt +++ b/android/src/main/java/ai/openspace/backgroundupload/RetryClassifier.kt @@ -40,6 +40,25 @@ object RetryClassifier { /** A wait up to this long happens inside the worker. A longer one releases the worker. */ const val IN_WORKER_BACKOFF_MAX_MS = 30_000L + /** + * The longest wait after a transport failure (no HTTP response). Every + * attempt starts on a network Android reports as validated, so the failure + * says little about the server, and a network change cannot wake a queued + * entry. HTTP failures keep the policy's maxMs. + */ + const val TRANSPORT_FAILURE_MAX_MS = 15L * 60 * 1000 + + /** + * The highest backoff streak an entry keeps when the app clears its wait. + * Streak 5 is the last wait a worker sleeps out in place (16 s at the + * default base); the next failure waits about 32 s, then the wait doubles. + */ + const val RETRY_NOW_MAX_STREAK = 5 + + /** [policy] for the wait after a transport failure: maxMs at most [TRANSPORT_FAILURE_MAX_MS]. */ + fun transportFailurePolicy(policy: Policy): Policy = + policy.copy(maxMs = min(policy.maxMs, TRANSPORT_FAILURE_MAX_MS)) + fun policy(defaults: RetryDefaults, override: RetryOverride?): Policy = Policy( baseMs = override?.baseMs ?: defaults.baseMs, maxMs = override?.maxMs ?: defaults.maxMs, diff --git a/android/src/main/java/ai/openspace/backgroundupload/Scheduler.kt b/android/src/main/java/ai/openspace/backgroundupload/Scheduler.kt index 53e02f59..8d2cf902 100644 --- a/android/src/main/java/ai/openspace/backgroundupload/Scheduler.kt +++ b/android/src/main/java/ai/openspace/backgroundupload/Scheduler.kt @@ -8,6 +8,7 @@ import androidx.work.WorkInfo import androidx.work.WorkManager import androidx.work.workDataOf import java.util.concurrent.TimeUnit +import kotlin.math.min /** * How far ahead a parked entry's expiry wake may be scheduled. A delayed @@ -23,13 +24,77 @@ const val EXPIRY_WAKE_HORIZON_MS = 24L * 60 * 60 * 1000 /** Whether an expiry wake at [expiresAt] is near enough to schedule now. */ fun expiryWakeIsNear(expiresAt: Long, now: Long): Boolean = expiresAt - now <= EXPIRY_WAKE_HORIZON_MS +/** + * When WorkManager 2.12.0 starts a row again after a system stop. Read from + * the 2.12.0 classes: a stop sets the row back to ENQUEUED and keeps its + * runAttemptCount and its last enqueue time (WorkerWrapper.resetWorkerStatus, + * for a row with no backoff for system interruptions, which the library + * never sets). An ENQUEUED row with a count above 0 then starts at + * lastEnqueueTime + min(30 s * 2^(count - 1), 5 h), and not before + * (WorkSpec.calculateNextRunTime, and the due check in WorkerWrapper). The + * library's rows use the default backoff: 30 s, exponential. + */ +internal object WorkManagerBackoff { + const val DELAY_MS = 30_000L + const val MAX_MS = 18_000_000L + + /** + * The latest time WorkManager starts a row again after a stop of the run + * that began at [startedAt]. [runAttemptCountAtStart] is the worker's + * runAttemptCount: the starts before this one. The row's last enqueue time + * is at or before [startedAt], so the real time is at or before this one. + */ + fun restartBy(startedAt: Long, runAttemptCountAtStart: Int): Long { + val exponent = runAttemptCountAtStart.coerceIn(0, 40) + return startedAt + min(DELAY_MS.toDouble() * Math.pow(2.0, exponent.toDouble()), MAX_MS.toDouble()).toLong() + } +} + +/** What [RowScheduler.schedule] does to an entry's main chain. */ +internal enum class MainChainStep { NONE, APPEND, REPLACE } + +/** + * The step for a "run now" on a main chain with [rows] (its unfinished + * rows). [workerRunning]: a worker of the entry runs in this process. + * + * - No row: a fresh chain (APPEND_OR_REPLACE starts one). + * - A run in progress (a RUNNING row, or a worker in this process): a + * queued row after it is enough; with none, append one. + * - No run in progress: a row WorkManager starts now is enough. When every + * queued row waits, REPLACE them with a fresh row. That is a row a stop + * left on WorkManager's backoff (see [WorkManagerBackoff]), and the rows + * blocked behind it. Nothing runs, so the replace cuts no request. + * WorkManager does not start a row before its time, and a row due within + * [START_MARGIN_MS] counts as starting now, so no row starts between the + * read and the replace. + */ +internal fun mainChainStep(rows: List, workerRunning: Boolean, now: Long): MainChainStep { + val open = rows.filter { !it.state.isFinished } + if (open.isEmpty()) return MainChainStep.APPEND + if (workerRunning || open.any { it.state == WorkInfo.State.RUNNING }) { + return if (WorkManagerScheduler.hasQueuedSuccessor(open.map { it.state })) MainChainStep.NONE else MainChainStep.APPEND + } + if (open.any { it.startsBy(now) }) return MainChainStep.NONE + return MainChainStep.REPLACE +} + +/** A row due this soon counts as one that starts now. */ +internal const val START_MARGIN_MS = 5_000L + +/** An ENQUEUED row that WorkManager may start now, or within [START_MARGIN_MS]. */ +internal fun WorkRow.startsBy(now: Long) = state == WorkInfo.State.ENQUEUED && nextScheduleAt <= now + START_MARGIN_MS + /** * Starts and stops worker runs for entries. A run only carries the entry id; * the worker reads everything else from the store. So an extra run is always * harmless: it finds nothing to do and exits. */ interface WorkScheduler { - /** A run as soon as possible, on the entry's main chain. */ + /** + * A run as soon as possible, on the entry's main chain. A row that + * WorkManager holds back after a stop is replaced, so the entry can start + * now. + */ fun schedule(entry: QueueEntry) /** @@ -47,6 +112,12 @@ interface WorkScheduler { /** Every unfinished WorkManager run of the library, main chains and wakes. For the boot snapshot. */ fun libraryRuns(): List + + /** + * The entry ids that have a run in progress, or a row WorkManager may + * start now. One query for the whole library. + */ + fun startableIds(): Set } /** @@ -62,6 +133,32 @@ data class PendingRun( val delayed: Boolean, ) +/** + * One unfinished WorkManager row. [nextScheduleAt] is the earliest time + * WorkManager starts it (`WorkInfo.nextScheduleTimeMillis`): Long.MAX_VALUE + * for a row that is not ENQUEUED. + */ +data class WorkRow( + val entryId: String?, + val state: WorkInfo.State, + val runAttemptCount: Int, + val nextScheduleAt: Long, + val delayed: Boolean, +) + +/** The WorkManager calls [RowScheduler] makes. Tests run the same rules against a fake. */ +interface WorkRows { + /** The unfinished rows of one unique name. */ + fun unfinished(name: String): List + + /** Every unfinished row of the library, main chains and wakes. */ + fun library(): List + + fun enqueue(name: String, policy: ExistingWorkPolicy, entry: QueueEntry, delayMs: Long) + + fun cancel(name: String) +} + /** * WorkManager unique work per entry id, APPEND_OR_REPLACE, as v9. * @@ -79,76 +176,122 @@ data class PendingRun( * No WorkManager Constraints: connectivity and wifi-only are checked inside * the worker, as v9, because the constraint path was unreliable. */ -class WorkManagerScheduler(context: Context) : WorkScheduler { - companion object { - /** v9 rows carry the tag "RNFileUploader"; this one is new so the v9 cancel does not touch v10 work. */ - const val WORK_TAG = "RNFileUploader.v10" - const val ID_TAG_PREFIX = "RNFileUploaderId:" - /** A string literal, because WorkManager persists it across builds. */ - const val ENTRY_ID_KEY = "entryId" - const val V9_WORK_TAG = "RNFileUploader" - - fun wakeName(id: String) = "$id#wake" - - /** Milliseconds from [now] until [at]; never negative. */ - fun initialDelayMs(at: Long?, now: Long): Long = if (at == null) 0L else (at - now).coerceAtLeast(0L) - - /** An unfinished row that is not RUNNING: a queued run that did not start yet. */ - fun hasQueuedSuccessor(states: List): Boolean = - states.any { !it.isFinished && it != WorkInfo.State.RUNNING } - } - - private val workManager = WorkManager.getInstance(context) +open class RowScheduler( + private val rows: WorkRows, + private val isWorkerRunning: (id: String) -> Boolean = WorkerGate::isRunning, + private val clock: () -> Long = System::currentTimeMillis, +) : WorkScheduler { + /** + * Makes sure the entry has a row that starts now or after the run in + * progress. See [mainChainStep]. + */ override fun schedule(entry: QueueEntry) { - // A queued successor already guarantees a run after the current one. - val states = workManager.getWorkInfosForUniqueWork(entry.id).get().map { it.state } - if (hasQueuedSuccessor(states)) return - workManager - .beginUniqueWork(entry.id, ExistingWorkPolicy.APPEND_OR_REPLACE, request(entry, 0L)) - .enqueue() + when (mainChainStep(rows.unfinished(entry.id), isWorkerRunning(entry.id), clock())) { + MainChainStep.NONE -> Unit + MainChainStep.APPEND -> rows.enqueue(entry.id, ExistingWorkPolicy.APPEND_OR_REPLACE, entry, 0L) + MainChainStep.REPLACE -> rows.enqueue(entry.id, ExistingWorkPolicy.REPLACE, entry, 0L) + } } override fun scheduleWake(entry: QueueEntry, at: Long, replace: Boolean) { - val delay = initialDelayMs(at, System.currentTimeMillis()) - workManager.enqueueUniqueWork( - wakeName(entry.id), + val delay = WorkManagerScheduler.initialDelayMs(at, clock()) + rows.enqueue( + WorkManagerScheduler.wakeName(entry.id), if (replace) ExistingWorkPolicy.REPLACE else ExistingWorkPolicy.KEEP, - request(entry, delay), + entry, + delay, ) } override fun cancel(id: String) { - workManager.cancelUniqueWork(id) - workManager.cancelUniqueWork(wakeName(id)) + rows.cancel(id) + rows.cancel(WorkManagerScheduler.wakeName(id)) } // A cancel reschedules all pending work even when the name has none, and // the boot sweep calls this for every queued entry. override fun cancelWake(id: String) { - val states = workManager.getWorkInfosForUniqueWork(wakeName(id)).get().map { it.state } - if (states.any { !it.isFinished }) workManager.cancelUniqueWork(wakeName(id)) + val name = WorkManagerScheduler.wakeName(id) + if (rows.unfinished(name).isNotEmpty()) rows.cancel(name) } // One query for every row, not one per entry: the boot sweep runs this on // the queue's only thread. override fun libraryRuns(): List = - workManager.getWorkInfosByTag(WORK_TAG).get() - .filter { !it.state.isFinished } - .map { info -> - PendingRun( - entryId = info.tags.firstOrNull { it.startsWith(ID_TAG_PREFIX) }?.removePrefix(ID_TAG_PREFIX), - state = info.state.name, - runAttemptCount = info.runAttemptCount, - nextScheduleAt = info.nextScheduleTimeMillis.takeIf { it != Long.MAX_VALUE }, - delayed = info.initialDelayMillis > 0, - ) - } + rows.library().map { row -> + PendingRun( + entryId = row.entryId, + state = row.state.name, + runAttemptCount = row.runAttemptCount, + nextScheduleAt = row.nextScheduleAt.takeIf { it != Long.MAX_VALUE }, + delayed = row.delayed, + ) + } + + override fun startableIds(): Set { + val now = clock() + return rows.library() + .filter { it.state == WorkInfo.State.RUNNING || it.startsBy(now) } + .mapNotNull { it.entryId } + .toSet() + } +} + +/** [RowScheduler] on the app's WorkManager. */ +class WorkManagerScheduler private constructor(private val workManager: WorkManager) : + RowScheduler(WorkManagerRows(workManager)) { + + constructor(context: Context) : this(WorkManager.getInstance(context)) + + companion object { + /** v9 rows carry the tag "RNFileUploader"; this one is new so the v9 cancel does not touch v10 work. */ + const val WORK_TAG = "RNFileUploader.v10" + const val ID_TAG_PREFIX = "RNFileUploaderId:" + /** A string literal, because WorkManager persists it across builds. */ + const val ENTRY_ID_KEY = "entryId" + const val V9_WORK_TAG = "RNFileUploader" + + fun wakeName(id: String) = "$id#wake" + + /** Milliseconds from [now] until [at]; never negative. */ + fun initialDelayMs(at: Long?, now: Long): Long = if (at == null) 0L else (at - now).coerceAtLeast(0L) + + /** An unfinished row that is not RUNNING: a queued run that did not start yet. */ + fun hasQueuedSuccessor(states: List): Boolean = + states.any { !it.isFinished && it != WorkInfo.State.RUNNING } + } /** First v10 launch: the v9 rows. Their workers are gone. */ fun cancelV9Work() { workManager.cancelAllWorkByTag(V9_WORK_TAG) } +} + +/** [WorkRows] on WorkManager. */ +private class WorkManagerRows(private val workManager: WorkManager) : WorkRows { + override fun unfinished(name: String): List = + workManager.getWorkInfosForUniqueWork(name).get().filter { !it.state.isFinished }.map(::row) + + override fun library(): List = + workManager.getWorkInfosByTag(WorkManagerScheduler.WORK_TAG).get().filter { !it.state.isFinished }.map(::row) + + override fun enqueue(name: String, policy: ExistingWorkPolicy, entry: QueueEntry, delayMs: Long) { + workManager.beginUniqueWork(name, policy, request(entry, delayMs)).enqueue() + } + + override fun cancel(name: String) { + workManager.cancelUniqueWork(name) + } + + private fun row(info: WorkInfo) = WorkRow( + entryId = info.tags.firstOrNull { it.startsWith(WorkManagerScheduler.ID_TAG_PREFIX) } + ?.removePrefix(WorkManagerScheduler.ID_TAG_PREFIX), + state = info.state, + runAttemptCount = info.runAttemptCount, + nextScheduleAt = info.nextScheduleTimeMillis, + delayed = info.initialDelayMillis > 0, + ) private fun request(entry: QueueEntry, delayMs: Long): OneTimeWorkRequest { val builder = if (entry.body?.kind == StagedBody.CHUNKED) { @@ -156,10 +299,13 @@ class WorkManagerScheduler(context: Context) : WorkScheduler { } else { OneTimeWorkRequestBuilder() } + // No setBackoffCriteria and no backoff for system interruptions: a + // stopped row keeps WorkManager's default (30 s, exponential, from its + // last enqueue time). [WorkManagerBackoff] relies on that. return builder - .addTag(WORK_TAG) - .addTag(ID_TAG_PREFIX + entry.id) - .setInputData(workDataOf(ENTRY_ID_KEY to entry.id)) + .addTag(WorkManagerScheduler.WORK_TAG) + .addTag(WorkManagerScheduler.ID_TAG_PREFIX + entry.id) + .setInputData(workDataOf(WorkManagerScheduler.ENTRY_ID_KEY to entry.id)) .setInitialDelay(delayMs, TimeUnit.MILLISECONDS) .build() } diff --git a/android/src/main/java/ai/openspace/backgroundupload/TransferHost.kt b/android/src/main/java/ai/openspace/backgroundupload/TransferHost.kt index 0b4a71c1..5cd57653 100644 --- a/android/src/main/java/ai/openspace/backgroundupload/TransferHost.kt +++ b/android/src/main/java/ai/openspace/backgroundupload/TransferHost.kt @@ -39,6 +39,12 @@ internal interface TransferHost { /** Why the system stopped this run. Read it only after the run was cancelled. */ fun stopKind(): StopKind + + /** + * The latest time WorkManager starts this run's own row again after a + * system stop ([WorkManagerBackoff.restartBy]). Null when unknown. + */ + fun rowRestartsBy(): Long? } /** diff --git a/android/src/main/java/ai/openspace/backgroundupload/UploaderModule.kt b/android/src/main/java/ai/openspace/backgroundupload/UploaderModule.kt index 349215bf..1f50951f 100644 --- a/android/src/main/java/ai/openspace/backgroundupload/UploaderModule.kt +++ b/android/src/main/java/ai/openspace/backgroundupload/UploaderModule.kt @@ -55,6 +55,9 @@ class UploaderModule(context: ReactApplicationContext) : private val scheduler = WorkManagerScheduler(context) private val controller = QueueController(store, journal, settings, EventReporter, scheduler) + // Before init: the lifecycle listener it registers can fire at once. + private val foreground = ForegroundTask(queueExecutor, { settings.saveIfBehind() }, { controller.retryNow() }) + // The v9 import, then the v9 work cancel, then the boot sweep. // getRequests() waits for the import only (file reads and row saves), so // the first call after an upgrade already shows the legacy rows. The @@ -79,11 +82,10 @@ class UploaderModule(context: ReactApplicationContext) : // A settings write that failed earlier is tried again each time the app // comes to the foreground, and when it leaves: the last chance before a - // relaunch with no JS reads the file. Off the UI thread, on the queue - // executor. - override fun onHostResume() { - queueExecutor.execute { settings.saveIfBehind() } - } + // relaunch with no JS reads the file. On the way in, queued entries also + // stop waiting for their backoff and run now. Off the UI thread, on the + // queue executor, after the boot sweep at the first launch. + override fun onHostResume() = foreground.onForeground() override fun onHostPause() { queueExecutor.execute { settings.saveIfBehind() } diff --git a/android/src/main/java/ai/openspace/backgroundupload/WorkerOps.kt b/android/src/main/java/ai/openspace/backgroundupload/WorkerOps.kt index 55e98dc1..0ea0fc43 100644 --- a/android/src/main/java/ai/openspace/backgroundupload/WorkerOps.kt +++ b/android/src/main/java/ai/openspace/backgroundupload/WorkerOps.kt @@ -318,8 +318,13 @@ class WorkerOps( return result } - /** A backoff longer than a worker waits: back to queued, woken at [nextAttemptAt]. */ - fun release(id: String, generation: Int, nextAttemptAt: Long, streak: Int): Boolean { + /** + * A backoff longer than a worker waits: back to queued, woken at + * [nextAttemptAt]. [wake] false saves the release with no wake row: the + * caller knows WorkManager starts the run's own row again at once, and + * that run makes the wake ([ensureWake]). + */ + fun release(id: String, generation: Int, nextAttemptAt: Long, streak: Int, wake: Boolean = true): Boolean { val now = clock() var applied = false val next = store.compute(id) { e -> @@ -330,6 +335,7 @@ class WorkerOps( } if (!applied || next == null) return false events.state(next.toRow()) + if (!wake) return true try { scheduler.scheduleWake(next, nextAttemptAt, replace = true) } catch (error: Exception) { @@ -358,10 +364,10 @@ class WorkerOps( * running one moves. Never journals. * * [wake] true also schedules a fresh wake row, at the entry's - * nextAttemptAt or now. A time-limit stop needs it: WorkManager runs the - * stopped row again only after its own backoff, which grows with every - * start of that row (up to 5 hours), and while that row waits, schedule() - * adds no row. A fresh row starts its count at 0. + * nextAttemptAt or now. A time-limit stop needs it when WorkManager holds + * the stopped row back: it runs that row again only after its own + * backoff, which grows with every start of that row (up to 5 hours). A + * fresh row starts its count at 0. */ fun stopped(id: String, generation: Int, wake: Boolean = false) { val now = clock() diff --git a/android/src/test/java/ai/openspace/backgroundupload/EntryRunTest.kt b/android/src/test/java/ai/openspace/backgroundupload/EntryRunTest.kt index 306b7bf7..64378192 100644 --- a/android/src/test/java/ai/openspace/backgroundupload/EntryRunTest.kt +++ b/android/src/test/java/ai/openspace/backgroundupload/EntryRunTest.kt @@ -87,6 +87,11 @@ internal class FakeHost(var clock: Long = 10_000L) : TransferHost { } override fun stopKind() = stop + + /** What rowRestartsBy() answers. Null: unknown. */ + var restartsBy: Long? = null + + override fun rowRestartsBy() = restartsBy } class EntryRunTest { @@ -208,6 +213,38 @@ class EntryRunTest { assertEquals(EntryState.COMPLETED, store.load("e1")!!.state) } + @Test + fun `a transport failure waits at most 15 minutes, an HTTP failure keeps the 2 hour maximum`() { + controller.configureRetry(RetryDefaults(baseMs = 1_000, maxMs = 7_200_000, jitter = 0.0)) + controller.enqueue(parsed()) + store.save(store.load("e1")!!.copy(backoffStreak = 12)) + host.handler = { _, _ -> throw IOException("connection reset") } + run() + // Streak 13 is 2^12 s, about 68 minutes, capped at 15. + assertEquals(host.clock + RetryClassifier.TRANSPORT_FAILURE_MAX_MS, store.load("e1")!!.nextAttemptAt) + assertEquals(13, store.load("e1")!!.backoffStreak) + + controller.enqueue(parsed(id = "e2")) + store.save(store.load("e2")!!.copy(backoffStreak = 12)) + respond(503) + run("e2") + assertEquals(host.clock + 4_096_000, store.load("e2")!!.nextAttemptAt) + store.save(store.load("e2")!!.copy(nextAttemptAt = null, backoffStreak = 13)) + respond(503) + run("e2") + assertEquals(host.clock + 7_200_000, store.load("e2")!!.nextAttemptAt) + } + + @Test + fun `a transport failure keeps a maximum below 15 minutes`() { + controller.configureRetry(RetryDefaults(baseMs = 1_000, maxMs = 60_000, jitter = 0.0)) + controller.enqueue(parsed()) + store.save(store.load("e1")!!.copy(backoffStreak = 12)) + host.handler = { _, _ -> throw IOException("connection reset") } + run() + assertEquals(host.clock + 60_000, store.load("e1")!!.nextAttemptAt) + } + @Test fun `a terminal response settles error http with the response and the live bytes`() { controller.enqueue(parsed()) @@ -443,6 +480,54 @@ class EntryRunTest { assertEquals(listOf(true), scheduler.wakeReplaces) } + @Test + fun `an idle timeout stop adds no fresh row when WorkManager starts the stopped row again at once`() { + controller.enqueue(parsed()) + host.stop = StopKind.TIME_LIMIT + host.restartsBy = host.clock // the stopped row is due now + host.network = ArrayDeque(listOf(Connectivity.NoInternet)) + host.onSleep = { throw CancellationException("timeout") } + assertThrows(CancellationException::class.java) { run() } + val e = store.load("e1")!! + assertEquals(EntryState.QUEUED, e.state) + assertNull(e.nextAttemptAt) + assertEquals(emptyList>(), scheduler.wakes) + } + + @Test + fun `an idle timeout stop adds a fresh row when WorkManager holds the stopped row back`() { + controller.enqueue(parsed()) + host.stop = StopKind.TIME_LIMIT + host.network = ArrayDeque(listOf(Connectivity.NoInternet)) + host.onSleep = { + host.restartsBy = host.clock + 1 // due a moment after the stop + throw CancellationException("timeout") + } + assertThrows(CancellationException::class.java) { run() } + assertEquals(listOf("e1" to host.clock), scheduler.wakes) + } + + @Test + fun `a timeout stop that cut a request adds no wake when the stopped row starts again at once`() { + controller.enqueue(parsed()) + store.save(store.load("e1")!!.copy(backoffStreak = 6)) + host.stop = StopKind.TIME_LIMIT + host.restartsBy = host.clock - 1 + host.handler = { _, _ -> throw CancellationException("timeout") } + assertThrows(CancellationException::class.java) { run() } + val e = store.load("e1")!! + assertEquals(EntryState.QUEUED, e.state) + assertEquals(7, e.backoffStreak) + val at = host.clock + 64_000 + assertEquals(at, e.nextAttemptAt) + assertEquals(emptyList>(), scheduler.wakes) + + // The stopped row runs again, sees the long wait and makes the wake itself. + host.restartsBy = null + run() + assertEquals(listOf("e1" to at), scheduler.wakes) + } + @Test fun `a timeout stop while it waits for a transmit slot counts no attempt`() { controller.enqueue(parsed()) @@ -910,6 +995,34 @@ class EntryRunTest { assertEquals(listOf(1_000L), host.sleeps) } + @Test + fun `a chunked entry stored at a long streak completes after 200, 503, 200 with no wake`() { + val src = File(tmp.newFolder(), "video.bin").apply { writeBytes(ByteArray(20) { it.toByte() }) } + controller.enqueue(parsed(descriptor = desc(url = null, method = "PUT", file = src.path, + parts = listOf(part(0, 10), part(10, 20))))) + store.save(store.load("e1")!!.copy(backoffStreak = 9)) + respond(200, 503, 200) + run() + val e = store.load("e1")!! + assertEquals(EntryState.COMPLETED, e.state) + assertEquals(3, host.requests.size) + // The 503 came after a part was accepted, so it waits 1 s in place, not 2^9 s. + assertEquals(listOf(1_000L), host.sleeps) + assertEquals(emptyList>(), scheduler.wakes) + } + + @Test + fun `a chunked part keeps the stored streak until a part is accepted in this run`() { + chunked() + store.save(store.load("e1")!!.copy(backoffStreak = 9)) + respond(503) + run() + val e = store.load("e1")!! + assertEquals(EntryState.QUEUED, e.state) + assertEquals(10, e.backoffStreak) + assertEquals(listOf("e1" to host.clock + 512_000), scheduler.wakes) // base 1 s * 2^(10-1) + } + /** Four 10-byte parts. [code] answers each request by its part index and how many times that part was sent. */ private fun fourParts(code: (part: Int, sent: Int) -> Int) { val src = File(tmp.newFolder(), "video.bin").apply { writeBytes(ByteArray(40) { it.toByte() }) } diff --git a/android/src/test/java/ai/openspace/backgroundupload/EntryTransitionsTest.kt b/android/src/test/java/ai/openspace/backgroundupload/EntryTransitionsTest.kt index 0d6eec64..43a7d2db 100644 --- a/android/src/test/java/ai/openspace/backgroundupload/EntryTransitionsTest.kt +++ b/android/src/test/java/ai/openspace/backgroundupload/EntryTransitionsTest.kt @@ -52,6 +52,14 @@ class EntryTransitionsTest { assertEquals(EntryState.QUEUED, EntryTransitions.toResumed(entry(state = EntryState.PAUSED), 9).state) } + @Test + fun `resume to queued starts the backoff over, resume to awaiting-auth keeps it`() { + val paused = entry(state = EntryState.PAUSED).copy(backoffStreak = 9) + assertEquals(0, EntryTransitions.toResumed(paused, 9).backoffStreak) + val parked = paused.copy(parkedGeneration = 3, headerGeneration = 3) + assertEquals(9, EntryTransitions.toResumed(parked, 9).backoffStreak) + } + @Test fun `pause keeps the parked generation and clears the wake time`() { val p = EntryTransitions.toPaused(entry(state = EntryState.AWAITING_AUTH, parkedGeneration = 2, nextAttemptAt = 50), 9) diff --git a/android/src/test/java/ai/openspace/backgroundupload/ForegroundTaskTest.kt b/android/src/test/java/ai/openspace/backgroundupload/ForegroundTaskTest.kt new file mode 100644 index 00000000..d3ee6e0e --- /dev/null +++ b/android/src/test/java/ai/openspace/backgroundupload/ForegroundTaskTest.kt @@ -0,0 +1,65 @@ +package ai.openspace.backgroundupload + +import org.junit.After +import org.junit.Assert.assertEquals +import org.junit.Rule +import org.junit.Test +import org.junit.rules.TemporaryFolder +import java.io.File +import java.util.concurrent.Executor + +class ForegroundTaskTest { + @get:Rule + val tmp = TemporaryFolder() + + @After + fun tearDown() { + Diag.log = null + } + + /** Holds each task until the test runs it, as the queue executor would later. */ + private class HeldExecutor : Executor { + val tasks = mutableListOf() + + override fun execute(command: Runnable) { + tasks += command + } + + fun runAll() { + val now = tasks.toList() + tasks.clear() + now.forEach { it.run() } + } + } + + @Test + fun `a return to the foreground calls retryNow once, on the queue executor`() { + val executor = HeldExecutor() + val calls = mutableListOf() + val task = ForegroundTask(executor, { calls += "save" }, { calls += "retryNow" }) + task.onForeground() + assertEquals(emptyList(), calls) // nothing on the calling (UI) thread + executor.runAll() + assertEquals(listOf("save", "retryNow"), calls) + } + + @Test + fun `a failed settings save still runs retryNow, and neither throws on the executor`() { + val file = File(tmp.newFolder(), "diagnostics.json") + Diag.log = DiagnosticLog(file) + val executor = HeldExecutor() + var retried = false + val task = ForegroundTask(executor, { throw IllegalStateException("settings") }) { + retried = true + throw IllegalStateException("work") + } + task.onForeground() + executor.runAll() + assertEquals(true, retried) + val errors = DiagnosticLog(file).drain() + assertEquals( + listOf("could not save the settings on return to the foreground", "retryNow on return to the foreground failed"), + errors.map { it.message }, + ) + } +} diff --git a/android/src/test/java/ai/openspace/backgroundupload/MainChainTest.kt b/android/src/test/java/ai/openspace/backgroundupload/MainChainTest.kt new file mode 100644 index 00000000..920aae06 --- /dev/null +++ b/android/src/test/java/ai/openspace/backgroundupload/MainChainTest.kt @@ -0,0 +1,196 @@ +package ai.openspace.backgroundupload + +import androidx.work.WorkInfo.State.BLOCKED +import androidx.work.WorkInfo.State.CANCELLED +import androidx.work.WorkInfo.State.ENQUEUED +import androidx.work.WorkInfo.State.RUNNING +import org.junit.Assert.assertEquals +import org.junit.Assert.assertTrue +import org.junit.Before +import org.junit.Rule +import org.junit.Test +import org.junit.rules.TemporaryFolder +import java.io.File + +/** + * The library's scheduling rules ([RowScheduler]) against a fake that + * follows WorkManager 2.12.0 ([FakeWorkManager]). The question each test + * asks: after the app acts, does the entry have a row WorkManager may start + * now? + */ +class MainChainTest { + @get:Rule + val tmp = TemporaryFolder() + + private val minute = 60_000L + private val work = FakeWorkManager(now = 100 * 60_000L) + private val running = mutableSetOf() + private lateinit var store: QueueStore + private lateinit var settings: QueueSettingsStore + private lateinit var controller: QueueController + private lateinit var scheduler: RowScheduler + + @Before + fun setUp() { + val root = tmp.newFolder("queue") + store = QueueStore(root, RequestIndex()) + val journal = EventJournal(tmp.newFolder("journal"), retryLater = { _, _ -> }) + settings = QueueSettingsStore(File(root, "settings.json")) + scheduler = RowScheduler(work, { it in running }, { work.now }) + controller = QueueController(store, journal, settings, RecordingEvents(), scheduler, { it in running }, { work.now }) + } + + /** The rows of [id] WorkManager may start now, main chain and wake. */ + private fun startable(id: String = "e1") = work.startableNow().filter { it.entryId == id } + + /** + * The L2 case: a main-chain row stopped at the job time limit at its 8th + * start, and the fresh wake row the stop scheduled. WorkManager holds the + * stopped row for 64 minutes from its last enqueue time. + */ + private fun stoppedAtEighthStartWithWake() { + store.save(entry(state = EntryState.QUEUED)) + work.add("e1", runAttemptCount = 8, lastEnqueueTime = work.now - 10 * minute) + work.add("e1#wake") + } + + // MARK: - the fake follows WorkManager 2.12.0 + + @Test + fun `a stopped row waits 30 s doubled per start from its last enqueue, at most 5 hours`() { + val enqueued = work.now + fun waitAfter(starts: Int) = work.nextScheduleAt(work.add("x$starts", runAttemptCount = starts, lastEnqueueTime = enqueued)) - enqueued + assertEquals(30_000L, waitAfter(1)) + assertEquals(64 * minute, waitAfter(8)) + assertEquals(128 * minute, waitAfter(9)) + assertEquals(5 * 60 * minute, waitAfter(20)) + assertEquals(enqueued + 5_000, work.nextScheduleAt(work.add("delayed", initialDelay = 5_000))) + } + + // MARK: - the app acts, and the entry can start now + + @Test + fun `the boot sweep leaves a row that can start now when the stopped main row waits`() { + stoppedAtEighthStartWithWake() + controller.sweep() + assertEquals(1, startable().size) + // One row for the entry, not two: the held-back row is gone. + assertEquals(listOf("e1"), work.rows.filter { !it.state.isFinished }.map { it.name }) + } + + @Test + fun `resume leaves a row that can start now when the stopped main row waits`() { + stoppedAtEighthStartWithWake() + controller.resume() + assertEquals(1, startable().size) + } + + @Test + fun `a same-id enqueue leaves a row that can start now when the stopped main row waits`() { + controller.enqueue(parsed()) + work.rows.clear() + work.add("e1", runAttemptCount = 9, lastEnqueueTime = work.now - minute) + work.add("e1#wake") + controller.enqueue(parsed()) + assertEquals(1, startable().size) + } + + @Test + fun `a return to the foreground starts an entry whose row waits after a quota stop`() { + // A quota stop schedules no wake; WorkManager holds the stopped row. + store.save(entry(state = EntryState.QUEUED)) + work.add("e1", runAttemptCount = 7, lastEnqueueTime = work.now - minute) + controller.retryNow() + assertEquals(1, startable().size) + } + + @Test + fun `a return to the foreground clears a long backoff and starts the entry now`() { + store.save(entry(state = EntryState.QUEUED, nextAttemptAt = work.now + 2 * 60 * minute).copy(backoffStreak = 13)) + work.add("e1#wake", initialDelay = 2 * 60 * minute) + controller.retryNow() + assertEquals(1, startable().size) + assertEquals(listOf("e1"), work.rows.filter { !it.state.isFinished }.map { it.name }) + } + + @Test + fun `the rows blocked behind a held-back row go with it`() { + store.save(entry(state = EntryState.QUEUED)) + work.add("e1", runAttemptCount = 8, lastEnqueueTime = work.now - minute) + work.add("e1", state = BLOCKED) + controller.resume() + assertEquals(1, startable().size) + assertEquals(listOf(CANCELLED, CANCELLED, ENQUEUED), work.rows.map { it.state }) + } + + // MARK: - what stays as it is + + @Test + fun `a stopped row that WorkManager starts now is kept`() { + store.save(entry(state = EntryState.QUEUED)) + val row = work.add("e1", runAttemptCount = 1, lastEnqueueTime = work.now - 10 * minute) + controller.resume() + assertEquals(listOf(row), work.rows) + assertEquals(listOf(row), startable()) + } + + @Test + fun `a stopped row due within the margin is kept, one due later is replaced`() { + store.save(entry(state = EntryState.QUEUED)) + // Started once, so WorkManager holds it 30 s from its last enqueue. + val soon = work.add("e1", runAttemptCount = 1, lastEnqueueTime = work.now - 30_000 + START_MARGIN_MS) + controller.resume() + assertEquals(listOf(soon), work.rows) + soon.lastEnqueueTime += 1 + controller.resume() + assertEquals(CANCELLED, soon.state) + assertEquals(1, startable().size) + } + + @Test + fun `a running row is never replaced`() { + store.save(entry(state = EntryState.QUEUED)) + val row = work.add("e1", state = RUNNING, runAttemptCount = 3) + controller.resume() + assertEquals(RUNNING, row.state) + // A run after the current one, as before. + assertEquals(listOf(RUNNING, BLOCKED), work.rows.map { it.state }) + } + + @Test + fun `a held-back row is kept while a worker of the entry runs in this process`() { + // The wake row runs the entry; a new main row would only wait at the gate. + store.save(entry(state = EntryState.QUEUED)) + val row = work.add("e1", runAttemptCount = 8, lastEnqueueTime = work.now - minute) + work.add("e1#wake", state = RUNNING, runAttemptCount = 1) + running += "e1" + controller.resume() + assertEquals(ENQUEUED, row.state) + assertEquals(2, work.rows.size) + } + + @Test + fun `startableIds lists the entries with a run in progress or a row due now`() { + work.add("due") + work.add("held", runAttemptCount = 8, lastEnqueueTime = work.now - minute) + work.add("run", state = RUNNING, runAttemptCount = 1) + work.add("later#wake", initialDelay = minute) + work.add("blocked", state = BLOCKED) + work.add("gone", state = CANCELLED) + assertEquals(setOf("due", "run"), scheduler.startableIds()) + } + + @Test + fun `restartBy is the latest time WorkManager starts a stopped row again`() { + // The worker's runAttemptCount counts the starts before this one. + val startedAt = work.now + assertEquals(startedAt + 30_000, WorkManagerBackoff.restartBy(startedAt, 0)) + assertEquals(startedAt + 8 * minute, WorkManagerBackoff.restartBy(startedAt, 4)) + assertEquals(startedAt + 64 * minute, WorkManagerBackoff.restartBy(startedAt, 7)) + assertEquals(startedAt + 5 * 60 * minute, WorkManagerBackoff.restartBy(startedAt, 40)) + // The same rule as the fake, one start later. + val row = work.add("e1", runAttemptCount = 8, lastEnqueueTime = startedAt) + assertEquals(work.nextScheduleAt(row), WorkManagerBackoff.restartBy(startedAt, 7)) + assertTrue(WorkManagerBackoff.restartBy(startedAt, 4) < startedAt + 10 * minute) + } +} diff --git a/android/src/test/java/ai/openspace/backgroundupload/QueueControllerTest.kt b/android/src/test/java/ai/openspace/backgroundupload/QueueControllerTest.kt index 52404144..702e398a 100644 --- a/android/src/test/java/ai/openspace/backgroundupload/QueueControllerTest.kt +++ b/android/src/test/java/ai/openspace/backgroundupload/QueueControllerTest.kt @@ -1143,8 +1143,9 @@ class QueueControllerTest { store.save(entry(id = "p", state = EntryState.AWAITING_AUTH, expiresAt = now + 50)) controller.sweep() assertEquals(EntryState.QUEUED, store.load("r")!!.state) - assertEquals(setOf("r", "q"), scheduler.scheduled.toSet()) - assertEquals(setOf("w" to now + 3_600_000, "p" to now + 50), scheduler.wakes.toSet()) + // "w" no longer waits for its backoff: the sweep runs it now. + assertEquals(setOf("r", "q", "w"), scheduler.scheduled.toSet()) + assertEquals(setOf("p" to now + 50), scheduler.wakes.toSet()) } @Test @@ -1154,11 +1155,129 @@ class QueueControllerTest { store.save(entry(id = "far", state = EntryState.AWAITING_AUTH, expiresAt = FAR_FUTURE)) store.save(entry(id = "near", state = EntryState.AWAITING_AUTH, expiresAt = now + EXPIRY_WAKE_HORIZON_MS)) controller.sweep() - assertEquals(setOf("q", "far"), scheduler.cancelledWakes.toSet()) - assertEquals( - setOf("w" to now + 3_600_000, "near" to now + EXPIRY_WAKE_HORIZON_MS), - scheduler.wakes.toSet(), - ) + // The backoff wake of "w" is stale too: the sweep cleared its wait. + assertEquals(setOf("q", "w", "far"), scheduler.cancelledWakes.toSet()) + assertEquals(setOf("near" to now + EXPIRY_WAKE_HORIZON_MS), scheduler.wakes.toSet()) + } + + @Test + fun `the boot sweep clears a long backoff and runs the entry now`() { + store.save(entry(id = "w", state = EntryState.QUEUED, nextAttemptAt = now + 7_200_000).copy(backoffStreak = 13)) + store.save(entry(id = "s", state = EntryState.QUEUED, nextAttemptAt = now + 4_000).copy(backoffStreak = 3)) + store.save(entry(id = "r", state = EntryState.RUNNING, nextAttemptAt = now + 7_200_000).copy(backoffStreak = 9)) + controller.sweep() + for (id in listOf("w", "s", "r")) { + val e = store.load(id)!! + assertEquals(id, EntryState.QUEUED, e.state) + assertNull(id, e.nextAttemptAt) + } + // At most 5: the next failure waits about 32 s, not 2 hours. + assertEquals(5, store.load("w")!!.backoffStreak) + assertEquals(3, store.load("s")!!.backoffStreak) + assertEquals(5, store.load("r")!!.backoffStreak) + assertEquals(setOf("w", "s", "r"), scheduler.scheduled.toSet()) + assertEquals(emptyList>(), scheduler.wakes) + assertFalse(events.rows.any { it.toMap().containsKey("nextAttemptAt") }) + } + + @Test + fun `the boot sweep keeps the wait of an entry whose worker runs in this process`() { + store.save(entry(id = "w", state = EntryState.QUEUED, nextAttemptAt = now + 20_000).copy(backoffStreak = 4)) + running += "w" + controller.sweep() + assertEquals(now + 20_000, store.load("w")!!.nextAttemptAt) + assertEquals(4, store.load("w")!!.backoffStreak) + } + + // MARK: - retryNow + + @Test + fun `retryNow clears the wait of queued entries, keeps at most 5 steps, and runs them now`() { + store.save(entry(id = "w", state = EntryState.QUEUED, nextAttemptAt = now + 7_200_000).copy(backoffStreak = 13)) + store.save(entry(id = "s", state = EntryState.QUEUED, nextAttemptAt = now + 60_000).copy(backoffStreak = 2)) + controller.retryNow() + assertNull(store.load("w")!!.nextAttemptAt) + assertEquals(5, store.load("w")!!.backoffStreak) + assertNull(store.load("s")!!.nextAttemptAt) + assertEquals(2, store.load("s")!!.backoffStreak) + assertEquals(setOf("w", "s"), scheduler.scheduled.toSet()) + assertEquals(setOf("w", "s"), scheduler.cancelledWakes.toSet()) + assertEquals(emptyList>(), scheduler.wakes) + assertEquals(listOf("state:w:queued", "state:s:queued").sorted(), events.log.sorted()) + assertFalse(events.rows.any { it.toMap().containsKey("nextAttemptAt") }) + } + + @Test + fun `retryNow leaves running, paused, parked and settled entries alone`() { + // A running entry waits out a short backoff in its own worker. + store.save(entry(id = "r", state = EntryState.RUNNING, nextAttemptAt = now + 20_000).copy(backoffStreak = 4)) + store.save(entry(id = "p", state = EntryState.PAUSED).copy(backoffStreak = 9)) + store.save(entry(id = "a", state = EntryState.AWAITING_AUTH, parkedGeneration = 0, nextAttemptAt = now + 60_000)) + store.save(entry(id = "x", state = EntryState.ERROR, settledEventId = "gone")) + scheduler.startable += setOf("r", "p", "a", "x") + controller.retryNow() + assertEquals(now + 20_000, store.load("r")!!.nextAttemptAt) + assertEquals(4, store.load("r")!!.backoffStreak) + assertEquals(9, store.load("p")!!.backoffStreak) + assertEquals(now + 60_000, store.load("a")!!.nextAttemptAt) + assertEquals(emptyList(), scheduler.scheduled) + assertEquals(emptyList(), events.log) + } + + @Test + fun `retryNow with keys clears only the waits of those keys`() { + store.save(entry(id = "c", key = "capture", state = EntryState.QUEUED, nextAttemptAt = now + 60_000)) + store.save(entry(id = "n", key = "note", state = EntryState.QUEUED, nextAttemptAt = now + 60_000)) + controller.retryNow(listOf("capture")) + assertNull(store.load("c")!!.nextAttemptAt) + assertEquals(now + 60_000, store.load("n")!!.nextAttemptAt) + assertEquals(listOf("c"), scheduler.scheduled) + controller.retryNow(emptyList()) + assertEquals(now + 60_000, store.load("n")!!.nextAttemptAt) + } + + @Test + fun `retryNow schedules a queued entry with no wait only when no row of it can start now`() { + store.save(entry(id = "has", state = EntryState.QUEUED)) + store.save(entry(id = "none", state = EntryState.QUEUED)) + scheduler.startable += "has" + controller.retryNow() + assertEquals(listOf("none"), scheduler.scheduled) + assertEquals(1, scheduler.startableQueries) // one WorkManager query for the whole queue + assertEquals(emptyList(), events.log) // nothing changed on the rows + } + + @Test + fun `retryNow saves the other entries when one can not be saved, and does not throw`() { + store.save(entry(id = "q", state = EntryState.QUEUED, nextAttemptAt = now + 60_000)) + store.save(entry(id = "x", state = EntryState.QUEUED, nextAttemptAt = now + 60_000)) + withEntryUnwritable("x") { controller.retryNow() } + assertNull(store.load("q")!!.nextAttemptAt) + assertEquals(now + 60_000, store.load("x")!!.nextAttemptAt) + assertTrue("q" in scheduler.scheduled) + } + + @Test + fun `resume clears the backoff of queued entries and resets the streak of the ones it resumes`() { + store.save(entry(id = "p", state = EntryState.PAUSED).copy(backoffStreak = 9)) + store.save(entry(id = "w", state = EntryState.QUEUED, nextAttemptAt = now + 7_200_000).copy(backoffStreak = 13)) + controller.resume() + assertEquals(EntryState.QUEUED, store.load("p")!!.state) + assertEquals(0, store.load("p")!!.backoffStreak) + assertNull(store.load("w")!!.nextAttemptAt) + assertEquals(5, store.load("w")!!.backoffStreak) + assertTrue(scheduler.scheduled.containsAll(listOf("p", "w"))) + assertEquals(emptyList>(), scheduler.wakes) + } + + @Test + fun `resume of some keys clears only the waits of those keys`() { + store.save(entry(id = "c", key = "capture", state = EntryState.QUEUED, nextAttemptAt = now + 60_000)) + store.save(entry(id = "n", key = "note", state = EntryState.QUEUED, nextAttemptAt = now + 60_000)) + controller.resume(listOf("capture")) + assertNull(store.load("c")!!.nextAttemptAt) + assertEquals(now + 60_000, store.load("n")!!.nextAttemptAt) + assertEquals(listOf("n" to now + 60_000), scheduler.wakes) } @Test diff --git a/android/src/test/java/ai/openspace/backgroundupload/TestSupport.kt b/android/src/test/java/ai/openspace/backgroundupload/TestSupport.kt index db56629e..f3cea70f 100644 --- a/android/src/test/java/ai/openspace/backgroundupload/TestSupport.kt +++ b/android/src/test/java/ai/openspace/backgroundupload/TestSupport.kt @@ -1,5 +1,8 @@ package ai.openspace.backgroundupload +import androidx.work.ExistingWorkPolicy +import androidx.work.WorkInfo + // Builders and fakes shared by the JVM tests. No android.* here. internal const val FAR_FUTURE = 4_000_000_000_000L @@ -119,6 +122,10 @@ internal class FakeScheduler : WorkScheduler { val cancelledWakes = mutableListOf() /** What libraryRuns() returns. */ val runs = mutableListOf() + /** What startableIds() returns. */ + val startable = mutableSetOf() + /** How many startableIds() calls were made. */ + var startableQueries = 0 override fun schedule(entry: QueueEntry) { scheduled += entry.id @@ -142,4 +149,101 @@ internal class FakeScheduler : WorkScheduler { } override fun libraryRuns(): List = runs.toList() + + override fun startableIds(): Set { + startableQueries++ + return startable.toSet() + } +} + +/** + * WorkManager 2.12.0's rules for unique work, as [RowScheduler] uses them, + * with a manual clock. Read from the 2.12.0 classes: + * + * - A start sets RUNNING and adds 1 to runAttemptCount (WorkerWrapper.trySetRunning). + * - A system stop sets the row back to ENQUEUED. The count and the last + * enqueue time stay (WorkerWrapper.resetWorkerStatus). + * - An ENQUEUED row with runAttemptCount above 0 is backed off. It starts + * at lastEnqueueTime + min(30 s * 2^(count - 1), 5 h) and not before + * (WorkSpec.isBackedOff, WorkSpec.Companion.calculateNextRunTime, and the + * due check in WorkerWrapper.runWorker). Any other ENQUEUED row starts at + * lastEnqueueTime + initialDelay. + * - When a row finishes, the rows blocked behind it become ENQUEUED with a + * new lastEnqueueTime. + */ +internal class FakeWorkManager(var now: Long = 10_000L) : WorkRows { + class Row( + val serial: Int, + val name: String, + val entryId: String, + var state: WorkInfo.State, + var runAttemptCount: Int, + var lastEnqueueTime: Long, + val initialDelay: Long, + ) + + val rows = mutableListOf() + private var serial = 0 + + fun nextScheduleAt(r: Row): Long = when { + r.state != WorkInfo.State.ENQUEUED -> Long.MAX_VALUE + r.runAttemptCount > 0 -> { + val backoff = WorkManagerBackoff.DELAY_MS.toDouble() * Math.pow(2.0, (r.runAttemptCount - 1).toDouble()) + r.lastEnqueueTime + minOf(backoff, WorkManagerBackoff.MAX_MS.toDouble()).toLong() + } + else -> r.lastEnqueueTime + r.initialDelay + } + + /** Adds a row as it stands, for a test's starting point. */ + fun add( + name: String, + state: WorkInfo.State = WorkInfo.State.ENQUEUED, + runAttemptCount: Int = 0, + lastEnqueueTime: Long = now, + initialDelay: Long = 0, + ): Row = Row(serial++, name, name.substringBefore("#"), state, runAttemptCount, lastEnqueueTime, initialDelay) + .also { rows += it } + + fun open(name: String) = rows.filter { it.name == name && !it.state.isFinished } + + /** The rows WorkManager may start now. */ + fun startableNow(): List = + rows.filter { it.state == WorkInfo.State.ENQUEUED && nextScheduleAt(it) <= now } + + override fun unfinished(name: String): List = open(name).map(::view) + + override fun library(): List = rows.filter { !it.state.isFinished }.map(::view) + + override fun enqueue(name: String, policy: ExistingWorkPolicy, entry: QueueEntry, delayMs: Long) { + val open = open(name) + val state = when (policy) { + ExistingWorkPolicy.KEEP -> if (open.isNotEmpty()) return else WorkInfo.State.ENQUEUED + ExistingWorkPolicy.REPLACE -> { + open.forEach { it.state = WorkInfo.State.CANCELLED } + WorkInfo.State.ENQUEUED + } + // Behind an unfinished row a new one waits; with none it starts a fresh chain. + ExistingWorkPolicy.APPEND_OR_REPLACE -> + if (open.isNotEmpty()) WorkInfo.State.BLOCKED else WorkInfo.State.ENQUEUED + else -> error("unused policy $policy") + } + rows += Row(serial++, name, entry.id, state, 0, now, delayMs) + } + + override fun cancel(name: String) { + open(name).forEach { it.state = WorkInfo.State.CANCELLED } + } + + fun start(row: Row) { + check(row in startableNow()) { "row ${row.serial} is not due" } + row.state = WorkInfo.State.RUNNING + row.runAttemptCount++ + } + + fun stop(row: Row) { + check(row.state == WorkInfo.State.RUNNING) + row.state = WorkInfo.State.ENQUEUED + } + + private fun view(r: Row) = WorkRow(r.entryId, r.state, r.runAttemptCount, nextScheduleAt(r), r.initialDelay > 0) } From aff4331107e47eb62d52353b7fa9760e0fbabc17 Mon Sep 17 00:00:00 2001 From: Dylan Murphy Date: Tue, 17 Nov 2026 18:01:16 -0500 Subject: [PATCH 02/10] Docs: Android clears a backoff wait when the app acts CHANGELOG (Unreleased): the behavior change for the boot sweep, resume() and a return to the foreground; the replaced WorkManager row; the 15-minute cap for a transport failure; the chunked streak base; and the fresh row after a time-limit stop that is now added only when WorkManager holds the stopped row back. README: "Backoff and the app" under the Android notes, resume() on Android, and the transport cap in the retry table. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 43 ++++++++++++++++++++++++++++++++++++------- README.md | 16 +++++++++++++++- 2 files changed, 51 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 604c09a9..c9b2b0f9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,13 +8,16 @@ transmit slot and sent nothing. Now a run that only waited goes back to takes one backoff step. The foreground-service time limit (Android 15 and later) now counts as a time-limit stop. -After every time-limit stop, the library schedules a fresh WorkManager row -for the entry: at once after a run that only waited, or when its backoff -ends, also for a short backoff. WorkManager runs the stopped row again too, -but only after its own backoff, which doubles with each start of that row. -By the WorkManager 2.12.0 code that wait is 64 minutes at the 8th start and -128 minutes at the 9th, so it passes an hour after about 9 starts. A fresh -row does not have that wait. Stops for other reasons go back to `queued` +After a time-limit stop, the library schedules a fresh WorkManager row for +the entry: at once after a run that only waited, or when its backoff ends, +also for a short backoff. WorkManager runs the stopped row again too, but +only after its own backoff, which doubles with each start of that row and +counts from the row's last enqueue time. By the WorkManager 2.12.0 code that +wait is 64 minutes at the 8th start and 128 minutes at the 9th, so it passes +an hour after about 9 starts. A fresh row does not have that wait. When that +wait is already over at the stop (for a 10-minute run, up to the 5th start), +the stopped row runs again at once and the library adds no fresh row, so +the entry does not get two rows. Stops for other reasons go back to `queued` with no new row, as before: the stopped row runs again when the system allows it and after WorkManager's own backoff. @@ -46,6 +49,32 @@ counts no attempt. So `attempts`, and the `attempt` field of later attempt events, are lower for uploads that only waited. The number of attempt events does not change. That is the correct count. +**Behavior change:** on Android, the boot sweep, `resume()` and a return to +the foreground now clear a backoff wait and start queued uploads at once. +Before, an entry on a long backoff (up to 2 hours) kept its wait when the +app opened, and `resume()` kept it too. `resume({ keys })` clears the waits +of those keys only. The entry keeps at most 5 backoff steps, so if the +server still fails, the next wait is about 32 s, then it doubles. A paused +entry that `resume()` moves back to `queued` starts its backoff over. The +app needs no code for this. + +Android: the boot sweep, `resume()`, a return to the foreground, +`updateHeaders()` and a same-id `mutate()` now replace a WorkManager row +that WorkManager holds back after a system stop, so the entry can start at +once. Before, the library added no row while that stopped row waited, and +the entry waited for WorkManager's own backoff (up to 5 hours). The library +replaces the row only when no run of the entry is in progress. + +Android: a transport failure (no HTTP response) now waits at most 15 +minutes. Every attempt starts on a network that Android reports as +validated. HTTP failures keep the retry policy's `maxMs` (2 hours by +default). + +Android: a chunked entry uses its stored backoff only until a part is +accepted in the run. After that each part counts its own failures from 0. +Before, one failed part after a long backoff sent the whole entry back for +the next, longer wait. + ## 11.3.0 Added `useRequests(filter?)` and `useRequestsWithProgress(filter?)`, React diff --git a/README.md b/README.md index 525c964f..a08767fa 100644 --- a/README.md +++ b/README.md @@ -220,6 +220,16 @@ at the next run, after a growing backoff. A body that needs more than 10 minutes headless cannot finish that way. Use `parts` for large bodies: accepted parts are kept across runs. iOS has no equal limit. +**Backoff and the app.** When the app starts, comes to the foreground, or +calls `resume()`, the library ends the retry backoff of every queued entry +(for `resume({ keys })`, of the entries of those keys) and runs it at once. +The app needs no code for this. The entry keeps at most 5 backoff steps, so +if the server still fails, the next wait is about 32 s and then doubles. The +library also replaces a WorkManager row that WorkManager holds back after a +system stop, so the entry can start at once. A chunked entry uses its stored +backoff only until a part is accepted in the run; after that each part +counts its own failures from zero. + **Backups.** The queue store (`files/rnbgupload-chunked/`) and the journal (`files/rnbgupload-settled/`) hold request headers, including auth tokens, and staged bodies. Set `android:allowBackup="false"` in the host app, or @@ -343,7 +353,7 @@ Retry classes: | Attempt outcome | Action | | --- | --- | | 2xx, or an `accept` rule matches | settle `completed` | -| network failure, 5xx, 408, 429 | exponential backoff (base 1 s, max 2 h, jitter 0.2) until `expiresAt` | +| network failure, 5xx, 408, 429 | exponential backoff (base 1 s, max 2 h, jitter 0.2) until `expiresAt`. On Android a network failure (no HTTP response) waits at most 15 minutes | | 401, 403 | park as `awaiting-auth`; resume on `updateHeaders()` | | other 4xx matching a `retry.transient` rule (`status`, optional `bodyIncludes`), under the rule's `maxRetries` | as transient. The first matching rule in the list wins. `maxRetries: 3` retries three matching responses; the fourth settles. No `maxRetries` means no cap. See "Transient rule counts" below. A rule for 401, 403, 408 or 429 is rejected at the call | | other 4xx in `retry.terminalHttp.exempt` (default `[404]`), when no rule matched | as transient, with no cap | @@ -474,6 +484,10 @@ event (`paused`, then `queued`). No outcome is produced and the bytes are kept. A paused entry past `expiresAt` settles `error` with `errorKind: 'expired'` when it is resumed. +On Android, `resume()` also ends the retry backoff of the queued entries in +its scope, and they run at once. A resumed entry starts its backoff over. +See "Backoff and the app" under Android platform notes. + Neither call rejects because the settings cannot be saved. On Android, either call can still reject `E_STORAGE` when an entry file cannot be saved. See [Settings on a full disk](#settings-on-a-full-disk). From a2424936fe8a224dd65f0b4b39f51b7abb983657 Mon Sep 17 00:00:00 2001 From: Dylan Murphy Date: Tue, 17 Nov 2026 18:37:53 -0500 Subject: [PATCH 03/10] Android clear-wait: clear only waits with no server response, and fix the review findings From the review of this branch: - A wait is cleared (boot sweep, resume(), foreground) only when its failure had no server response, or an older version saved the entry, and 30 s or more after that failure. A wait after an HTTP response keeps its time, so foregrounds cannot spend a transient rule's retries or send back to back. New persisted fields: waitCause, failedAt. resume() keeps at most 5 steps instead of 0. - scheduleRun makes the main row before it cancels the stale wake. - retryNow goes on past an entry whose save or scheduling fails, emits what it saved, and writes one error. - A held-back main row is replaced also while a wake worker runs; only a BLOCKED row or one that starts now counts as the row after a run. - The sweep schedules an entry as found when only its clear fails to save. - The start margin is 30 s. - Only connect-phase failures (ConnectException, UnknownHostException, NoRouteToHostException, PortUnreachableException, SSLHandshakeException, SSLPeerUnverifiedException) cap the wait at 15 minutes. - A chunked part that set the stored streak goes on from it, and only its own acceptance resets it (new persisted field backoffPartStart). - The foreground retryNow runs at most once per 30 s, and the library query filters states in WorkManager. Co-Authored-By: Claude Opus 5.5 --- .../backgroundupload/ChunkedUploadWorker.kt | 22 ++- .../backgroundupload/EnqueueRules.kt | 1 + .../ai/openspace/backgroundupload/EntryRun.kt | 41 +++-- .../backgroundupload/EntryTransitions.kt | 100 ++++++++--- .../backgroundupload/ForegroundTask.kt | 23 ++- .../backgroundupload/QueueController.kt | 87 ++++++---- .../openspace/backgroundupload/QueueEntry.kt | 23 +++ .../backgroundupload/RetryClassifier.kt | 37 +++- .../openspace/backgroundupload/Scheduler.kt | 57 ++++--- .../backgroundupload/UploadWorker.kt | 2 +- .../openspace/backgroundupload/WorkerOps.kt | 38 ++++- .../backgroundupload/EntryRunTest.kt | 159 +++++++++++++++++- .../backgroundupload/EntryTransitionsTest.kt | 40 ++++- .../backgroundupload/ForegroundTaskTest.kt | 21 ++- .../backgroundupload/MainChainTest.kt | 109 ++++++++++-- .../backgroundupload/QueueControllerTest.kt | 122 ++++++++++++-- .../backgroundupload/RetryClassifierTest.kt | 22 +++ .../openspace/backgroundupload/TestSupport.kt | 16 +- 18 files changed, 766 insertions(+), 154 deletions(-) diff --git a/android/src/main/java/ai/openspace/backgroundupload/ChunkedUploadWorker.kt b/android/src/main/java/ai/openspace/backgroundupload/ChunkedUploadWorker.kt index b1880834..dd87cd44 100644 --- a/android/src/main/java/ai/openspace/backgroundupload/ChunkedUploadWorker.kt +++ b/android/src/main/java/ai/openspace/backgroundupload/ChunkedUploadWorker.kt @@ -50,7 +50,7 @@ internal class ChunkedTransfer(private val run: EntryRun) { run.progressStarted(total, acceptedBytes.get()) try { - ChunkedEngine.run(pending) { index -> executePart(index, blob, start.backoffStreak) } + ChunkedEngine.run(pending) { index -> executePart(index, blob, start.backoffStreak, start.backoffPartStart) } } catch (failed: PartFailed) { return failed.settlement } @@ -69,13 +69,17 @@ internal class ChunkedTransfer(private val run: EntryRun) { } /** - * [initialStreak] is the entry's stored streak. A part uses it only until - * a part is accepted in this run. After that the server answers, so the - * part counts its own failures from 0. Without that, one failed part after - * a long backoff would release the whole entry for the next, longer wait. + * [initialStreak] is the entry's stored streak, and [streakPart] the start + * of the part whose failures set it. That part goes on from the stored + * streak, so a part that keeps failing keeps backing off across runs. The + * other parts count their own failures from 0, so an old streak does not + * slow the parts that work. When no part is named (an older version, or a + * stop that cut the run), every part uses the stored streak until a part + * is accepted in this run. */ - internal suspend fun executePart(index: Int, blob: File, initialStreak: Int) { - var base = initialStreak + internal suspend fun executePart(index: Int, blob: File, initialStreak: Int, streakPart: Long?) { + val partStart = run.ops.latest(run.entryId, run.generation).descriptor!!.parts!![index].start + var base = if (streakPart == null || streakPart == partStart) initialStreak else 0 var failures = 0 while (true) { if (index in acceptedHere) return @@ -116,9 +120,9 @@ internal class ChunkedTransfer(private val run: EntryRun) { } EntryRun.AttemptResult.Transient -> { onPartProgress(index, 0L) - if (acceptedHere.isNotEmpty()) base = 0 + if (streakPart == null && acceptedHere.isNotEmpty()) base = 0 failures++ - run.backoffOrRelease(a.policy, base + failures, a.entry.expiresAt) + run.backoffOrRelease(a.policy, base + failures, a.entry.expiresAt, a.cause, partStart) } is EntryRun.AttemptResult.Terminal -> { onPartProgress(index, 0L) diff --git a/android/src/main/java/ai/openspace/backgroundupload/EnqueueRules.kt b/android/src/main/java/ai/openspace/backgroundupload/EnqueueRules.kt index 7f761691..7e46604a 100644 --- a/android/src/main/java/ai/openspace/backgroundupload/EnqueueRules.kt +++ b/android/src/main/java/ai/openspace/backgroundupload/EnqueueRules.kt @@ -159,6 +159,7 @@ object EnqueueRules { updatedAt = now, nextAttemptAt = null, backoffStreak = 0, + backoffPartStart = null, headerGeneration = headerGeneration, parkedGeneration = null, generation = if (reopen) existing.generation + 1 else existing.generation, diff --git a/android/src/main/java/ai/openspace/backgroundupload/EntryRun.kt b/android/src/main/java/ai/openspace/backgroundupload/EntryRun.kt index 3d721a89..3d3afdca 100644 --- a/android/src/main/java/ai/openspace/backgroundupload/EntryRun.kt +++ b/android/src/main/java/ai/openspace/backgroundupload/EntryRun.kt @@ -28,7 +28,14 @@ internal class EntryRun( class ParkException(val headerGeneration: Int) : Exception("awaiting auth") /** A backoff too long to wait here. */ - class BackoffException(val nextAttemptAt: Long, val streak: Int) : Exception("released for backoff") + class BackoffException( + val nextAttemptAt: Long, + val streak: Int, + /** [WaitCause] of the failure. */ + val waitCause: String, + /** The chunked part whose failure it was; null for a simple entry. */ + val partStart: Long?, + ) : Exception("released for backoff") class ExpiredException : Exception("expired before completion") @@ -61,6 +68,8 @@ internal class EntryRun( val url: String, val policy: RetryClassifier.Policy, val result: AttemptResult, + /** [WaitCause]: whether the server answered. */ + val cause: String, ) { val method: String get() = entry.descriptor!!.method } @@ -132,7 +141,7 @@ internal class EntryRun( if (ops.park(entryId, generation, park.headerGeneration) != WorkerOps.ParkResult.REISSUE) return } catch (backoff: BackoffException) { endProgress(completed = false) - ops.release(entryId, generation, backoff.nextAttemptAt, backoff.streak) + ops.release(entryId, generation, backoff.nextAttemptAt, backoff.streak, cause = backoff.waitCause, partStart = backoff.partStart) return } catch (error: ExpiredException) { endProgress(completed = false) @@ -244,7 +253,11 @@ internal class EntryRun( val streak = e!!.backoffStreak + 1 val now = host.now() val backoff = RetryClassifier.backoffMs(policy(e), streak) - ops.release(entryId, generation, RetryClassifier.nextAttemptAt(now, backoff, e.expiresAt), streak, wake) + // A cut request has no response. Which part it cut is not known. + ops.release( + entryId, generation, RetryClassifier.nextAttemptAt(now, backoff, e.expiresAt), streak, wake, + cause = WaitCause.NO_RESPONSE, partStart = null, + ) }.onFailure { if (it is WorkerOps.WakeNotScheduledException) { Diag.error("released '$entryId' after a timeout stop, but could not schedule its wake", it.cause) @@ -359,9 +372,11 @@ internal class EntryRun( val result = if (verdict is RetryClassifier.Verdict.Terminal) { AttemptResult.Terminal(verdict.errorKind, verdict.message, null) } else AttemptResult.Transient - // The attempt started on a validated network (waitForNetwork), so a - // transport failure waits at most 15 minutes. - return Attempt(entry, url, RetryClassifier.transportFailurePolicy(policy), result) + // The attempt started on a validated network (waitForNetwork). A + // failure that proves the request never reached the server waits at + // most 15 minutes; any other keeps the entry's maximum. + val waitPolicy = if (RetryClassifier.reachedNoServer(error)) RetryClassifier.transportFailurePolicy(policy) else policy + return Attempt(entry, url, waitPolicy, result, WaitCause.NO_RESPONSE) } // A part counts on its own, so each part gets a rule's full maxRetries. @@ -388,23 +403,29 @@ internal class EntryRun( is RetryClassifier.Verdict.Transient -> AttemptResult.Transient is RetryClassifier.Verdict.Terminal -> AttemptResult.Terminal("http", verdict.message, response) } - return Attempt(entry, url, policy, result) + return Attempt(entry, url, policy, result, WaitCause.RESPONSE) } /** * A short backoff waits here, with the row still running and showing * nextAttemptAt; a long one throws [BackoffException] to release the run. */ - suspend fun backoffOrRelease(policy: RetryClassifier.Policy, streak: Int, expiresAt: Long) { + suspend fun backoffOrRelease( + policy: RetryClassifier.Policy, + streak: Int, + expiresAt: Long, + cause: String, + partStart: Long? = null, + ) { val backoff = RetryClassifier.backoffMs(policy, streak) val now = host.now() if (backoff <= RetryClassifier.IN_WORKER_BACKOFF_MAX_MS) { val wait = min(backoff, max(0L, expiresAt - now)) - ops.backingOff(entryId, generation, now + wait, streak) + ops.backingOff(entryId, generation, now + wait, streak, cause, partStart) host.sleep(wait) return } - throw BackoffException(RetryClassifier.nextAttemptAt(now, backoff, expiresAt), streak) + throw BackoffException(RetryClassifier.nextAttemptAt(now, backoff, expiresAt), streak, cause, partStart) } fun progressStarted(total: Long, sent: Long) { diff --git a/android/src/main/java/ai/openspace/backgroundupload/EntryTransitions.kt b/android/src/main/java/ai/openspace/backgroundupload/EntryTransitions.kt index 63556d08..11550467 100644 --- a/android/src/main/java/ai/openspace/backgroundupload/EntryTransitions.kt +++ b/android/src/main/java/ai/openspace/backgroundupload/EntryTransitions.kt @@ -24,15 +24,29 @@ object EntryTransitions { /** * A short backoff that the worker waits out itself. The row stays running - * and shows when the next attempt is due. The failure's step is saved now, - * not at the next release. Chunked parts keep their own streaks, so the - * entry keeps the highest; an accepted part resets it to 0. + * and shows when the next attempt is due. The failure's step, its [cause] + * and its time are saved now, not at the next release. Chunked parts keep + * their own streaks, so the entry keeps the highest and the part + * ([partStart]) that holds it; only that part's acceptance resets it. */ - fun toBackingOff(e: QueueEntry, nextAttemptAt: Long, streak: Int, now: Long) = e.copy( - nextAttemptAt = nextAttemptAt, - backoffStreak = max(e.backoffStreak, streak), - updatedAt = now, - ) + fun toBackingOff( + e: QueueEntry, + nextAttemptAt: Long, + streak: Int, + now: Long, + cause: String? = null, + partStart: Long? = null, + ): QueueEntry { + val higher = streak >= e.backoffStreak + return e.copy( + nextAttemptAt = nextAttemptAt, + backoffStreak = max(e.backoffStreak, streak), + backoffPartStart = if (higher) partStart else e.backoffPartStart, + waitCause = cause, + failedAt = now, + updatedAt = now, + ) + } /** One attempt starts: attempts + 1, its X-Request-Id, and no pending backoff. */ fun toAttempt(e: QueueEntry, requestId: String, now: Long) = e.copy( @@ -42,13 +56,32 @@ object EntryTransitions { updatedAt = now, ) - /** A backoff too long to wait inside the worker. The streak is kept so the next wait keeps growing. */ - fun toReleased(e: QueueEntry, nextAttemptAt: Long, streak: Int, now: Long) = e.copy( - state = EntryState.QUEUED, - nextAttemptAt = nextAttemptAt, - backoffStreak = streak, - updatedAt = now, - ) + /** + * A backoff too long to wait inside the worker. The streak is kept so the + * next wait keeps growing, with the failure's [cause] and time. A chunked + * part ([partStart] set) keeps a higher streak another part holds, so a + * part that keeps failing does not start over because a sibling released + * the run. + */ + fun toReleased( + e: QueueEntry, + nextAttemptAt: Long, + streak: Int, + now: Long, + cause: String? = null, + partStart: Long? = null, + ): QueueEntry { + val keepOther = partStart != null && e.backoffStreak > streak + return e.copy( + state = EntryState.QUEUED, + nextAttemptAt = nextAttemptAt, + backoffStreak = if (keepOther) e.backoffStreak else streak, + backoffPartStart = if (keepOther) e.backoffPartStart else partStart, + waitCause = cause, + failedAt = now, + updatedAt = now, + ) + } fun toSettled(e: QueueEntry, state: EntryState, eventId: String, bytesSent: Long, now: Long) = e.copy( state = state, @@ -71,24 +104,43 @@ object EntryTransitions { * between. It compares the entry's own headerGeneration, which * updateHeaders() saves with the new headers. The settings value can be * older after a settings write that never landed. Back to queued, the - * backoff starts over: the app asked for the entry to run. + * streak keeps at most [RetryClassifier.RETRY_NOW_MAX_STREAK] steps, as + * after a cleared wait: a pause and resume in normal work must not give a + * failing entry a burst of attempts. */ fun toResumed(e: QueueEntry, now: Long): QueueEntry = if (e.parkedGeneration != null && e.parkedGeneration == e.headerGeneration) { e.copy(state = EntryState.AWAITING_AUTH, updatedAt = now) } else { - e.copy(state = EntryState.QUEUED, parkedGeneration = null, backoffStreak = 0, updatedAt = now) + e.copy( + state = EntryState.QUEUED, + parkedGeneration = null, + backoffStreak = min(e.backoffStreak, RetryClassifier.RETRY_NOW_MAX_STREAK), + updatedAt = now, + ) } - /** Whether [e] is queued and waits for a backoff to end. */ - fun waitsOnBackoff(e: QueueEntry) = e.state == EntryState.QUEUED && e.nextAttemptAt != null + /** + * Whether the app may end [e]'s backoff wait now (boot sweep, resume(), + * return to the foreground). Only a queued entry whose wait followed a + * failure with no server response, or whose cause is unknown (an older + * version saved it), and only [RetryClassifier.CLEAR_MIN_AGE_MS] or more + * after that failure. A wait after a server response keeps its time: the + * server asked for it in effect, and clearing it would spend a transient + * rule's retries faster, so the request could settle as failed sooner. + */ + fun clearsWait(e: QueueEntry, now: Long): Boolean = + e.state == EntryState.QUEUED && + e.nextAttemptAt != null && + e.waitCause != WaitCause.RESPONSE && + now - (e.failedAt ?: e.updatedAt) >= RetryClassifier.CLEAR_MIN_AGE_MS /** - * The app started, resumed, or came to the foreground: a queued entry - * stops waiting for its backoff and runs now. The streak keeps at most - * [RetryClassifier.RETRY_NOW_MAX_STREAK] steps, so the entry sends once - * now, and a server that is still down is not asked again every few - * seconds. + * The app started, resumed, or came to the foreground, and [clearsWait] + * holds: the entry stops waiting for its backoff and runs now. The streak + * keeps at most [RetryClassifier.RETRY_NOW_MAX_STREAK] steps, so the entry + * sends once now, and a server that is still down is not asked again + * every few seconds. */ fun toRetryNow(e: QueueEntry, now: Long) = e.copy( nextAttemptAt = null, diff --git a/android/src/main/java/ai/openspace/backgroundupload/ForegroundTask.kt b/android/src/main/java/ai/openspace/backgroundupload/ForegroundTask.kt index d1392b4d..77de255d 100644 --- a/android/src/main/java/ai/openspace/backgroundupload/ForegroundTask.kt +++ b/android/src/main/java/ai/openspace/backgroundupload/ForegroundTask.kt @@ -6,18 +6,35 @@ import java.util.concurrent.Executor * What the module does when the app comes to the foreground, off the UI * thread, on [executor] (the queue executor, so it never overlaps another * queue call). First a settings write that failed earlier is tried again, - * then queued entries stop waiting for their backoff and run now - * ([QueueController.retryNow]). Neither step throws on the executor. + * then [retryNow] ([QueueController.retryNow]). Neither step throws on the + * executor. + * + * The host's resume event fires on every Activity resume, also after a + * permission dialog, a picker or a share sheet. So [retryNow] runs at most + * once per [MIN_INTERVAL_MS]; the settings save runs every time. */ internal class ForegroundTask( private val executor: Executor, private val saveSettings: () -> Unit, private val retryNow: () -> Unit, + private val clock: () -> Long = System::currentTimeMillis, ) { + companion object { + const val MIN_INTERVAL_MS = 30_000L + } + + // Read and written only on [executor], which runs one task at a time. + private var lastRetryAt: Long? = null + fun onForeground() { executor.execute { runCatching(saveSettings).onFailure { Diag.error("could not save the settings on return to the foreground", it) } - runCatching(retryNow).onFailure { Diag.error("retryNow on return to the foreground failed", it) } + val now = clock() + val last = lastRetryAt + if (last == null || now - last >= MIN_INTERVAL_MS) { + lastRetryAt = now + runCatching(retryNow).onFailure { Diag.error("retryNow on return to the foreground failed", it) } + } } } } diff --git a/android/src/main/java/ai/openspace/backgroundupload/QueueController.kt b/android/src/main/java/ai/openspace/backgroundupload/QueueController.kt index d5217ddb..a99921f7 100644 --- a/android/src/main/java/ai/openspace/backgroundupload/QueueController.kt +++ b/android/src/main/java/ai/openspace/backgroundupload/QueueController.kt @@ -210,8 +210,9 @@ class QueueController( * Undoes the pause of the same scope: [keys] null turns the gate off, a * list removes those keys from the set. A row comes back only when no * scope still pauses it, so the gate on + its key resumed stays paused. - * A resumed entry starts its backoff over. A queued entry of the same - * scope that waits for a backoff runs now, as with [retryNow]. + * A resumed entry keeps at most 5 backoff steps. A queued entry of the + * same scope whose wait may be cleared ([EntryTransitions.clearsWait]) + * runs now. * * A failed entry save throws E_STORAGE after every other entry is moved * and every queued entry is scheduled. The entry that failed stays paused. @@ -225,7 +226,7 @@ class QueueController( val result = transformAll { e -> when { e.state == EntryState.PAUSED && !s.isPaused(e.key) -> EntryTransitions.toResumed(e, now) - inScope(e, keys) && EntryTransitions.waitsOnBackoff(e) -> EntryTransitions.toRetryNow(e, now) + inScope(e, keys) && EntryTransitions.clearsWait(e, now) -> EntryTransitions.toRetryNow(e, now) else -> e } } @@ -236,21 +237,24 @@ class QueueController( } /** - * The app started, came to the foreground, or resumed: queued entries in - * scope run now. [keys] null is every key; an empty list changes nothing. + * The app came to the foreground: queued entries in scope that may run + * now are started. [keys] null is every key; an empty list changes nothing. * - * An entry that waits for a backoff stops waiting ([EntryTransitions.toRetryNow]) - * and is scheduled. A queued entry with no wait is scheduled only when no - * row of it can start now ([WorkScheduler.startableIds], one query): a row - * that WorkManager holds back after a stop is replaced ([WorkScheduler.schedule]). - * Running, paused, parked and settled entries are left alone. A queued - * entry sends nothing, and the per-id gate holds any new row while a - * worker of the entry still winds down, so this never starts a second - * transfer next to one in progress. + * An entry whose wait [EntryTransitions.clearsWait] allows stops waiting + * ([EntryTransitions.toRetryNow]) and is scheduled. A wait after a server + * response keeps its time. A queued entry with no wait, whatever its last + * failure, is scheduled only when no row of it can start now + * ([WorkScheduler.startableIds], one query): that is not a retry, it only + * makes sure a row exists, and a row WorkManager holds back after a stop + * is replaced ([WorkScheduler.schedule]). Running, paused, parked and + * settled entries are left alone. A queued entry sends nothing, and the + * per-id gate holds any new row while a worker of the entry still winds + * down, so this never starts a second transfer next to one in progress. * - * It reads the queue once, and saves only the entries that wait. Never - * throws: a failed entry save keeps that entry's wait and writes an - * error; the other entries go on. + * It reads every entry file once, and saves only the entries it changes. + * Never throws: an entry whose save or scheduling fails is skipped, the + * others go on, and one error goes to the diagnostics. Every saved + * change emits its state event. */ fun retryNow(keys: List? = null) { if (keys != null && keys.isEmpty()) return @@ -258,32 +262,34 @@ class QueueController( val queued = store.all().filter { !it.legacy && it.state == EntryState.QUEUED && inScope(it, keys) } val cleared = mutableListOf() var failures = 0 - var first: IOException? = null + var first: Throwable? = null + fun failed(error: Throwable) { + failures++ + if (first == null) first = error + } store.locked { - queued.filter { EntryTransitions.waitsOnBackoff(it) }.forEach { e -> + queued.filter { EntryTransitions.clearsWait(it, now) }.forEach { e -> try { var next: QueueEntry? = null // The snapshot can be stale: the store decides under the lock. store.compute(e.id) { cur -> - if (cur != null && EntryTransitions.waitsOnBackoff(cur)) EntryTransitions.toRetryNow(cur, now).also { next = it } else cur + if (cur != null && EntryTransitions.clearsWait(cur, now)) EntryTransitions.toRetryNow(cur, now).also { next = it } else cur } next?.let { cleared += it } } catch (error: IOException) { - failures++ - if (first == null) first = error + failed(error) } } } - cleared.forEach { scheduleRun(it) } + cleared.forEach { e -> runCatching { scheduleRun(e) }.onFailure(::failed) } val idle = queued.filter { it.nextAttemptAt == null } if (idle.isNotEmpty()) { - val startable = runCatching { scheduler.startableIds() } - .onFailure { Diag.error("retryNow could not read WorkManager; it schedules every queued entry", it) } - .getOrNull() - idle.filter { startable == null || it.id !in startable }.forEach { scheduleRun(it) } + val startable = runCatching { scheduler.startableIds() }.onFailure(::failed).getOrNull() + idle.filter { startable == null || it.id !in startable } + .forEach { e -> runCatching { scheduleRun(e) }.onFailure(::failed) } } cleared.forEach { events.state(it.toRow()) } - first?.let { Diag.error("retryNow could not save $failures entries; they keep their wait", it) } + first?.let { Diag.error("retryNow: $failures entry saves or WorkManager calls failed; the next app start or foreground tries again", it) } } private fun inScope(e: QueueEntry, keys: List?) = keys == null || e.key in keys @@ -487,8 +493,9 @@ class QueueController( * 2. A running entry with no worker: a process death mid-run. Queue it. * A live row that disagrees with the paused settings, the gate or its * key (a death partway through pause() or resume()): make it agree. - * A queued entry that waits for a backoff runs now, with at most 5 - * backoff steps kept ([EntryTransitions.toRetryNow]). + * A queued entry whose wait may be cleared ([EntryTransitions.clearsWait]) + * runs now, with at most 5 backoff steps kept. When only that save + * fails, the entry is scheduled as it was found. * 3. A settled entry with records of its generation other than its own: * orphans from a cancel race. Ack them. * 4. A completed or cancelled entry whose own record is gone: the ack @@ -549,12 +556,21 @@ class QueueController( } else if (!paused && cur.state == EntryState.PAUSED) { cur = EntryTransitions.toResumed(cur, now) } - // The app started: a queued entry no longer waits for its backoff. - if (EntryTransitions.waitsOnBackoff(cur)) { + // The app started: a queued entry whose wait may be cleared runs now. + val repaired = cur + if (EntryTransitions.clearsWait(cur, now)) { cur = EntryTransitions.toRetryNow(cur, now) } if (cur !== e) { - if (trySave(cur)) changed += cur else continue + cur = when { + trySave(cur) -> cur + // Only the clear failed to save: go on as found. The wake + // path (KEEP) below makes a lost wake again. + repaired === e -> e + cur !== repaired && trySave(repaired) -> repaired + else -> continue + } + if (cur !== e) changed += cur } if (!paused || cur.state == EntryState.AWAITING_AUTH) { toSchedule += cur @@ -615,8 +631,11 @@ class QueueController( if (at != null && at - now > RetryClassifier.IN_WORKER_BACKOFF_MAX_MS) { scheduler.scheduleWake(entry, at, replace = !keepWake) } else { - cancelStaleWake(entry.id) + // The main row first: these are two separate WorkManager calls, + // and a process death between them must not leave the entry + // with neither a wake nor a main row. scheduler.schedule(entry) + cancelStaleWake(entry.id) } } EntryState.AWAITING_AUTH -> @@ -630,6 +649,8 @@ class QueueController( } // A running wake is the entry's worker; it finishes its own run. + // [isWorkerRunning] reads the in-process gate, so it assumes WorkManager + // runs workers in the app's process. The library sets no android:process. private fun cancelStaleWake(id: String) { if (!isWorkerRunning(id)) scheduler.cancelWake(id) } diff --git a/android/src/main/java/ai/openspace/backgroundupload/QueueEntry.kt b/android/src/main/java/ai/openspace/backgroundupload/QueueEntry.kt index ea0fbc9e..e9003bb4 100644 --- a/android/src/main/java/ai/openspace/backgroundupload/QueueEntry.kt +++ b/android/src/main/java/ai/openspace/backgroundupload/QueueEntry.kt @@ -88,6 +88,15 @@ data class StagedBody( } } +/** The values of [QueueEntry.waitCause]. Strings, because they are persisted. */ +object WaitCause { + /** The server answered (a 5xx, 408, 429, or a status a rule retries). */ + const val RESPONSE = "response" + + /** No response: a transport failure, or a stop that cut the request. */ + const val NO_RESPONSE = "no-response" +} + /** One queue entry. [QueueStore] persists it as `entry.json` with Gson. */ data class QueueEntry( val id: String, @@ -129,6 +138,20 @@ data class QueueEntry( * name `transientHits`; Gson skips that unknown name, so it reads as no hits. */ val transientHitsByRule: Map? = null, + /** + * What made the last failure wait for [nextAttemptAt]: [WaitCause.RESPONSE] + * (the server answered) or [WaitCause.NO_RESPONSE]. Null when an older + * version saved the entry: unknown. + */ + val waitCause: String? = null, + /** When that failure happened. Null when an older version saved the entry. */ + val failedAt: Long? = null, + /** + * A chunked entry: the start of the part whose failures set + * [backoffStreak]. Null when unknown (an older version, or a stop that cut + * the run); then every part uses the streak until a part is accepted. + */ + val backoffPartStart: Long? = null, ) { val isLive get() = state.isLive val isSettled get() = !state.isLive diff --git a/android/src/main/java/ai/openspace/backgroundupload/RetryClassifier.kt b/android/src/main/java/ai/openspace/backgroundupload/RetryClassifier.kt index 19491aeb..2ce46170 100644 --- a/android/src/main/java/ai/openspace/backgroundupload/RetryClassifier.kt +++ b/android/src/main/java/ai/openspace/backgroundupload/RetryClassifier.kt @@ -41,17 +41,42 @@ object RetryClassifier { const val IN_WORKER_BACKOFF_MAX_MS = 30_000L /** - * The longest wait after a transport failure (no HTTP response). Every - * attempt starts on a network Android reports as validated, so the failure - * says little about the server, and a network change cannot wake a queued - * entry. HTTP failures keep the policy's maxMs. + * The longest wait after a failure in the connect phase + * ([reachedNoServer]). Every attempt starts on a network Android reports + * as validated, so such a failure says little about the server, and a + * network change cannot wake a queued entry. Every other failure keeps + * the policy's maxMs: a failure after the request went out may have + * reached the server, and a sooner repeat would make a duplicate likelier. */ const val TRANSPORT_FAILURE_MAX_MS = 15L * 60 * 1000 + /** The app clears a wait only this long after the failure that set it. */ + const val CLEAR_MIN_AGE_MS = 30_000L + + /** + * Whether [error] proves the request never reached the server: it failed + * before a connection was made or secured. From OkHttp 4.12.0: + * RealConnection.connectSocket throws ConnectException ("Failed to + * connect to"); the DNS lookup (Dns.SYSTEM) throws UnknownHostException; + * the socket connect can throw NoRouteToHostException or + * PortUnreachableException; the TLS handshake throws SSLHandshakeException, + * and connectTls throws SSLPeerUnverifiedException. RetryAndFollowUpInterceptor + * rethrows the first of these from a RouteException. A + * SocketTimeoutException is left out: a connect timeout and a read or + * write timeout throw the same class. + */ + fun reachedNoServer(error: Throwable): Boolean = + error is java.net.ConnectException || + error is java.net.UnknownHostException || + error is java.net.NoRouteToHostException || + error is java.net.PortUnreachableException || + error is javax.net.ssl.SSLHandshakeException || + error is javax.net.ssl.SSLPeerUnverifiedException + /** * The highest backoff streak an entry keeps when the app clears its wait. - * Streak 5 is the last wait a worker sleeps out in place (16 s at the - * default base); the next failure waits about 32 s, then the wait doubles. + * Streak 5 is the last wait a worker sleeps out in place (16 times + * baseMs); the next failure waits 32 times baseMs, then the wait doubles. */ const val RETRY_NOW_MAX_STREAK = 5 diff --git a/android/src/main/java/ai/openspace/backgroundupload/Scheduler.kt b/android/src/main/java/ai/openspace/backgroundupload/Scheduler.kt index 8d2cf902..4ba7bef2 100644 --- a/android/src/main/java/ai/openspace/backgroundupload/Scheduler.kt +++ b/android/src/main/java/ai/openspace/backgroundupload/Scheduler.kt @@ -6,6 +6,7 @@ import androidx.work.OneTimeWorkRequest import androidx.work.OneTimeWorkRequestBuilder import androidx.work.WorkInfo import androidx.work.WorkManager +import androidx.work.WorkQuery import androidx.work.workDataOf import java.util.concurrent.TimeUnit import kotlin.math.min @@ -55,31 +56,40 @@ internal enum class MainChainStep { NONE, APPEND, REPLACE } /** * The step for a "run now" on a main chain with [rows] (its unfinished - * rows). [workerRunning]: a worker of the entry runs in this process. + * rows). * * - No row: a fresh chain (APPEND_OR_REPLACE starts one). - * - A run in progress (a RUNNING row, or a worker in this process): a - * queued row after it is enough; with none, append one. - * - No run in progress: a row WorkManager starts now is enough. When every + * - A RUNNING row: a row BLOCKED behind it runs after it, so that is + * enough; with none, append one. + * - No RUNNING row: a row WorkManager starts now is enough. When every * queued row waits, REPLACE them with a fresh row. That is a row a stop * left on WorkManager's backoff (see [WorkManagerBackoff]), and the rows - * blocked behind it. Nothing runs, so the replace cuts no request. - * WorkManager does not start a row before its time, and a row due within - * [START_MARGIN_MS] counts as starting now, so no row starts between the - * read and the replace. + * blocked behind it. A worker of the entry that runs under the wake name + * does not change this: when it ends it may leave no row that can start. + * The fresh row then waits at the per-id gate until that worker ends. + * + * WorkManager does not start a row before its time, and a row due within + * [START_MARGIN_MS] counts as starting now. That margin is the protection + * against replacing a row that just started: the query, the enqueue and + * the cancel run one after the other on WorkManager's executor, and the + * enqueue is not awaited. It is a margin, not a guarantee. */ -internal fun mainChainStep(rows: List, workerRunning: Boolean, now: Long): MainChainStep { +internal fun mainChainStep(rows: List, now: Long): MainChainStep { val open = rows.filter { !it.state.isFinished } if (open.isEmpty()) return MainChainStep.APPEND - if (workerRunning || open.any { it.state == WorkInfo.State.RUNNING }) { - return if (WorkManagerScheduler.hasQueuedSuccessor(open.map { it.state })) MainChainStep.NONE else MainChainStep.APPEND + if (open.any { it.state == WorkInfo.State.RUNNING }) { + val after = open.any { it.state == WorkInfo.State.BLOCKED || it.startsBy(now) } + return if (after) MainChainStep.NONE else MainChainStep.APPEND } if (open.any { it.startsBy(now) }) return MainChainStep.NONE return MainChainStep.REPLACE } -/** A row due this soon counts as one that starts now. */ -internal const val START_MARGIN_MS = 5_000L +/** + * A row due this soon counts as one that starts now. 30 s is WorkManager's + * shortest backoff, so the margin delays an entry by at most that much. + */ +internal const val START_MARGIN_MS = 30_000L /** An ENQUEUED row that WorkManager may start now, or within [START_MARGIN_MS]. */ internal fun WorkRow.startsBy(now: Long) = state == WorkInfo.State.ENQUEUED && nextScheduleAt <= now + START_MARGIN_MS @@ -151,8 +161,8 @@ interface WorkRows { /** The unfinished rows of one unique name. */ fun unfinished(name: String): List - /** Every unfinished row of the library, main chains and wakes. */ - fun library(): List + /** Every row of the library, main chains and wakes, in one of [states]. One query. */ + fun library(states: List): List fun enqueue(name: String, policy: ExistingWorkPolicy, entry: QueueEntry, delayMs: Long) @@ -178,16 +188,18 @@ interface WorkRows { */ open class RowScheduler( private val rows: WorkRows, - private val isWorkerRunning: (id: String) -> Boolean = WorkerGate::isRunning, private val clock: () -> Long = System::currentTimeMillis, ) : WorkScheduler { + private companion object { + val UNFINISHED = listOf(WorkInfo.State.ENQUEUED, WorkInfo.State.RUNNING, WorkInfo.State.BLOCKED) + } /** * Makes sure the entry has a row that starts now or after the run in * progress. See [mainChainStep]. */ override fun schedule(entry: QueueEntry) { - when (mainChainStep(rows.unfinished(entry.id), isWorkerRunning(entry.id), clock())) { + when (mainChainStep(rows.unfinished(entry.id), clock())) { MainChainStep.NONE -> Unit MainChainStep.APPEND -> rows.enqueue(entry.id, ExistingWorkPolicy.APPEND_OR_REPLACE, entry, 0L) MainChainStep.REPLACE -> rows.enqueue(entry.id, ExistingWorkPolicy.REPLACE, entry, 0L) @@ -219,7 +231,7 @@ open class RowScheduler( // One query for every row, not one per entry: the boot sweep runs this on // the queue's only thread. override fun libraryRuns(): List = - rows.library().map { row -> + rows.library(UNFINISHED).map { row -> PendingRun( entryId = row.entryId, state = row.state.name, @@ -231,7 +243,7 @@ open class RowScheduler( override fun startableIds(): Set { val now = clock() - return rows.library() + return rows.library(listOf(WorkInfo.State.ENQUEUED, WorkInfo.State.RUNNING)) .filter { it.state == WorkInfo.State.RUNNING || it.startsBy(now) } .mapNotNull { it.entryId } .toSet() @@ -273,8 +285,11 @@ private class WorkManagerRows(private val workManager: WorkManager) : WorkRows { override fun unfinished(name: String): List = workManager.getWorkInfosForUniqueWork(name).get().filter { !it.state.isFinished }.map(::row) - override fun library(): List = - workManager.getWorkInfosByTag(WorkManagerScheduler.WORK_TAG).get().filter { !it.state.isFinished }.map(::row) + // The states are filtered in the query, so finished rows are never loaded. + override fun library(states: List): List = + workManager.getWorkInfos( + WorkQuery.Builder.fromTags(listOf(WorkManagerScheduler.WORK_TAG)).addStates(states).build(), + ).get().map(::row) override fun enqueue(name: String, policy: ExistingWorkPolicy, entry: QueueEntry, delayMs: Long) { workManager.beginUniqueWork(name, policy, request(entry, delayMs)).enqueue() diff --git a/android/src/main/java/ai/openspace/backgroundupload/UploadWorker.kt b/android/src/main/java/ai/openspace/backgroundupload/UploadWorker.kt index a49f7548..9f1dbfb2 100644 --- a/android/src/main/java/ai/openspace/backgroundupload/UploadWorker.kt +++ b/android/src/main/java/ai/openspace/backgroundupload/UploadWorker.kt @@ -47,7 +47,7 @@ internal class SimpleTransfer(private val run: EntryRun) { EntryRun.AttemptResult.Transient -> { run.reportProgress(0L, total) streak++ - run.backoffOrRelease(a.policy, streak, a.entry.expiresAt) + run.backoffOrRelease(a.policy, streak, a.entry.expiresAt, a.cause) } is EntryRun.AttemptResult.Terminal -> return Settlement.Failed(r.errorKind, r.message, r.response, null, a.url, a.method, bytesSent = run.liveBytes) diff --git a/android/src/main/java/ai/openspace/backgroundupload/WorkerOps.kt b/android/src/main/java/ai/openspace/backgroundupload/WorkerOps.kt index 0ea0fc43..a2c7d0bc 100644 --- a/android/src/main/java/ai/openspace/backgroundupload/WorkerOps.kt +++ b/android/src/main/java/ai/openspace/backgroundupload/WorkerOps.kt @@ -117,31 +117,47 @@ class WorkerOps( * and carries [nextAttemptAt]. It also saves the failure's backoff * [streak], so a stop during the wait keeps that step. Best effort; a * lost write hides the time and loses the step. For a chunked entry, a - * sibling part's next attempt clears the time. + * sibling part's next attempt clears the time. [cause] is a [WaitCause]; + * [partStart] names the chunked part that failed. */ - fun backingOff(id: String, generation: Int, nextAttemptAt: Long, streak: Int) { + fun backingOff( + id: String, + generation: Int, + nextAttemptAt: Long, + streak: Int, + cause: String? = null, + partStart: Long? = null, + ) { val now = clock() var applied = false val next = store.update(id) { e -> if (EntryTransitions.isOwnedRun(e, generation)) { applied = true - EntryTransitions.toBackingOff(e, nextAttemptAt, streak, now) + EntryTransitions.toBackingOff(e, nextAttemptAt, streak, now, cause, partStart) } else e } if (applied && next != null) events.state(next.toRow()) } - /** A chunked part the server accepted. Best effort, as v9: a lost flag only re-sends that part later. */ + /** + * A chunked part the server accepted. Best effort, as v9: a lost flag only + * re-sends that part later. The stored streak goes back to 0 only when + * this part set it, or no part is named: a part that keeps failing keeps + * its streak while its siblings are accepted. + */ fun markAccepted(id: String, generation: Int, index: Int): QueueEntry? = store.update(id) { e -> val parts = e.descriptor?.parts if (e.generation != generation || parts == null || index !in parts.indices) e else { val next = ChunkedParts.withAccepted(parts, index) + val owner = e.backoffPartStart + val resets = owner == null || owner == parts[index].start e.copy( descriptor = e.descriptor.copy(parts = next), bytesSent = ChunkedParts.acceptedBytes(next), - backoffStreak = 0, + backoffStreak = if (resets) 0 else e.backoffStreak, + backoffPartStart = if (resets) null else owner, ) } } @@ -324,13 +340,21 @@ class WorkerOps( * caller knows WorkManager starts the run's own row again at once, and * that run makes the wake ([ensureWake]). */ - fun release(id: String, generation: Int, nextAttemptAt: Long, streak: Int, wake: Boolean = true): Boolean { + fun release( + id: String, + generation: Int, + nextAttemptAt: Long, + streak: Int, + wake: Boolean = true, + cause: String? = null, + partStart: Long? = null, + ): Boolean { val now = clock() var applied = false val next = store.compute(id) { e -> if (EntryTransitions.isOwnedRun(e, generation)) { applied = true - EntryTransitions.toReleased(e!!, nextAttemptAt, streak, now) + EntryTransitions.toReleased(e!!, nextAttemptAt, streak, now, cause, partStart) } else e } if (!applied || next == null) return false diff --git a/android/src/test/java/ai/openspace/backgroundupload/EntryRunTest.kt b/android/src/test/java/ai/openspace/backgroundupload/EntryRunTest.kt index 64378192..00b96626 100644 --- a/android/src/test/java/ai/openspace/backgroundupload/EntryRunTest.kt +++ b/android/src/test/java/ai/openspace/backgroundupload/EntryRunTest.kt @@ -214,21 +214,25 @@ class EntryRunTest { } @Test - fun `a transport failure waits at most 15 minutes, an HTTP failure keeps the 2 hour maximum`() { + fun `a connect failure waits at most 15 minutes, an HTTP failure keeps the 2 hour maximum`() { controller.configureRetry(RetryDefaults(baseMs = 1_000, maxMs = 7_200_000, jitter = 0.0)) controller.enqueue(parsed()) store.save(store.load("e1")!!.copy(backoffStreak = 12)) - host.handler = { _, _ -> throw IOException("connection reset") } + host.handler = { _, _ -> throw java.net.ConnectException("Failed to connect to example.com/1.2.3.4:443") } run() // Streak 13 is 2^12 s, about 68 minutes, capped at 15. - assertEquals(host.clock + RetryClassifier.TRANSPORT_FAILURE_MAX_MS, store.load("e1")!!.nextAttemptAt) - assertEquals(13, store.load("e1")!!.backoffStreak) + val e = store.load("e1")!! + assertEquals(host.clock + RetryClassifier.TRANSPORT_FAILURE_MAX_MS, e.nextAttemptAt) + assertEquals(13, e.backoffStreak) + assertEquals(WaitCause.NO_RESPONSE, e.waitCause) + assertEquals(host.clock, e.failedAt) controller.enqueue(parsed(id = "e2")) store.save(store.load("e2")!!.copy(backoffStreak = 12)) respond(503) run("e2") assertEquals(host.clock + 4_096_000, store.load("e2")!!.nextAttemptAt) + assertEquals(WaitCause.RESPONSE, store.load("e2")!!.waitCause) store.save(store.load("e2")!!.copy(nextAttemptAt = null, backoffStreak = 13)) respond(503) run("e2") @@ -236,15 +240,86 @@ class EntryRunTest { } @Test - fun `a transport failure keeps a maximum below 15 minutes`() { + fun `a failure after the request may have gone out keeps the 2 hour maximum`() { + // A read timeout, or a reset after the body left: the server may have + // the request, so a sooner repeat would make a duplicate likelier. + for ((n, error) in listOf(java.net.SocketTimeoutException("timeout"), IOException("connection reset")).withIndex()) { + controller.enqueue(parsed(id = "t$n")) + store.save(store.load("t$n")!!.copy(backoffStreak = 13)) + host.handler = { _, _ -> throw error } + run("t$n") + assertEquals(error.message, host.clock + 7_200_000, store.load("t$n")!!.nextAttemptAt) + assertEquals(WaitCause.NO_RESPONSE, store.load("t$n")!!.waitCause) + } + } + + @Test + fun `a connect failure keeps a maximum below 15 minutes`() { controller.configureRetry(RetryDefaults(baseMs = 1_000, maxMs = 60_000, jitter = 0.0)) controller.enqueue(parsed()) store.save(store.load("e1")!!.copy(backoffStreak = 12)) - host.handler = { _, _ -> throw IOException("connection reset") } + host.handler = { _, _ -> throw java.net.UnknownHostException("example.com") } run() assertEquals(host.clock + 60_000, store.load("e1")!!.nextAttemptAt) } + // MARK: - the app clears a wait (review probes P1 and P2) + + @Test + fun `repeated returns to the foreground do not send again after a 503`() { + // Probe P1: before, 10 foregrounds gave 10 requests with 0 ms between. + controller.enqueue(parsed()) + store.save(store.load("e1")!!.copy(backoffStreak = 13)) + host.handler = { _, _ -> UploadResponse(503, "down", mapOf()) } + repeat(10) { + controller.retryNow() + run() + } + assertEquals(1, host.requests.size) + assertEquals(WaitCause.RESPONSE, store.load("e1")!!.waitCause) + } + + @Test + fun `repeated returns to the foreground after a connect failure send at most once per 30 s`() { + controller.enqueue(parsed()) + store.save(store.load("e1")!!.copy(backoffStreak = 13)) + val sentAt = mutableListOf() + host.handler = { _, _ -> + sentAt += host.clock + throw java.net.ConnectException("Failed to connect") + } + repeat(10) { + controller.retryNow() + run() + host.clock += 10_000 + } + // The first clear comes 30 s after the failure (its own wait was 15 + // minutes). Later requests come from a clear or from the entry's own + // backoff, and never less than 30 s apart. + assertEquals(sentAt.toString(), sentAt[0] + RetryClassifier.CLEAR_MIN_AGE_MS, sentAt[1]) + assertTrue(sentAt.toString(), sentAt.zipWithNext().all { (a, b) -> b - a >= RetryClassifier.CLEAR_MIN_AGE_MS }) + assertTrue(sentAt.toString(), sentAt.size < 10) + } + + @Test + fun `returns to the foreground do not spend a transient rule's retries`() { + // Probe P2: before, six foregrounds used all 5 retries and the entry + // settled error with no time between. + controller.configureRetry( + RetryDefaults(baseMs = 60_000, jitter = 0.0, transient = listOf(UploadOutcome.TransientRule(409, maxRetries = 5))), + ) + controller.enqueue(parsed()) + host.handler = { _, _ -> UploadResponse(409, "not yet", mapOf()) } + repeat(20) { + controller.retryNow() + run() + } + val e = store.load("e1")!! + assertEquals(EntryState.QUEUED, e.state) + assertEquals(1, host.requests.size) + assertEquals(1, e.transientHitsByRule!!.values.sum()) + } + @Test fun `a terminal response settles error http with the response and the live bytes`() { controller.enqueue(parsed()) @@ -1023,6 +1098,78 @@ class EntryRunTest { assertEquals(listOf("e1" to host.clock + 512_000), scheduler.wakes) // base 1 s * 2^(10-1) } + @Test + fun `the part whose failures set the streak goes on from it, the others start from 0`() { + chunked() + // Part 1 (start 10) set streak 9 in an earlier run. + store.save(store.load("e1")!!.copy(backoffStreak = 9, backoffPartStart = 10)) + host.handler = { r, _ -> + UploadResponse(if (r.url.endsWith("start=10")) 503 else 200, "", mapOf()) + } + run() + val e = store.load("e1")!! + assertEquals(EntryState.QUEUED, e.state) + // Part 1's next failure is step 10: 2^9 s. The accepted siblings did not reset it. + assertEquals(10, e.backoffStreak) + assertEquals(10L, e.backoffPartStart) + assertEquals(listOf("e1" to host.clock + 512_000), scheduler.wakes) + assertTrue(e.descriptor!!.parts!![0].accepted) + assertFalse(e.descriptor!!.parts!![1].accepted) + } + + @Test + fun `a part other than the one that set the streak counts its own failures from 0`() { + chunked() + store.save(store.load("e1")!!.copy(backoffStreak = 9, backoffPartStart = 10)) + var failed = false + host.handler = { r, _ -> + if (r.url.endsWith("start=0") && !failed) { + failed = true + UploadResponse(503, "", mapOf()) + } else UploadResponse(200, "", mapOf()) + } + run() + assertEquals(EntryState.COMPLETED, store.load("e1")!!.state) + assertEquals(listOf(1_000L), host.sleeps) // step 1, not step 10 + } + + @Test + fun `a part that always fails backs off further in each run while its siblings are accepted`() { + // Probe P6: before, part 0 started from step 1 in every run: 48 + // attempts in 8 runs, a 32 s wait every time. + val n = 40 + val src = File(tmp.newFolder(), "video.bin").apply { writeBytes(ByteArray(n * 10) { it.toByte() }) } + controller.enqueue( + parsed(descriptor = desc(url = null, method = "PUT", file = src.path, parts = (0 until n).map { part(it * 10L, it * 10L + 10) })), + ) + var part0 = 0 + var acceptedThisRun = 0 + host.handler = { r, _ -> + val index = r.url.substringAfter("start=").toInt() / 10 + if (index == 0) { + kotlinx.coroutines.yield() + part0++ + UploadResponse(503, "", mapOf()) + } else if (acceptedThisRun++ == 0) { + UploadResponse(200, "", mapOf()) + } else { + awaitCancellation() + } + } + val streaks = mutableListOf() + repeat(8) { + acceptedThisRun = 0 + run() + val e = store.load("e1")!! + streaks += e.backoffStreak + host.clock = e.nextAttemptAt ?: host.clock + } + // Six attempts in the first run (1, 2, 4, 8, 16 s in place, then 32 s), + // then one per run, each wait twice the one before. + assertEquals(6 + 7, part0) + assertEquals((6..13).toList(), streaks) + } + /** Four 10-byte parts. [code] answers each request by its part index and how many times that part was sent. */ private fun fourParts(code: (part: Int, sent: Int) -> Int) { val src = File(tmp.newFolder(), "video.bin").apply { writeBytes(ByteArray(40) { it.toByte() }) } diff --git a/android/src/test/java/ai/openspace/backgroundupload/EntryTransitionsTest.kt b/android/src/test/java/ai/openspace/backgroundupload/EntryTransitionsTest.kt index 43a7d2db..cc70e487 100644 --- a/android/src/test/java/ai/openspace/backgroundupload/EntryTransitionsTest.kt +++ b/android/src/test/java/ai/openspace/backgroundupload/EntryTransitionsTest.kt @@ -53,13 +53,49 @@ class EntryTransitionsTest { } @Test - fun `resume to queued starts the backoff over, resume to awaiting-auth keeps it`() { + fun `resume to queued keeps at most 5 backoff steps, resume to awaiting-auth keeps them all`() { val paused = entry(state = EntryState.PAUSED).copy(backoffStreak = 9) - assertEquals(0, EntryTransitions.toResumed(paused, 9).backoffStreak) + assertEquals(5, EntryTransitions.toResumed(paused, 9).backoffStreak) + assertEquals(3, EntryTransitions.toResumed(paused.copy(backoffStreak = 3), 9).backoffStreak) val parked = paused.copy(parkedGeneration = 3, headerGeneration = 3) assertEquals(9, EntryTransitions.toResumed(parked, 9).backoffStreak) } + @Test + fun `a wait is cleared only after a failure with no response, or of unknown cause, 30 s or more ago`() { + val now = 100_000L + val wait = entry(state = EntryState.QUEUED, nextAttemptAt = now + 3_600_000) + fun clears(cause: String?, failedAt: Long?) = + EntryTransitions.clearsWait(wait.copy(waitCause = cause, failedAt = failedAt), now) + assertTrue(clears(WaitCause.NO_RESPONSE, now - 30_000)) + assertFalse(clears(WaitCause.NO_RESPONSE, now - 29_999)) + assertFalse(clears(WaitCause.RESPONSE, now - 3_600_000)) + // An older version saved it: the cause is unknown, and its time is updatedAt. + assertTrue(clears(null, null)) + assertFalse(EntryTransitions.clearsWait(wait.copy(updatedAt = now - 1_000), now)) + assertFalse(EntryTransitions.clearsWait(entry(state = EntryState.QUEUED), now)) + assertFalse(EntryTransitions.clearsWait(wait.copy(state = EntryState.RUNNING, failedAt = 0), now)) + } + + @Test + fun `a failure saves its cause and time, and the part that holds the highest streak`() { + val chunked = entry(state = EntryState.RUNNING) + val first = EntryTransitions.toBackingOff(chunked, 9_000, 6, now = 5, cause = WaitCause.RESPONSE, partStart = 0) + assertEquals(WaitCause.RESPONSE, first.waitCause) + assertEquals(5L, first.failedAt) + assertEquals(0L, first.backoffPartStart) + // A sibling's lower streak keeps the part that holds 6. + val sibling = EntryTransitions.toBackingOff(first, 9_500, 1, now = 6, cause = WaitCause.NO_RESPONSE, partStart = 10) + assertEquals(6, sibling.backoffStreak) + assertEquals(0L, sibling.backoffPartStart) + assertEquals(WaitCause.NO_RESPONSE, sibling.waitCause) + val released = EntryTransitions.toReleased(sibling, 40_000, 2, now = 7, cause = WaitCause.RESPONSE, partStart = 10) + assertEquals(6, released.backoffStreak) + assertEquals(0L, released.backoffPartStart) + // A simple entry's release sets the streak it is given, as before. + assertEquals(2, EntryTransitions.toReleased(chunked.copy(backoffStreak = 6), 40_000, 2, now = 7).backoffStreak) + } + @Test fun `pause keeps the parked generation and clears the wake time`() { val p = EntryTransitions.toPaused(entry(state = EntryState.AWAITING_AUTH, parkedGeneration = 2, nextAttemptAt = 50), 9) diff --git a/android/src/test/java/ai/openspace/backgroundupload/ForegroundTaskTest.kt b/android/src/test/java/ai/openspace/backgroundupload/ForegroundTaskTest.kt index d3ee6e0e..6d5c37eb 100644 --- a/android/src/test/java/ai/openspace/backgroundupload/ForegroundTaskTest.kt +++ b/android/src/test/java/ai/openspace/backgroundupload/ForegroundTaskTest.kt @@ -49,10 +49,10 @@ class ForegroundTaskTest { Diag.log = DiagnosticLog(file) val executor = HeldExecutor() var retried = false - val task = ForegroundTask(executor, { throw IllegalStateException("settings") }) { + val task = ForegroundTask(executor, { throw IllegalStateException("settings") }, retryNow = { retried = true throw IllegalStateException("work") - } + }) task.onForeground() executor.runAll() assertEquals(true, retried) @@ -62,4 +62,21 @@ class ForegroundTaskTest { errors.map { it.message }, ) } + + @Test + fun `retryNow runs at most once per 30 s, the settings save every time`() { + val executor = HeldExecutor() + var now = 1_000_000L + val calls = mutableListOf() + val task = ForegroundTask(executor, { calls += "save" }, { calls += "retryNow" }) { now } + task.onForeground() + executor.runAll() + now += 29_999 // a permission dialog, a picker, a share sheet + task.onForeground() + executor.runAll() + now += 1 + task.onForeground() + executor.runAll() + assertEquals(listOf("save", "retryNow", "save", "save", "retryNow"), calls) + } } diff --git a/android/src/test/java/ai/openspace/backgroundupload/MainChainTest.kt b/android/src/test/java/ai/openspace/backgroundupload/MainChainTest.kt index 920aae06..8f9aab8d 100644 --- a/android/src/test/java/ai/openspace/backgroundupload/MainChainTest.kt +++ b/android/src/test/java/ai/openspace/backgroundupload/MainChainTest.kt @@ -1,10 +1,12 @@ package ai.openspace.backgroundupload +import androidx.work.ExistingWorkPolicy import androidx.work.WorkInfo.State.BLOCKED import androidx.work.WorkInfo.State.CANCELLED import androidx.work.WorkInfo.State.ENQUEUED import androidx.work.WorkInfo.State.RUNNING import org.junit.Assert.assertEquals +import org.junit.Assert.assertNull import org.junit.Assert.assertTrue import org.junit.Before import org.junit.Rule @@ -29,17 +31,42 @@ class MainChainTest { private lateinit var settings: QueueSettingsStore private lateinit var controller: QueueController private lateinit var scheduler: RowScheduler + private lateinit var journal: EventJournal + private val events = RecordingEvents() + + /** WorkManager calls in order, and an enqueue that throws, as a process death before it lands. */ + private inner class LoggedRows : WorkRows by work { + val log = mutableListOf() + var enqueueFails = false + + override fun enqueue(name: String, policy: ExistingWorkPolicy, entry: QueueEntry, delayMs: Long) { + if (enqueueFails) throw IllegalStateException("the process died before the enqueue") + log += "enqueue:$name:$policy" + work.enqueue(name, policy, entry, delayMs) + } + + override fun cancel(name: String) { + log += "cancel:$name" + work.cancel(name) + } + } + + private val rows = LoggedRows() @Before fun setUp() { val root = tmp.newFolder("queue") store = QueueStore(root, RequestIndex()) - val journal = EventJournal(tmp.newFolder("journal"), retryLater = { _, _ -> }) + journal = EventJournal(tmp.newFolder("journal"), retryLater = { _, _ -> }) settings = QueueSettingsStore(File(root, "settings.json")) - scheduler = RowScheduler(work, { it in running }, { work.now }) - controller = QueueController(store, journal, settings, RecordingEvents(), scheduler, { it in running }, { work.now }) + scheduler = RowScheduler(rows) { work.now } + controller = QueueController(store, journal, settings, events, scheduler, { it in running }, { work.now }) } + /** A queued entry that waits after a transport failure 10 minutes ago. */ + private fun transportWait(id: String = "e1", streak: Int = 12) = entry(id = id, state = EntryState.QUEUED, nextAttemptAt = work.now + 60 * minute) + .copy(backoffStreak = streak, waitCause = WaitCause.NO_RESPONSE, failedAt = work.now - 10 * minute) + /** The rows of [id] WorkManager may start now, main chain and wake. */ private fun startable(id: String = "e1") = work.startableNow().filter { it.entryId == id } @@ -158,15 +185,19 @@ class MainChainTest { } @Test - fun `a held-back row is kept while a worker of the entry runs in this process`() { - // The wake row runs the entry; a new main row would only wait at the gate. - store.save(entry(state = EntryState.QUEUED)) - val row = work.add("e1", runAttemptCount = 8, lastEnqueueTime = work.now - minute) + fun `a held-back main row is replaced while a wake worker of the entry runs`() { + // Probe P5. The wake's worker released the entry and still holds the + // gate. Then it schedules its wake at the old time and ends. + store.save(transportWait()) + val held = work.add("e1", runAttemptCount = 8, lastEnqueueTime = work.now - minute) work.add("e1#wake", state = RUNNING, runAttemptCount = 1) running += "e1" - controller.resume() - assertEquals(ENQUEUED, row.state) - assertEquals(2, work.rows.size) + controller.retryNow() + scheduler.scheduleWake(store.load("e1")!!, work.now + 60 * minute, replace = true) + running -= "e1" + assertEquals(CANCELLED, held.state) + assertNull(store.load("e1")!!.nextAttemptAt) + assertEquals(listOf("e1"), startable().map { it.name }) } @Test @@ -178,6 +209,8 @@ class MainChainTest { work.add("blocked", state = BLOCKED) work.add("gone", state = CANCELLED) assertEquals(setOf("due", "run"), scheduler.startableIds()) + // One query, and it asks only for the states that can start. + assertEquals(listOf(listOf(ENQUEUED, RUNNING)), work.libraryQueries) } @Test @@ -193,4 +226,60 @@ class MainChainTest { assertEquals(work.nextScheduleAt(row), WorkManagerBackoff.restartBy(startedAt, 7)) assertTrue(WorkManagerBackoff.restartBy(startedAt, 4) < startedAt + 10 * minute) } + + // MARK: - order, and cost + + @Test + fun `the main row is made before the wake is cancelled`() { + store.save(transportWait()) + work.add("e1#wake", initialDelay = 60 * minute) + controller.retryNow() + assertEquals(listOf("enqueue:e1:APPEND_OR_REPLACE", "cancel:e1#wake"), rows.log) + } + + @Test + fun `a process death before the main row lands keeps the wake`() { + // Probe P3: with the wake cancelled first, the entry had no row at all. + store.save(transportWait()) + work.add("e1#wake", initialDelay = 60 * minute) + rows.enqueueFails = true + controller.retryNow() + assertEquals(listOf("e1#wake"), work.rows.filter { !it.state.isFinished }.map { it.name }) + } + + @Test + fun `the boot sweep then an immediate return to the foreground clear a wait once`() { + store.save(transportWait()) + work.add("e1#wake", initialDelay = 60 * minute) + controller.sweep() + controller.retryNow() + assertEquals(listOf("state:e1:queued"), events.log) + assertEquals(listOf("enqueue:e1:APPEND_OR_REPLACE", "cancel:e1#wake"), rows.log) + assertEquals(1, startable().size) + } + + @Test + fun `retryNow costs no save and no WorkManager query for entries whose wait stays`() { + for (n in 1..3) { + store.save(transportWait("r$n").copy(waitCause = WaitCause.RESPONSE)) + } + controller.retryNow() + assertEquals(0, work.nameQueries) + assertEquals(emptyList>(), work.libraryQueries) + assertEquals(emptyList(), rows.log) + assertEquals(emptyList(), events.log) // no save changed a row + } + + @Test + fun `retryNow costs one save and two name queries per cleared entry, and one query for the idle ones`() { + for (n in 1..3) store.save(transportWait("t$n")) + for (n in 1..2) store.save(entry(id = "i$n", state = EntryState.QUEUED)) + work.add("i1") + controller.retryNow() + assertEquals(3, events.log.size) // the three saves + assertEquals(1, work.libraryQueries.size) + // Cleared: schedule() and cancelWake() read one name each. Idle "i2" + // has no row that can start, so it is scheduled too; "i1" has one. + assertEquals(2 * 4, work.nameQueries) + } } diff --git a/android/src/test/java/ai/openspace/backgroundupload/QueueControllerTest.kt b/android/src/test/java/ai/openspace/backgroundupload/QueueControllerTest.kt index 702e398a..4629fee5 100644 --- a/android/src/test/java/ai/openspace/backgroundupload/QueueControllerTest.kt +++ b/android/src/test/java/ai/openspace/backgroundupload/QueueControllerTest.kt @@ -1139,7 +1139,7 @@ class QueueControllerTest { fun `sweep queues a running entry with no worker and schedules queued work`() { store.save(entry(id = "r", state = EntryState.RUNNING)) store.save(entry(id = "q", state = EntryState.QUEUED)) - store.save(entry(id = "w", state = EntryState.QUEUED, nextAttemptAt = now + 3_600_000)) + store.save(entry(id = "w", state = EntryState.QUEUED, nextAttemptAt = now + 3_600_000).copy(waitCause = WaitCause.NO_RESPONSE, failedAt = now - 60_000)) store.save(entry(id = "p", state = EntryState.AWAITING_AUTH, expiresAt = now + 50)) controller.sweep() assertEquals(EntryState.QUEUED, store.load("r")!!.state) @@ -1151,7 +1151,7 @@ class QueueControllerTest { @Test fun `sweep cancels the stale wakes of due work and of a parked entry with a far expiry`() { store.save(entry(id = "q", state = EntryState.QUEUED)) - store.save(entry(id = "w", state = EntryState.QUEUED, nextAttemptAt = now + 3_600_000)) + store.save(entry(id = "w", state = EntryState.QUEUED, nextAttemptAt = now + 3_600_000).copy(waitCause = WaitCause.NO_RESPONSE, failedAt = now - 60_000)) store.save(entry(id = "far", state = EntryState.AWAITING_AUTH, expiresAt = FAR_FUTURE)) store.save(entry(id = "near", state = EntryState.AWAITING_AUTH, expiresAt = now + EXPIRY_WAKE_HORIZON_MS)) controller.sweep() @@ -1162,9 +1162,9 @@ class QueueControllerTest { @Test fun `the boot sweep clears a long backoff and runs the entry now`() { - store.save(entry(id = "w", state = EntryState.QUEUED, nextAttemptAt = now + 7_200_000).copy(backoffStreak = 13)) - store.save(entry(id = "s", state = EntryState.QUEUED, nextAttemptAt = now + 4_000).copy(backoffStreak = 3)) - store.save(entry(id = "r", state = EntryState.RUNNING, nextAttemptAt = now + 7_200_000).copy(backoffStreak = 9)) + store.save(entry(id = "w", state = EntryState.QUEUED, nextAttemptAt = now + 7_200_000).copy(waitCause = WaitCause.NO_RESPONSE, failedAt = now - 60_000, backoffStreak = 13)) + store.save(entry(id = "s", state = EntryState.QUEUED, nextAttemptAt = now + 4_000).copy(waitCause = WaitCause.NO_RESPONSE, failedAt = now - 60_000, backoffStreak = 3)) + store.save(entry(id = "r", state = EntryState.RUNNING, nextAttemptAt = now + 7_200_000).copy(waitCause = WaitCause.NO_RESPONSE, failedAt = now - 60_000, backoffStreak = 9)) controller.sweep() for (id in listOf("w", "s", "r")) { val e = store.load(id)!! @@ -1182,7 +1182,7 @@ class QueueControllerTest { @Test fun `the boot sweep keeps the wait of an entry whose worker runs in this process`() { - store.save(entry(id = "w", state = EntryState.QUEUED, nextAttemptAt = now + 20_000).copy(backoffStreak = 4)) + store.save(entry(id = "w", state = EntryState.QUEUED, nextAttemptAt = now + 20_000).copy(waitCause = WaitCause.NO_RESPONSE, failedAt = now - 60_000, backoffStreak = 4)) running += "w" controller.sweep() assertEquals(now + 20_000, store.load("w")!!.nextAttemptAt) @@ -1193,8 +1193,8 @@ class QueueControllerTest { @Test fun `retryNow clears the wait of queued entries, keeps at most 5 steps, and runs them now`() { - store.save(entry(id = "w", state = EntryState.QUEUED, nextAttemptAt = now + 7_200_000).copy(backoffStreak = 13)) - store.save(entry(id = "s", state = EntryState.QUEUED, nextAttemptAt = now + 60_000).copy(backoffStreak = 2)) + store.save(entry(id = "w", state = EntryState.QUEUED, nextAttemptAt = now + 7_200_000).copy(waitCause = WaitCause.NO_RESPONSE, failedAt = now - 60_000, backoffStreak = 13)) + store.save(entry(id = "s", state = EntryState.QUEUED, nextAttemptAt = now + 60_000).copy(waitCause = WaitCause.NO_RESPONSE, failedAt = now - 60_000, backoffStreak = 2)) controller.retryNow() assertNull(store.load("w")!!.nextAttemptAt) assertEquals(5, store.load("w")!!.backoffStreak) @@ -1210,7 +1210,7 @@ class QueueControllerTest { @Test fun `retryNow leaves running, paused, parked and settled entries alone`() { // A running entry waits out a short backoff in its own worker. - store.save(entry(id = "r", state = EntryState.RUNNING, nextAttemptAt = now + 20_000).copy(backoffStreak = 4)) + store.save(entry(id = "r", state = EntryState.RUNNING, nextAttemptAt = now + 20_000).copy(waitCause = WaitCause.NO_RESPONSE, failedAt = now - 60_000, backoffStreak = 4)) store.save(entry(id = "p", state = EntryState.PAUSED).copy(backoffStreak = 9)) store.save(entry(id = "a", state = EntryState.AWAITING_AUTH, parkedGeneration = 0, nextAttemptAt = now + 60_000)) store.save(entry(id = "x", state = EntryState.ERROR, settledEventId = "gone")) @@ -1226,8 +1226,8 @@ class QueueControllerTest { @Test fun `retryNow with keys clears only the waits of those keys`() { - store.save(entry(id = "c", key = "capture", state = EntryState.QUEUED, nextAttemptAt = now + 60_000)) - store.save(entry(id = "n", key = "note", state = EntryState.QUEUED, nextAttemptAt = now + 60_000)) + store.save(entry(id = "c", key = "capture", state = EntryState.QUEUED, nextAttemptAt = now + 60_000).copy(waitCause = WaitCause.NO_RESPONSE, failedAt = now - 60_000)) + store.save(entry(id = "n", key = "note", state = EntryState.QUEUED, nextAttemptAt = now + 60_000).copy(waitCause = WaitCause.NO_RESPONSE, failedAt = now - 60_000)) controller.retryNow(listOf("capture")) assertNull(store.load("c")!!.nextAttemptAt) assertEquals(now + 60_000, store.load("n")!!.nextAttemptAt) @@ -1249,8 +1249,8 @@ class QueueControllerTest { @Test fun `retryNow saves the other entries when one can not be saved, and does not throw`() { - store.save(entry(id = "q", state = EntryState.QUEUED, nextAttemptAt = now + 60_000)) - store.save(entry(id = "x", state = EntryState.QUEUED, nextAttemptAt = now + 60_000)) + store.save(entry(id = "q", state = EntryState.QUEUED, nextAttemptAt = now + 60_000).copy(waitCause = WaitCause.NO_RESPONSE, failedAt = now - 60_000)) + store.save(entry(id = "x", state = EntryState.QUEUED, nextAttemptAt = now + 60_000).copy(waitCause = WaitCause.NO_RESPONSE, failedAt = now - 60_000)) withEntryUnwritable("x") { controller.retryNow() } assertNull(store.load("q")!!.nextAttemptAt) assertEquals(now + 60_000, store.load("x")!!.nextAttemptAt) @@ -1258,12 +1258,12 @@ class QueueControllerTest { } @Test - fun `resume clears the backoff of queued entries and resets the streak of the ones it resumes`() { + fun `resume clears the backoff of queued entries and caps the streak of the ones it resumes`() { store.save(entry(id = "p", state = EntryState.PAUSED).copy(backoffStreak = 9)) - store.save(entry(id = "w", state = EntryState.QUEUED, nextAttemptAt = now + 7_200_000).copy(backoffStreak = 13)) + store.save(entry(id = "w", state = EntryState.QUEUED, nextAttemptAt = now + 7_200_000).copy(waitCause = WaitCause.NO_RESPONSE, failedAt = now - 60_000, backoffStreak = 13)) controller.resume() assertEquals(EntryState.QUEUED, store.load("p")!!.state) - assertEquals(0, store.load("p")!!.backoffStreak) + assertEquals(5, store.load("p")!!.backoffStreak) assertNull(store.load("w")!!.nextAttemptAt) assertEquals(5, store.load("w")!!.backoffStreak) assertTrue(scheduler.scheduled.containsAll(listOf("p", "w"))) @@ -1272,13 +1272,99 @@ class QueueControllerTest { @Test fun `resume of some keys clears only the waits of those keys`() { - store.save(entry(id = "c", key = "capture", state = EntryState.QUEUED, nextAttemptAt = now + 60_000)) - store.save(entry(id = "n", key = "note", state = EntryState.QUEUED, nextAttemptAt = now + 60_000)) + store.save(entry(id = "c", key = "capture", state = EntryState.QUEUED, nextAttemptAt = now + 60_000).copy(waitCause = WaitCause.NO_RESPONSE, failedAt = now - 60_000)) + store.save(entry(id = "n", key = "note", state = EntryState.QUEUED, nextAttemptAt = now + 60_000).copy(waitCause = WaitCause.NO_RESPONSE, failedAt = now - 60_000)) controller.resume(listOf("capture")) assertNull(store.load("c")!!.nextAttemptAt) assertEquals(now + 60_000, store.load("n")!!.nextAttemptAt) assertEquals(listOf("n" to now + 60_000), scheduler.wakes) } + // MARK: - which waits are cleared + + private fun waitAfter(id: String, cause: String?, age: Long, at: Long = now + 3_600_000) = + entry(id = id, state = EntryState.QUEUED, nextAttemptAt = at) + .copy(backoffStreak = 9, waitCause = cause, failedAt = if (cause == null) null else now - age) + + /** Each of the three triggers, on a fresh copy of [saved]. */ + private fun eachTrigger(saved: () -> Unit, check: (trigger: String) -> Unit) { + for (trigger in listOf("sweep", "resume", "foreground")) { + saved() + when (trigger) { + "sweep" -> controller.sweep() + "resume" -> controller.resume() + else -> controller.retryNow() + } + check(trigger) + } + } + + @Test + fun `a wait after a server response keeps its time at the boot sweep, resume and a return to the foreground`() { + val at = now + 3_600_000 + eachTrigger({ store.save(waitAfter("e1", WaitCause.RESPONSE, age = 3_600_000, at = at)) }) { trigger -> + assertEquals(trigger, at, store.load("e1")!!.nextAttemptAt) + assertEquals(trigger, 9, store.load("e1")!!.backoffStreak) + } + assertEquals(emptyList(), scheduler.scheduled) + } + + @Test + fun `a transport-failure wait 30 s or older is cleared by each trigger`() { + eachTrigger({ store.save(waitAfter("e1", WaitCause.NO_RESPONSE, age = 30_000)) }) { trigger -> + assertNull(trigger, store.load("e1")!!.nextAttemptAt) + assertEquals(trigger, 5, store.load("e1")!!.backoffStreak) + } + } + + @Test + fun `a transport-failure wait younger than 30 s keeps its time at each trigger`() { + val at = now + 3_600_000 + eachTrigger({ store.save(waitAfter("e1", WaitCause.NO_RESPONSE, age = 29_999, at = at)) }) { trigger -> + assertEquals(trigger, at, store.load("e1")!!.nextAttemptAt) + } + } + + @Test + fun `a wait an older version saved is cleared by each trigger`() { + now = 100_000 // the entry was saved at 1_000 + eachTrigger({ store.save(waitAfter("e1", cause = null, age = 0)) }) { trigger -> + assertNull(trigger, store.load("e1")!!.nextAttemptAt) + } + } + + @Test + fun `retryNow goes on past an entry whose scheduling fails, emits what it saved, and writes one error`() { + // Probe P4. + val file = File(tmp.newFolder(), "diagnostics.json") + Diag.log = DiagnosticLog(file) + var calls = 0 + val failing = object : WorkScheduler by scheduler { + override fun schedule(entry: QueueEntry) { + if (calls++ == 0) throw IllegalStateException("WorkManager failed") + scheduler.schedule(entry) + } + } + val c = QueueController(store, journal, settings, events, failing, { false }, { now }) + store.save(waitAfter("a", WaitCause.NO_RESPONSE, age = 60_000)) + store.save(waitAfter("b", WaitCause.NO_RESPONSE, age = 60_000)) + c.retryNow() + assertEquals(1, scheduler.scheduled.size) + assertEquals(listOf("state:a:queued", "state:b:queued"), events.log.sorted()) + val errors = DiagnosticLog(file).drain() + assertEquals(1, errors.size) + assertTrue(errors.single().message, errors.single().message.startsWith("retryNow: 1 entry saves or WorkManager calls failed")) + } + + @Test + fun `a sweep whose clear can not be saved schedules the entry as found, and its wake is made again`() { + val at = now + 3_600_000 + store.save(waitAfter("w", WaitCause.NO_RESPONSE, age = 60_000, at = at)) + withEntryUnwritable("w") { controller.sweep() } + assertEquals(at, store.load("w")!!.nextAttemptAt) + assertEquals(listOf("w" to at), scheduler.wakes) + assertEquals(listOf(false), scheduler.wakeReplaces) // KEEP: a wake that exists stays + } + @Test fun `sweep skips an entry whose worker runs in this process`() { diff --git a/android/src/test/java/ai/openspace/backgroundupload/RetryClassifierTest.kt b/android/src/test/java/ai/openspace/backgroundupload/RetryClassifierTest.kt index 44c6d5aa..58923ebf 100644 --- a/android/src/test/java/ai/openspace/backgroundupload/RetryClassifierTest.kt +++ b/android/src/test/java/ai/openspace/backgroundupload/RetryClassifierTest.kt @@ -182,4 +182,26 @@ class RetryClassifierTest { assertEquals(rules, RetryClassifier.policy(defaults, RetryOverride(null, null, null, null, transient = rules)).transient) assertEquals(emptyList(), RetryClassifier.policy(RetryDefaults(transient = rules), RetryOverride(null, null, null, null, transient = emptyList())).transient) } + + @Test + fun `only a failure before the connection is made or secured proves the request never reached the server`() { + assertTrue(RetryClassifier.reachedNoServer(java.net.ConnectException("Failed to connect to example.com"))) + assertTrue(RetryClassifier.reachedNoServer(java.net.UnknownHostException("example.com"))) + assertTrue(RetryClassifier.reachedNoServer(java.net.NoRouteToHostException())) + assertTrue(RetryClassifier.reachedNoServer(java.net.PortUnreachableException())) + assertTrue(RetryClassifier.reachedNoServer(javax.net.ssl.SSLHandshakeException("handshake"))) + assertTrue(RetryClassifier.reachedNoServer(javax.net.ssl.SSLPeerUnverifiedException("hostname"))) + // A connect timeout and a read or write timeout throw the same class. + assertFalse(RetryClassifier.reachedNoServer(java.net.SocketTimeoutException("timeout"))) + assertFalse(RetryClassifier.reachedNoServer(IOException("unexpected end of stream"))) + assertFalse(RetryClassifier.reachedNoServer(java.net.SocketException("Connection reset"))) + assertFalse(RetryClassifier.reachedNoServer(javax.net.ssl.SSLException("Read error"))) + } + + @Test + fun `only a connect failure caps the policy at 15 minutes`() { + val policy = RetryClassifier.Policy(1_000, 7_200_000, 0.0, listOf(404)) + assertEquals(RetryClassifier.TRANSPORT_FAILURE_MAX_MS, RetryClassifier.transportFailurePolicy(policy).maxMs) + assertEquals(60_000L, RetryClassifier.transportFailurePolicy(policy.copy(maxMs = 60_000)).maxMs) + } } diff --git a/android/src/test/java/ai/openspace/backgroundupload/TestSupport.kt b/android/src/test/java/ai/openspace/backgroundupload/TestSupport.kt index f3cea70f..a7ac4e34 100644 --- a/android/src/test/java/ai/openspace/backgroundupload/TestSupport.kt +++ b/android/src/test/java/ai/openspace/backgroundupload/TestSupport.kt @@ -210,9 +210,21 @@ internal class FakeWorkManager(var now: Long = 10_000L) : WorkRows { fun startableNow(): List = rows.filter { it.state == WorkInfo.State.ENQUEUED && nextScheduleAt(it) <= now } - override fun unfinished(name: String): List = open(name).map(::view) + override fun unfinished(name: String): List { + nameQueries++ + return open(name).map(::view) + } + + /** How many library() queries were made, and the states each asked for. */ + val libraryQueries = mutableListOf>() - override fun library(): List = rows.filter { !it.state.isFinished }.map(::view) + /** How many unfinished(name) queries were made. */ + var nameQueries = 0 + + override fun library(states: List): List { + libraryQueries += states + return rows.filter { it.state in states }.map(::view) + } override fun enqueue(name: String, policy: ExistingWorkPolicy, entry: QueueEntry, delayMs: Long) { val open = open(name) From 11cac34609b524c5c8178e2c9a1298469263633d Mon Sep 17 00:00:00 2001 From: Dylan Murphy Date: Tue, 17 Nov 2026 18:37:59 -0500 Subject: [PATCH 04/10] Docs: which waits the app clears, and how often CHANGELOG (Unreleased) and README: a wait is cleared only after a failure with no server response (or an older entry) and 30 s or more after it; a wait after a server response keeps its time; a return to the foreground is every Activity resume, limited to once per 30 s; the next wait after a clear is 32 times baseMs; the 15-minute cap covers connect-phase failures only; the chunked part rule. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 57 ++++++++++++++++++++++++++++++++++------------------ README.md | 31 ++++++++++++++++------------ 2 files changed, 55 insertions(+), 33 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c9b2b0f9..d0cbda99 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -50,30 +50,47 @@ events, are lower for uploads that only waited. The number of attempt events does not change. That is the correct count. **Behavior change:** on Android, the boot sweep, `resume()` and a return to -the foreground now clear a backoff wait and start queued uploads at once. -Before, an entry on a long backoff (up to 2 hours) kept its wait when the -app opened, and `resume()` kept it too. `resume({ keys })` clears the waits -of those keys only. The entry keeps at most 5 backoff steps, so if the -server still fails, the next wait is about 32 s, then it doubles. A paused -entry that `resume()` moves back to `queued` starts its backoff over. The -app needs no code for this. +the foreground now end some backoff waits and start those uploads at once. +A wait is ended only when the failure behind it had no server response (a +transport failure, or a stop that cut the request), or when an older +version saved the entry, and only 30 s or more after that failure. A wait +after a server response (a 5xx, 408, 429, or a status a `transient` rule +retries) keeps its time: the server asked for it, and ending it early +would spend a rule's `maxRetries` faster. Before, an entry on a long +backoff (up to 2 hours) kept its wait when the app opened, and `resume()` +kept it too. `resume({ keys })` affects those keys only. The entry keeps at +most 5 backoff steps, so if the request fails again, the next wait is 32 +times `baseMs` (32 s at the default 1 s base), and then it doubles. A paused entry that `resume()` moves back to `queued` also keeps at +most 5 steps. "A return to the foreground" is every Activity resume, also +after a permission dialog or a picker, so the library does this at most +once every 30 s. The app needs no code for this. Entries now save the cause +and time of their last failure; entries saved by an older version read as +unknown. Android: the boot sweep, `resume()`, a return to the foreground, `updateHeaders()` and a same-id `mutate()` now replace a WorkManager row that WorkManager holds back after a system stop, so the entry can start at -once. Before, the library added no row while that stopped row waited, and -the entry waited for WorkManager's own backoff (up to 5 hours). The library -replaces the row only when no run of the entry is in progress. - -Android: a transport failure (no HTTP response) now waits at most 15 -minutes. Every attempt starts on a network that Android reports as -validated. HTTP failures keep the retry policy's `maxMs` (2 hours by -default). - -Android: a chunked entry uses its stored backoff only until a part is -accepted in the run. After that each part counts its own failures from 0. -Before, one failed part after a long backoff sent the whole entry back for -the next, longer wait. +once. This applies to every queued entry with no backoff wait, whatever its +last failure: it only makes sure a row exists. Before, the library added no +row while that stopped row waited, and the entry waited for WorkManager's +own backoff (up to 5 hours). The library replaces the row only when no row +of the entry's main chain runs, and keeps a row that WorkManager starts +within 30 s. It makes the new row before it cancels the old wake. + +Android: a failure in the connect phase now waits at most 15 minutes. That +is a failure that proves the request never reached the server: +`ConnectException`, `UnknownHostException`, `NoRouteToHostException`, +`PortUnreachableException`, `SSLHandshakeException`, +`SSLPeerUnverifiedException`. Every attempt starts on a network that +Android reports as validated. Every other failure, a timeout included, +keeps the retry policy's `maxMs` (2 hours by default), because the server +may have the request. + +Android: in a chunked entry, the part whose failures set the stored backoff +goes on from it in the next run, and only that part's acceptance resets it. +The other parts count their own failures from 0. Before, every part +started at the stored backoff, so one failed part after a long backoff +sent the whole entry back for the next, longer wait. ## 11.3.0 diff --git a/README.md b/README.md index a08767fa..8609f62f 100644 --- a/README.md +++ b/README.md @@ -220,15 +220,20 @@ at the next run, after a growing backoff. A body that needs more than 10 minutes headless cannot finish that way. Use `parts` for large bodies: accepted parts are kept across runs. iOS has no equal limit. -**Backoff and the app.** When the app starts, comes to the foreground, or -calls `resume()`, the library ends the retry backoff of every queued entry -(for `resume({ keys })`, of the entries of those keys) and runs it at once. -The app needs no code for this. The entry keeps at most 5 backoff steps, so -if the server still fails, the next wait is about 32 s and then doubles. The -library also replaces a WorkManager row that WorkManager holds back after a -system stop, so the entry can start at once. A chunked entry uses its stored -backoff only until a part is accepted in the run; after that each part -counts its own failures from zero. +**Backoff and the app.** When the app starts, calls `resume()`, or comes +to the foreground (every Activity resume, at most once every 30 s), the +library ends the retry backoff of a queued entry and runs it at once, when +two things hold: the failure behind the wait had no server response (or an +older version saved the entry), and it happened 30 s or more ago. A wait +after a server response, such as a 503 or a status a `transient` rule +retries, keeps its time. `resume({ keys })` affects those keys only. The +app needs no code for this. The entry keeps at most 5 backoff steps, so if +it fails again the next wait is 32 times `baseMs` (32 s at the default +1 s), then it doubles. For every queued entry with no wait, the library +also replaces a WorkManager row that WorkManager holds back after a system +stop, so the entry can start at once. In a chunked entry, the part whose +failures set the backoff goes on from it; the other parts count their own +failures from zero. **Backups.** The queue store (`files/rnbgupload-chunked/`) and the journal (`files/rnbgupload-settled/`) hold request headers, including auth tokens, @@ -353,7 +358,7 @@ Retry classes: | Attempt outcome | Action | | --- | --- | | 2xx, or an `accept` rule matches | settle `completed` | -| network failure, 5xx, 408, 429 | exponential backoff (base 1 s, max 2 h, jitter 0.2) until `expiresAt`. On Android a network failure (no HTTP response) waits at most 15 minutes | +| network failure, 5xx, 408, 429 | exponential backoff (base 1 s, max 2 h, jitter 0.2) until `expiresAt`. On Android a failure in the connect phase (the request never reached the server) waits at most 15 minutes | | 401, 403 | park as `awaiting-auth`; resume on `updateHeaders()` | | other 4xx matching a `retry.transient` rule (`status`, optional `bodyIncludes`), under the rule's `maxRetries` | as transient. The first matching rule in the list wins. `maxRetries: 3` retries three matching responses; the fourth settles. No `maxRetries` means no cap. See "Transient rule counts" below. A rule for 401, 403, 408 or 429 is rejected at the call | | other 4xx in `retry.terminalHttp.exempt` (default `[404]`), when no rule matched | as transient, with no cap | @@ -484,9 +489,9 @@ event (`paused`, then `queued`). No outcome is produced and the bytes are kept. A paused entry past `expiresAt` settles `error` with `errorKind: 'expired'` when it is resumed. -On Android, `resume()` also ends the retry backoff of the queued entries in -its scope, and they run at once. A resumed entry starts its backoff over. -See "Backoff and the app" under Android platform notes. +On Android, `resume()` also ends some retry backoff waits of the queued +entries in its scope, and they run at once. A resumed entry keeps at most 5 +backoff steps. See "Backoff and the app" under Android platform notes. Neither call rejects because the settings cannot be saved. On Android, either call can still reject `E_STORAGE` when an entry file cannot be saved. From 90803b23fe366bc38f8a8b9a89ad2b1130cf1f8d Mon Sep 17 00:00:00 2001 From: Dylan Murphy Date: Tue, 17 Nov 2026 18:38:49 -0500 Subject: [PATCH 05/10] Test the main-chain step table and the 30 s start margin Co-Authored-By: Claude Opus 5.5 --- .../backgroundupload/MainChainTest.kt | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/android/src/test/java/ai/openspace/backgroundupload/MainChainTest.kt b/android/src/test/java/ai/openspace/backgroundupload/MainChainTest.kt index 8f9aab8d..f7ecba8a 100644 --- a/android/src/test/java/ai/openspace/backgroundupload/MainChainTest.kt +++ b/android/src/test/java/ai/openspace/backgroundupload/MainChainTest.kt @@ -174,6 +174,31 @@ class MainChainTest { assertEquals(1, startable().size) } + @Test + fun `a row due in 20 s is kept`() { + store.save(entry(state = EntryState.QUEUED)) + val soon = work.add("e1", runAttemptCount = 1, lastEnqueueTime = work.now - 10_000) + controller.resume() + assertEquals(listOf(soon), work.rows) + } + + @Test + fun `the main-chain step for each set of rows`() { + val now = work.now + fun row(state: androidx.work.WorkInfo.State, next: Long = Long.MAX_VALUE) = WorkRow("e1", state, 1, next, false) + val due = row(ENQUEUED, now) + val held = row(ENQUEUED, now + 60 * minute) + assertEquals(MainChainStep.APPEND, mainChainStep(emptyList(), now)) + assertEquals(MainChainStep.APPEND, mainChainStep(listOf(row(RUNNING)), now)) + assertEquals(MainChainStep.NONE, mainChainStep(listOf(row(RUNNING), row(BLOCKED)), now)) + // A held-back row is not "the row after the run in progress". + assertEquals(MainChainStep.APPEND, mainChainStep(listOf(row(RUNNING), held), now)) + assertEquals(MainChainStep.NONE, mainChainStep(listOf(due), now)) + assertEquals(MainChainStep.REPLACE, mainChainStep(listOf(held), now)) + assertEquals(MainChainStep.REPLACE, mainChainStep(listOf(held, row(BLOCKED)), now)) + assertEquals(MainChainStep.APPEND, mainChainStep(listOf(row(CANCELLED)), now)) + } + @Test fun `a running row is never replaced`() { store.save(entry(state = EntryState.QUEUED)) From 29670c090e6c5cf43fd724d33221b4a06b44360f Mon Sep 17 00:00:00 2001 From: Dylan Murphy Date: Tue, 17 Nov 2026 18:43:47 -0500 Subject: [PATCH 06/10] Test the part-streak reset and that retryNow writes only what it changes Co-Authored-By: Claude Opus 5.5 --- .../backgroundupload/QueueControllerTest.kt | 9 +++++++++ .../openspace/backgroundupload/WorkerOpsTest.kt | 15 +++++++++++++++ 2 files changed, 24 insertions(+) diff --git a/android/src/test/java/ai/openspace/backgroundupload/QueueControllerTest.kt b/android/src/test/java/ai/openspace/backgroundupload/QueueControllerTest.kt index 4629fee5..4dddf72b 100644 --- a/android/src/test/java/ai/openspace/backgroundupload/QueueControllerTest.kt +++ b/android/src/test/java/ai/openspace/backgroundupload/QueueControllerTest.kt @@ -1308,6 +1308,15 @@ class QueueControllerTest { assertEquals(emptyList(), scheduler.scheduled) } + @Test + fun `retryNow writes no entry file whose wait stays`() { + store.save(waitAfter("r", WaitCause.RESPONSE, age = 3_600_000)) + store.save(waitAfter("y", WaitCause.NO_RESPONSE, age = 1_000)) + val files = listOf("r", "y").map { File(store.entryDir(it), QueueStore.ENTRY_FILE).apply { setLastModified(1_000) } } + controller.retryNow() + assertEquals(listOf(1_000L, 1_000L), files.map { it.lastModified() }) + } + @Test fun `a transport-failure wait 30 s or older is cleared by each trigger`() { eachTrigger({ store.save(waitAfter("e1", WaitCause.NO_RESPONSE, age = 30_000)) }) { trigger -> diff --git a/android/src/test/java/ai/openspace/backgroundupload/WorkerOpsTest.kt b/android/src/test/java/ai/openspace/backgroundupload/WorkerOpsTest.kt index 4a3c2eea..d737c6e3 100644 --- a/android/src/test/java/ai/openspace/backgroundupload/WorkerOpsTest.kt +++ b/android/src/test/java/ai/openspace/backgroundupload/WorkerOpsTest.kt @@ -393,6 +393,21 @@ class WorkerOpsTest { assertEquals(0, e.backoffStreak) } + @Test + fun `an accepted part resets the stored streak only when that part set it`() { + store.save(entry( + state = EntryState.RUNNING, + descriptor = desc(url = null, file = "/f", parts = listOf(part(0, 10), part(10, 25))), + body = StagedBody(StagedBody.CHUNKED, "blob", null, 25), + ).copy(backoffStreak = 9, backoffPartStart = 10)) + ops.markAccepted("e1", 1, 0) + assertEquals(9, store.load("e1")!!.backoffStreak) + assertEquals(10L, store.load("e1")!!.backoffPartStart) + ops.markAccepted("e1", 1, 1) + assertEquals(0, store.load("e1")!!.backoffStreak) + assertEquals(null, store.load("e1")!!.backoffPartStart) + } + // MARK: - header generation of an attempt @Test From a12af5eb14b6e3dbe80c2625cbd75aeb779ea4c1 Mon Sep 17 00:00:00 2001 From: Dylan Murphy Date: Tue, 17 Nov 2026 19:03:17 -0500 Subject: [PATCH 07/10] Android clear-wait: keep a needed wake, part takeover, pause keeps the wait From the delta review of this branch: - cancelStaleWake reads the stored entry again and keeps a wake for a queued entry whose wait is still longer than a worker sleeps out. A stale snapshot could cancel the wake a worker made meanwhile (P7). A return to the foreground also makes the wake again (KEEP) for a queued entry whose wait stays and that has no unfinished row. rowSummary() replaces startableIds(): one query gives both sets. - The chunked part that releases a run takes the stored streak with its own count, so a slower part's higher streak does not hide a part that fails every time (P8). - reachedNoServer also needs no suppressed exception: OkHttp attaches the earlier tries of a call as suppressed, and one may have sent the body (P10). - The foreground throttle uses SystemClock.elapsedRealtime. - pause() keeps the wait with its cause; resume() gives back what is left and ends it only under the clear rule (P9). A paused row does not show nextAttemptAt. - executePart takes the part's start from the start entry, with no store read. Co-Authored-By: Claude Opus 5.5 --- .../backgroundupload/ChunkedUploadWorker.kt | 11 ++-- .../ai/openspace/backgroundupload/EntryRun.kt | 6 +- .../backgroundupload/EntryTransitions.kt | 46 +++++++------ .../backgroundupload/ForegroundTask.kt | 9 ++- .../backgroundupload/QueueController.kt | 31 +++++++-- .../openspace/backgroundupload/QueueEntry.kt | 4 +- .../backgroundupload/RetryClassifier.kt | 53 ++++++++------- .../openspace/backgroundupload/Scheduler.kt | 24 ++++--- .../backgroundupload/EntryRunTest.kt | 57 ++++++++++++++++ .../backgroundupload/EntryTransitionsTest.kt | 23 +++++-- .../backgroundupload/ForegroundTaskTest.kt | 4 +- .../backgroundupload/MainChainTest.kt | 66 +++++++++++++++++-- .../backgroundupload/QueueControllerTest.kt | 15 +++++ .../backgroundupload/RetryClassifierTest.kt | 9 +++ .../openspace/backgroundupload/TestSupport.kt | 10 +-- 15 files changed, 280 insertions(+), 88 deletions(-) diff --git a/android/src/main/java/ai/openspace/backgroundupload/ChunkedUploadWorker.kt b/android/src/main/java/ai/openspace/backgroundupload/ChunkedUploadWorker.kt index dd87cd44..81c7e6b0 100644 --- a/android/src/main/java/ai/openspace/backgroundupload/ChunkedUploadWorker.kt +++ b/android/src/main/java/ai/openspace/backgroundupload/ChunkedUploadWorker.kt @@ -50,7 +50,9 @@ internal class ChunkedTransfer(private val run: EntryRun) { run.progressStarted(total, acceptedBytes.get()) try { - ChunkedEngine.run(pending) { index -> executePart(index, blob, start.backoffStreak, start.backoffPartStart) } + ChunkedEngine.run(pending) { index -> + executePart(index, blob, start.backoffStreak, start.backoffPartStart, parts[index].start) + } } catch (failed: PartFailed) { return failed.settlement } @@ -75,10 +77,11 @@ internal class ChunkedTransfer(private val run: EntryRun) { * other parts count their own failures from 0, so an old streak does not * slow the parts that work. When no part is named (an older version, or a * stop that cut the run), every part uses the stored streak until a part - * is accepted in this run. + * is accepted in this run. [partStart] is the part's start from the + * entry the run began with: within one generation the parts keep their + * ranges and order. */ - internal suspend fun executePart(index: Int, blob: File, initialStreak: Int, streakPart: Long?) { - val partStart = run.ops.latest(run.entryId, run.generation).descriptor!!.parts!![index].start + internal suspend fun executePart(index: Int, blob: File, initialStreak: Int, streakPart: Long?, partStart: Long) { var base = if (streakPart == null || streakPart == partStart) initialStreak else 0 var failures = 0 while (true) { diff --git a/android/src/main/java/ai/openspace/backgroundupload/EntryRun.kt b/android/src/main/java/ai/openspace/backgroundupload/EntryRun.kt index 3d3afdca..fff032ec 100644 --- a/android/src/main/java/ai/openspace/backgroundupload/EntryRun.kt +++ b/android/src/main/java/ai/openspace/backgroundupload/EntryRun.kt @@ -372,9 +372,9 @@ internal class EntryRun( val result = if (verdict is RetryClassifier.Verdict.Terminal) { AttemptResult.Terminal(verdict.errorKind, verdict.message, null) } else AttemptResult.Transient - // The attempt started on a validated network (waitForNetwork). A - // failure that proves the request never reached the server waits at - // most 15 minutes; any other keeps the entry's maximum. + // The attempt started on a validated network (waitForNetwork). A call + // that failed while it connected, with no earlier try in the call, + // waits at most 15 minutes; any other failure keeps the entry's maximum. val waitPolicy = if (RetryClassifier.reachedNoServer(error)) RetryClassifier.transportFailurePolicy(policy) else policy return Attempt(entry, url, waitPolicy, result, WaitCause.NO_RESPONSE) } diff --git a/android/src/main/java/ai/openspace/backgroundupload/EntryTransitions.kt b/android/src/main/java/ai/openspace/backgroundupload/EntryTransitions.kt index 11550467..76979e4c 100644 --- a/android/src/main/java/ai/openspace/backgroundupload/EntryTransitions.kt +++ b/android/src/main/java/ai/openspace/backgroundupload/EntryTransitions.kt @@ -58,10 +58,11 @@ object EntryTransitions { /** * A backoff too long to wait inside the worker. The streak is kept so the - * next wait keeps growing, with the failure's [cause] and time. A chunked - * part ([partStart] set) keeps a higher streak another part holds, so a - * part that keeps failing does not start over because a sibling released - * the run. + * next wait keeps growing, with the failure's [cause] and time. For a + * chunked entry the part that released the run ([partStart]) takes the + * streak with its own count, also over a higher one another part held: + * that part goes on from it in the next run, and a part that only was in + * flight, or a slower one, does not hide a part that fails every time. */ fun toReleased( e: QueueEntry, @@ -70,18 +71,15 @@ object EntryTransitions { now: Long, cause: String? = null, partStart: Long? = null, - ): QueueEntry { - val keepOther = partStart != null && e.backoffStreak > streak - return e.copy( - state = EntryState.QUEUED, - nextAttemptAt = nextAttemptAt, - backoffStreak = if (keepOther) e.backoffStreak else streak, - backoffPartStart = if (keepOther) e.backoffPartStart else partStart, - waitCause = cause, - failedAt = now, - updatedAt = now, - ) - } + ) = e.copy( + state = EntryState.QUEUED, + nextAttemptAt = nextAttemptAt, + backoffStreak = streak, + backoffPartStart = partStart, + waitCause = cause, + failedAt = now, + updatedAt = now, + ) fun toSettled(e: QueueEntry, state: EntryState, eventId: String, bytesSent: Long, now: Long) = e.copy( state = state, @@ -94,19 +92,23 @@ object EntryTransitions { /** * pause(). parkedGeneration is kept so resume() can return the entry to - * awaiting-auth. nextAttemptAt is cleared: resume() retries at once. + * awaiting-auth. A backoff wait is kept with its cause and time: + * resume() ends it only as [clearsWait] allows, so a pause and resume + * cannot spend a transient rule's retries. A paused row does not show it. */ fun toPaused(e: QueueEntry, now: Long) = - e.copy(state = EntryState.PAUSED, nextAttemptAt = null, updatedAt = now) + e.copy(state = EntryState.PAUSED, updatedAt = now) /** * resume(): back to awaiting-auth only when no updateHeaders() came in * between. It compares the entry's own headerGeneration, which * updateHeaders() saves with the new headers. The settings value can be * older after a settings write that never landed. Back to queued, the - * streak keeps at most [RetryClassifier.RETRY_NOW_MAX_STREAK] steps, as - * after a cleared wait: a pause and resume in normal work must not give a - * failing entry a burst of attempts. + * entry keeps what is left of its wait (resume() then applies + * [clearsWait]), and the streak keeps at most + * [RetryClassifier.RETRY_NOW_MAX_STREAK] steps, as after a cleared wait: + * a pause and resume in normal work must not give a failing entry a + * burst of attempts. */ fun toResumed(e: QueueEntry, now: Long): QueueEntry = if (e.parkedGeneration != null && e.parkedGeneration == e.headerGeneration) { @@ -116,6 +118,8 @@ object EntryTransitions { state = EntryState.QUEUED, parkedGeneration = null, backoffStreak = min(e.backoffStreak, RetryClassifier.RETRY_NOW_MAX_STREAK), + // The wait kept over the pause; one that ended during it is gone. + nextAttemptAt = e.nextAttemptAt?.takeIf { it > now }, updatedAt = now, ) } diff --git a/android/src/main/java/ai/openspace/backgroundupload/ForegroundTask.kt b/android/src/main/java/ai/openspace/backgroundupload/ForegroundTask.kt index 77de255d..01313f70 100644 --- a/android/src/main/java/ai/openspace/backgroundupload/ForegroundTask.kt +++ b/android/src/main/java/ai/openspace/backgroundupload/ForegroundTask.kt @@ -1,5 +1,6 @@ package ai.openspace.backgroundupload +import android.os.SystemClock import java.util.concurrent.Executor /** @@ -11,13 +12,15 @@ import java.util.concurrent.Executor * * The host's resume event fires on every Activity resume, also after a * permission dialog, a picker or a share sheet. So [retryNow] runs at most - * once per [MIN_INTERVAL_MS]; the settings save runs every time. + * once per [MIN_INTERVAL_MS]; the settings save runs every time. The + * interval is measured on [elapsedClock], a monotonic clock + * (SystemClock.elapsedRealtime), so a wall clock set back cannot stop it. */ internal class ForegroundTask( private val executor: Executor, private val saveSettings: () -> Unit, private val retryNow: () -> Unit, - private val clock: () -> Long = System::currentTimeMillis, + private val elapsedClock: () -> Long = SystemClock::elapsedRealtime, ) { companion object { const val MIN_INTERVAL_MS = 30_000L @@ -29,7 +32,7 @@ internal class ForegroundTask( fun onForeground() { executor.execute { runCatching(saveSettings).onFailure { Diag.error("could not save the settings on return to the foreground", it) } - val now = clock() + val now = elapsedClock() val last = lastRetryAt if (last == null || now - last >= MIN_INTERVAL_MS) { lastRetryAt = now diff --git a/android/src/main/java/ai/openspace/backgroundupload/QueueController.kt b/android/src/main/java/ai/openspace/backgroundupload/QueueController.kt index a99921f7..01283bf0 100644 --- a/android/src/main/java/ai/openspace/backgroundupload/QueueController.kt +++ b/android/src/main/java/ai/openspace/backgroundupload/QueueController.kt @@ -225,7 +225,10 @@ class QueueController( val now = clock() val result = transformAll { e -> when { - e.state == EntryState.PAUSED && !s.isPaused(e.key) -> EntryTransitions.toResumed(e, now) + e.state == EntryState.PAUSED && !s.isPaused(e.key) -> EntryTransitions.toResumed(e, now).let { + // The wait kept over the pause follows the clear rule too. + if (EntryTransitions.clearsWait(it, now)) EntryTransitions.toRetryNow(it, now) else it + } inScope(e, keys) && EntryTransitions.clearsWait(e, now) -> EntryTransitions.toRetryNow(e, now) else -> e } @@ -244,9 +247,10 @@ class QueueController( * ([EntryTransitions.toRetryNow]) and is scheduled. A wait after a server * response keeps its time. A queued entry with no wait, whatever its last * failure, is scheduled only when no row of it can start now - * ([WorkScheduler.startableIds], one query): that is not a retry, it only + * ([WorkScheduler.rowSummary], one query): that is not a retry, it only * makes sure a row exists, and a row WorkManager holds back after a stop - * is replaced ([WorkScheduler.schedule]). Running, paused, parked and + * is replaced ([WorkScheduler.schedule]). A queued entry whose wait stays + * and that has no unfinished row at all gets its wake again (KEEP). Running, paused, parked and * settled entries are left alone. A queued entry sends nothing, and the * per-id gate holds any new row while a worker of the entry still winds * down, so this never starts a second transfer next to one in progress. @@ -282,11 +286,16 @@ class QueueController( } } cleared.forEach { e -> runCatching { scheduleRun(e) }.onFailure(::failed) } + val clearedIds = cleared.map { it.id }.toSet() val idle = queued.filter { it.nextAttemptAt == null } - if (idle.isNotEmpty()) { - val startable = runCatching { scheduler.startableIds() }.onFailure(::failed).getOrNull() - idle.filter { startable == null || it.id !in startable } + val waiting = queued.filter { it.nextAttemptAt != null && it.id !in clearedIds } + if (idle.isNotEmpty() || waiting.isNotEmpty()) { + val rows = runCatching { scheduler.rowSummary() }.onFailure(::failed).getOrNull() + idle.filter { rows == null || it.id !in rows.startable } .forEach { e -> runCatching { scheduleRun(e) }.onFailure(::failed) } + // A wait that stays needs its wake. KEEP leaves one that exists. + waiting.filter { rows == null || it.id !in rows.withRow } + .forEach { e -> runCatching { scheduleRun(e, keepWake = true) }.onFailure(::failed) } } cleared.forEach { events.state(it.toRow()) } first?.let { Diag.error("retryNow: $failures entry saves or WorkManager calls failed; the next app start or foreground tries again", it) } @@ -651,8 +660,16 @@ class QueueController( // A running wake is the entry's worker; it finishes its own run. // [isWorkerRunning] reads the in-process gate, so it assumes WorkManager // runs workers in the app's process. The library sets no android:process. + // + // The caller's entry can be a stale snapshot: a worker may have released + // the entry with a long wait and made its wake since. So the stored entry + // is read again, and a wake it still needs is kept. private fun cancelStaleWake(id: String) { - if (!isWorkerRunning(id)) scheduler.cancelWake(id) + if (isWorkerRunning(id)) return + val stored = runCatching { store.load(id) }.getOrNull() + val at = stored?.nextAttemptAt + if (stored?.state == EntryState.QUEUED && at != null && at - clock() > RetryClassifier.IN_WORKER_BACKOFF_MAX_MS) return + scheduler.cancelWake(id) } /** The entries [transformAll] saved, and the saves that failed. */ diff --git a/android/src/main/java/ai/openspace/backgroundupload/QueueEntry.kt b/android/src/main/java/ai/openspace/backgroundupload/QueueEntry.kt index e9003bb4..ed149f51 100644 --- a/android/src/main/java/ai/openspace/backgroundupload/QueueEntry.kt +++ b/android/src/main/java/ai/openspace/backgroundupload/QueueEntry.kt @@ -165,7 +165,9 @@ data class QueueEntry( totalBytes = totalBytes, attempts = attempts, updatedAt = updatedAt, - nextAttemptAt = nextAttemptAt, + // A paused entry keeps its wait for resume(), but the row does not + // show it: the entry waits for resume(), not for that time. + nextAttemptAt = nextAttemptAt.takeIf { state == EntryState.QUEUED || state == EntryState.RUNNING }, createdAt = createdAt, ) diff --git a/android/src/main/java/ai/openspace/backgroundupload/RetryClassifier.kt b/android/src/main/java/ai/openspace/backgroundupload/RetryClassifier.kt index 2ce46170..552bd3c1 100644 --- a/android/src/main/java/ai/openspace/backgroundupload/RetryClassifier.kt +++ b/android/src/main/java/ai/openspace/backgroundupload/RetryClassifier.kt @@ -41,12 +41,13 @@ object RetryClassifier { const val IN_WORKER_BACKOFF_MAX_MS = 30_000L /** - * The longest wait after a failure in the connect phase - * ([reachedNoServer]). Every attempt starts on a network Android reports - * as validated, so such a failure says little about the server, and a - * network change cannot wake a queued entry. Every other failure keeps - * the policy's maxMs: a failure after the request went out may have - * reached the server, and a sooner repeat would make a duplicate likelier. + * The longest wait after a call that failed while it connected, with no + * earlier try in the call ([reachedNoServer]). Every attempt starts on a + * network Android reports as validated, so such a failure says little + * about the server, and a network change cannot wake a queued entry. + * Every other failure keeps the policy's maxMs: a failure after the + * request went out may have reached the server, and a sooner repeat + * would make a duplicate likelier. */ const val TRANSPORT_FAILURE_MAX_MS = 15L * 60 * 1000 @@ -54,24 +55,32 @@ object RetryClassifier { const val CLEAR_MIN_AGE_MS = 30_000L /** - * Whether [error] proves the request never reached the server: it failed - * before a connection was made or secured. From OkHttp 4.12.0: - * RealConnection.connectSocket throws ConnectException ("Failed to - * connect to"); the DNS lookup (Dns.SYSTEM) throws UnknownHostException; - * the socket connect can throw NoRouteToHostException or - * PortUnreachableException; the TLS handshake throws SSLHandshakeException, - * and connectTls throws SSLPeerUnverifiedException. RetryAndFollowUpInterceptor - * rethrows the first of these from a RouteException. A - * SocketTimeoutException is left out: a connect timeout and a read or - * write timeout throw the same class. + * Whether this call failed while it connected, with no earlier try inside + * the same call. Then this call sent no request. (An earlier attempt, in + * an earlier call, may still have reached the server.) + * + * From OkHttp 4.12.0: RealConnection.connectSocket throws ConnectException + * ("Failed to connect to"); the DNS lookup (Dns.SYSTEM) throws + * UnknownHostException; the socket connect can throw + * NoRouteToHostException or PortUnreachableException; the TLS handshake + * throws SSLHandshakeException, and connectTls throws + * SSLPeerUnverifiedException. OkHttp retries inside one call: it throws + * the last failure with the earlier tries it recovered from attached as + * suppressed (Util.withSuppressed in RetryAndFollowUpInterceptor), and a + * RouteException adds later connect failures as suppressed too. An + * earlier try may have sent the body, so any suppressed exception rules + * the failure out. A SocketTimeoutException is left out: a connect + * timeout and a read or write timeout throw the same class. */ fun reachedNoServer(error: Throwable): Boolean = - error is java.net.ConnectException || - error is java.net.UnknownHostException || - error is java.net.NoRouteToHostException || - error is java.net.PortUnreachableException || - error is javax.net.ssl.SSLHandshakeException || - error is javax.net.ssl.SSLPeerUnverifiedException + error.suppressed.isEmpty() && ( + error is java.net.ConnectException || + error is java.net.UnknownHostException || + error is java.net.NoRouteToHostException || + error is java.net.PortUnreachableException || + error is javax.net.ssl.SSLHandshakeException || + error is javax.net.ssl.SSLPeerUnverifiedException + ) /** * The highest backoff streak an entry keeps when the app clears its wait. diff --git a/android/src/main/java/ai/openspace/backgroundupload/Scheduler.kt b/android/src/main/java/ai/openspace/backgroundupload/Scheduler.kt index 4ba7bef2..aec92e8b 100644 --- a/android/src/main/java/ai/openspace/backgroundupload/Scheduler.kt +++ b/android/src/main/java/ai/openspace/backgroundupload/Scheduler.kt @@ -123,13 +123,16 @@ interface WorkScheduler { /** Every unfinished WorkManager run of the library, main chains and wakes. For the boot snapshot. */ fun libraryRuns(): List - /** - * The entry ids that have a run in progress, or a row WorkManager may - * start now. One query for the whole library. - */ - fun startableIds(): Set + /** The library's unfinished rows by entry id. One query for the whole library. */ + fun rowSummary(): RowSummary } +/** + * [startable]: the ids with a run in progress, or a row WorkManager may + * start now. [withRow]: the ids with any unfinished row, main chain or wake. + */ +data class RowSummary(val startable: Set, val withRow: Set) + /** * One unfinished WorkManager run. [entryId] comes from the run's id tag. * [delayed] is a run enqueued with an initial delay, which only a wake has. @@ -241,12 +244,13 @@ open class RowScheduler( ) } - override fun startableIds(): Set { + override fun rowSummary(): RowSummary { val now = clock() - return rows.library(listOf(WorkInfo.State.ENQUEUED, WorkInfo.State.RUNNING)) - .filter { it.state == WorkInfo.State.RUNNING || it.startsBy(now) } - .mapNotNull { it.entryId } - .toSet() + val open = rows.library(UNFINISHED) + return RowSummary( + startable = open.filter { it.state == WorkInfo.State.RUNNING || it.startsBy(now) }.mapNotNull { it.entryId }.toSet(), + withRow = open.mapNotNull { it.entryId }.toSet(), + ) } } diff --git a/android/src/test/java/ai/openspace/backgroundupload/EntryRunTest.kt b/android/src/test/java/ai/openspace/backgroundupload/EntryRunTest.kt index 00b96626..0fb8d86f 100644 --- a/android/src/test/java/ai/openspace/backgroundupload/EntryRunTest.kt +++ b/android/src/test/java/ai/openspace/backgroundupload/EntryRunTest.kt @@ -320,6 +320,29 @@ class EntryRunTest { assertEquals(1, e.transientHitsByRule!!.values.sum()) } + @Test + fun `pause and resume do not spend a transient rule's retries`() { + // Probe P9: before, five pause and resume cycles used all 5 retries + // with no time between. + controller.configureRetry( + RetryDefaults(baseMs = 60_000, jitter = 0.0, transient = listOf(UploadOutcome.TransientRule(409, maxRetries = 5))), + ) + controller.enqueue(parsed()) + host.handler = { _, _ -> UploadResponse(409, "not yet", mapOf()) } + run() + val at = store.load("e1")!!.nextAttemptAt + repeat(20) { + controller.pause() + controller.resume() + run() + } + val e = store.load("e1")!! + assertEquals(EntryState.QUEUED, e.state) + assertEquals(1, host.requests.size) + assertEquals(1, e.transientHitsByRule!!.values.sum()) + assertEquals(at, e.nextAttemptAt) + } + @Test fun `a terminal response settles error http with the response and the live bytes`() { controller.enqueue(parsed()) @@ -1170,6 +1193,40 @@ class EntryRunTest { assertEquals((6..13).toList(), streaks) } + @Test + fun `a part that always fails takes the streak over from a slower part`() { + // Probe P8: part 1 held streak 9 and is in flight for the whole run. + // Before, part 0 started from 0 in every run and released at step 6, + // and the higher streak of part 1 hid it: 48 attempts in 8 runs. + val n = 40 + val src = File(tmp.newFolder(), "video.bin").apply { writeBytes(ByteArray(n * 10) { it.toByte() }) } + controller.enqueue( + parsed(descriptor = desc(url = null, method = "PUT", file = src.path, parts = (0 until n).map { part(it * 10L, it * 10L + 10) })), + ) + store.save(store.load("e1")!!.copy(backoffStreak = 9, backoffPartStart = 10)) + var part0 = 0 + host.handler = { r, _ -> + val index = r.url.substringAfter("start=").toInt() / 10 + if (index == 0) { + kotlinx.coroutines.yield() + part0++ + UploadResponse(503, "", mapOf()) + } else { + awaitCancellation() + } + } + val streaks = mutableListOf() + repeat(8) { + run() + val e = store.load("e1")!! + streaks += e.backoffStreak + host.clock = e.nextAttemptAt ?: host.clock + } + assertEquals(6 + 7, part0) + assertEquals((6..13).toList(), streaks) + assertEquals(0L, store.load("e1")!!.backoffPartStart) + } + /** Four 10-byte parts. [code] answers each request by its part index and how many times that part was sent. */ private fun fourParts(code: (part: Int, sent: Int) -> Int) { val src = File(tmp.newFolder(), "video.bin").apply { writeBytes(ByteArray(40) { it.toByte() }) } diff --git a/android/src/test/java/ai/openspace/backgroundupload/EntryTransitionsTest.kt b/android/src/test/java/ai/openspace/backgroundupload/EntryTransitionsTest.kt index cc70e487..24a97fea 100644 --- a/android/src/test/java/ai/openspace/backgroundupload/EntryTransitionsTest.kt +++ b/android/src/test/java/ai/openspace/backgroundupload/EntryTransitionsTest.kt @@ -89,19 +89,32 @@ class EntryTransitionsTest { assertEquals(6, sibling.backoffStreak) assertEquals(0L, sibling.backoffPartStart) assertEquals(WaitCause.NO_RESPONSE, sibling.waitCause) + // The part that releases the run takes the streak with its own count. val released = EntryTransitions.toReleased(sibling, 40_000, 2, now = 7, cause = WaitCause.RESPONSE, partStart = 10) - assertEquals(6, released.backoffStreak) - assertEquals(0L, released.backoffPartStart) + assertEquals(2, released.backoffStreak) + assertEquals(10L, released.backoffPartStart) // A simple entry's release sets the streak it is given, as before. assertEquals(2, EntryTransitions.toReleased(chunked.copy(backoffStreak = 6), 40_000, 2, now = 7).backoffStreak) } @Test - fun `pause keeps the parked generation and clears the wake time`() { - val p = EntryTransitions.toPaused(entry(state = EntryState.AWAITING_AUTH, parkedGeneration = 2, nextAttemptAt = 50), 9) + fun `pause keeps the parked generation and the wait, and the paused row does not show the wait`() { + val waiting = entry(state = EntryState.QUEUED, nextAttemptAt = 500).copy(waitCause = WaitCause.RESPONSE, failedAt = 5) + val p = EntryTransitions.toPaused(waiting.copy(parkedGeneration = 2), 9) assertEquals(EntryState.PAUSED, p.state) assertEquals(2, p.parkedGeneration) - assertNull(p.nextAttemptAt) + assertEquals(500L, p.nextAttemptAt) + assertEquals(WaitCause.RESPONSE, p.waitCause) + assertEquals(5L, p.failedAt) + assertNull(p.toRow().nextAttemptAt) + assertEquals(500L, waiting.toRow().nextAttemptAt) + } + + @Test + fun `resume gives back what is left of the wait, and drops one that ended`() { + val paused = entry(state = EntryState.PAUSED, nextAttemptAt = 500) + assertEquals(500L, EntryTransitions.toResumed(paused, 9).nextAttemptAt) + assertNull(EntryTransitions.toResumed(paused, 500).nextAttemptAt) } @Test diff --git a/android/src/test/java/ai/openspace/backgroundupload/ForegroundTaskTest.kt b/android/src/test/java/ai/openspace/backgroundupload/ForegroundTaskTest.kt index 6d5c37eb..1d43e598 100644 --- a/android/src/test/java/ai/openspace/backgroundupload/ForegroundTaskTest.kt +++ b/android/src/test/java/ai/openspace/backgroundupload/ForegroundTaskTest.kt @@ -36,7 +36,7 @@ class ForegroundTaskTest { fun `a return to the foreground calls retryNow once, on the queue executor`() { val executor = HeldExecutor() val calls = mutableListOf() - val task = ForegroundTask(executor, { calls += "save" }, { calls += "retryNow" }) + val task = ForegroundTask(executor, { calls += "save" }, { calls += "retryNow" }) { 0L } task.onForeground() assertEquals(emptyList(), calls) // nothing on the calling (UI) thread executor.runAll() @@ -52,7 +52,7 @@ class ForegroundTaskTest { val task = ForegroundTask(executor, { throw IllegalStateException("settings") }, retryNow = { retried = true throw IllegalStateException("work") - }) + }, elapsedClock = { 0L }) task.onForeground() executor.runAll() assertEquals(true, retried) diff --git a/android/src/test/java/ai/openspace/backgroundupload/MainChainTest.kt b/android/src/test/java/ai/openspace/backgroundupload/MainChainTest.kt index f7ecba8a..5da7c87d 100644 --- a/android/src/test/java/ai/openspace/backgroundupload/MainChainTest.kt +++ b/android/src/test/java/ai/openspace/backgroundupload/MainChainTest.kt @@ -226,16 +226,18 @@ class MainChainTest { } @Test - fun `startableIds lists the entries with a run in progress or a row due now`() { + fun `rowSummary lists the entries that can start, and the entries with any unfinished row`() { work.add("due") work.add("held", runAttemptCount = 8, lastEnqueueTime = work.now - minute) work.add("run", state = RUNNING, runAttemptCount = 1) work.add("later#wake", initialDelay = minute) work.add("blocked", state = BLOCKED) work.add("gone", state = CANCELLED) - assertEquals(setOf("due", "run"), scheduler.startableIds()) - // One query, and it asks only for the states that can start. - assertEquals(listOf(listOf(ENQUEUED, RUNNING)), work.libraryQueries) + val rows = scheduler.rowSummary() + assertEquals(setOf("due", "run"), rows.startable) + assertEquals(setOf("due", "held", "run", "later", "blocked"), rows.withRow) + // One query, and it asks WorkManager for unfinished rows only. + assertEquals(listOf(listOf(ENQUEUED, RUNNING, BLOCKED)), work.libraryQueries) } @Test @@ -284,17 +286,69 @@ class MainChainTest { } @Test - fun `retryNow costs no save and no WorkManager query for entries whose wait stays`() { + fun `retryNow costs no save and one WorkManager query for entries whose wait stays`() { for (n in 1..3) { store.save(transportWait("r$n").copy(waitCause = WaitCause.RESPONSE)) + work.add("r$n#wake", initialDelay = 60 * minute) } controller.retryNow() assertEquals(0, work.nameQueries) - assertEquals(emptyList>(), work.libraryQueries) + assertEquals(1, work.libraryQueries.size) assertEquals(emptyList(), rows.log) assertEquals(emptyList(), events.log) // no save changed a row } + @Test + fun `a return to the foreground makes the wake again for a wait that stays and has no row`() { + store.save(transportWait().copy(waitCause = WaitCause.RESPONSE)) + controller.retryNow() + assertEquals(listOf("enqueue:e1#wake:KEEP"), rows.log) + assertEquals(listOf("e1#wake"), work.rows.filter { !it.state.isFinished }.map { it.name }) + // With a wake there, KEEP adds nothing. + controller.retryNow() + assertEquals(1, work.rows.size) + } + + @Test + fun `a stale snapshot does not cancel the wake a worker made meanwhile`() { + // Probe P7. retryNow read the entry queued with no wait. Then, before + // the wake cancel, a worker released it with a 60-minute wait and made + // its wake, and the fresh main row ran, kept that wake (ensureWake) and + // ended. + lateinit var ops: WorkerOps + var released = false + val racing = object : WorkRows by rows { + override fun library(states: List): List { + if (!released) { + released = true + val e = store.load("e1")!! + store.save(e.copy(state = EntryState.RUNNING)) + ops.release("e1", e.generation, work.now + 60 * minute, 7, cause = WaitCause.RESPONSE) + } + return work.library(states) + } + + override fun enqueue(name: String, policy: ExistingWorkPolicy, entry: QueueEntry, delayMs: Long) { + rows.enqueue(name, policy, entry, delayMs) + if (name == "e1") { + val row = work.rows.last() + row.state = RUNNING + val e = store.load("e1")!! + ops.ensureWake(e, e.nextAttemptAt!!) + row.state = androidx.work.WorkInfo.State.SUCCEEDED + } + } + } + val sched = RowScheduler(racing) { work.now } + ops = WorkerOps(store, journal, settings, events, sched) { work.now } + val c = QueueController(store, journal, settings, events, sched, { false }, { work.now }) + store.save(entry(state = EntryState.QUEUED)) + c.retryNow() + val e = store.load("e1")!! + assertEquals(work.now + 60 * minute, e.nextAttemptAt) + assertEquals(listOf("e1#wake"), work.rows.filter { !it.state.isFinished }.map { it.name }) + } + @Test fun `retryNow costs one save and two name queries per cleared entry, and one query for the idle ones`() { for (n in 1..3) store.save(transportWait("t$n")) diff --git a/android/src/test/java/ai/openspace/backgroundupload/QueueControllerTest.kt b/android/src/test/java/ai/openspace/backgroundupload/QueueControllerTest.kt index 4dddf72b..85c088e4 100644 --- a/android/src/test/java/ai/openspace/backgroundupload/QueueControllerTest.kt +++ b/android/src/test/java/ai/openspace/backgroundupload/QueueControllerTest.kt @@ -1308,6 +1308,21 @@ class QueueControllerTest { assertEquals(emptyList(), scheduler.scheduled) } + @Test + fun `pause and resume keep a no-response wait younger than 30 s, and clear an older one`() { + val at = now + 3_600_000 + store.save(waitAfter("e1", WaitCause.NO_RESPONSE, age = 1_000, at = at)) + controller.pause() + controller.resume() + assertEquals(at, store.load("e1")!!.nextAttemptAt) + assertEquals(listOf("e1" to at), scheduler.wakes) // it gets its wake back + controller.pause() + now += 30_000 + controller.resume() + assertNull(store.load("e1")!!.nextAttemptAt) + assertEquals(5, store.load("e1")!!.backoffStreak) + } + @Test fun `retryNow writes no entry file whose wait stays`() { store.save(waitAfter("r", WaitCause.RESPONSE, age = 3_600_000)) diff --git a/android/src/test/java/ai/openspace/backgroundupload/RetryClassifierTest.kt b/android/src/test/java/ai/openspace/backgroundupload/RetryClassifierTest.kt index 58923ebf..11acb893 100644 --- a/android/src/test/java/ai/openspace/backgroundupload/RetryClassifierTest.kt +++ b/android/src/test/java/ai/openspace/backgroundupload/RetryClassifierTest.kt @@ -198,6 +198,15 @@ class RetryClassifierTest { assertFalse(RetryClassifier.reachedNoServer(javax.net.ssl.SSLException("Read error"))) } + @Test + fun `a connect failure after an earlier try in the same call does not count`() { + // Probe P10. OkHttp attaches the earlier tries of a call as suppressed; + // the first may have sent the body on a pooled connection that died. + val error = java.net.ConnectException("Failed to connect to example.com") + error.addSuppressed(IOException("unexpected end of stream")) + assertFalse(RetryClassifier.reachedNoServer(error)) + } + @Test fun `only a connect failure caps the policy at 15 minutes`() { val policy = RetryClassifier.Policy(1_000, 7_200_000, 0.0, listOf(404)) diff --git a/android/src/test/java/ai/openspace/backgroundupload/TestSupport.kt b/android/src/test/java/ai/openspace/backgroundupload/TestSupport.kt index a7ac4e34..1c1138ed 100644 --- a/android/src/test/java/ai/openspace/backgroundupload/TestSupport.kt +++ b/android/src/test/java/ai/openspace/backgroundupload/TestSupport.kt @@ -122,9 +122,11 @@ internal class FakeScheduler : WorkScheduler { val cancelledWakes = mutableListOf() /** What libraryRuns() returns. */ val runs = mutableListOf() - /** What startableIds() returns. */ + /** What rowSummary() returns as startable. */ val startable = mutableSetOf() - /** How many startableIds() calls were made. */ + /** What rowSummary() returns as withRow, besides the startable ids. */ + val withRow = mutableSetOf() + /** How many rowSummary() calls were made. */ var startableQueries = 0 override fun schedule(entry: QueueEntry) { @@ -150,9 +152,9 @@ internal class FakeScheduler : WorkScheduler { override fun libraryRuns(): List = runs.toList() - override fun startableIds(): Set { + override fun rowSummary(): RowSummary { startableQueries++ - return startable.toSet() + return RowSummary(startable.toSet(), startable + withRow) } } From ce2428b0f69ff4fe010e7ba4fe98d4dacd2011c1 Mon Sep 17 00:00:00 2001 From: Dylan Murphy Date: Tue, 17 Nov 2026 19:03:22 -0500 Subject: [PATCH 08/10] Docs: pause keeps the wait, part takeover, and what the 15-minute cap covers Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 57 ++++++++++++++++++++++++++++++++-------------------- README.md | 8 +++++--- 2 files changed, 40 insertions(+), 25 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d0cbda99..2b6e3ce9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -60,12 +60,19 @@ would spend a rule's `maxRetries` faster. Before, an entry on a long backoff (up to 2 hours) kept its wait when the app opened, and `resume()` kept it too. `resume({ keys })` affects those keys only. The entry keeps at most 5 backoff steps, so if the request fails again, the next wait is 32 -times `baseMs` (32 s at the default 1 s base), and then it doubles. A paused entry that `resume()` moves back to `queued` also keeps at -most 5 steps. "A return to the foreground" is every Activity resume, also -after a permission dialog or a picker, so the library does this at most -once every 30 s. The app needs no code for this. Entries now save the cause -and time of their last failure; entries saved by an older version read as -unknown. +times `baseMs` (32 s at the default 1 s base), and then it doubles. "A +return to the foreground" is every Activity resume, also after a +permission dialog or a picker, so the library does this at most once every +30 s, measured on a clock that does not change with the wall clock. The +app needs no code for this. Entries now save the cause and time of their +last failure; entries saved by an older version read as unknown. + +**Behavior change:** on Android, `pause()` now keeps a backoff wait with +its cause. `resume()` gives the entry back what is left of it, ended only +by the rule above. Before, `pause()` cleared the wait, so a pause and a +resume sent the request again at once, also after a server response. A +paused row still does not show `nextAttemptAt`. A paused entry that +`resume()` moves back to `queued` keeps at most 5 backoff steps. Android: the boot sweep, `resume()`, a return to the foreground, `updateHeaders()` and a same-id `mutate()` now replace a WorkManager row @@ -75,22 +82,28 @@ last failure: it only makes sure a row exists. Before, the library added no row while that stopped row waited, and the entry waited for WorkManager's own backoff (up to 5 hours). The library replaces the row only when no row of the entry's main chain runs, and keeps a row that WorkManager starts -within 30 s. It makes the new row before it cancels the old wake. - -Android: a failure in the connect phase now waits at most 15 minutes. That -is a failure that proves the request never reached the server: -`ConnectException`, `UnknownHostException`, `NoRouteToHostException`, -`PortUnreachableException`, `SSLHandshakeException`, -`SSLPeerUnverifiedException`. Every attempt starts on a network that -Android reports as validated. Every other failure, a timeout included, -keeps the retry policy's `maxMs` (2 hours by default), because the server -may have the request. - -Android: in a chunked entry, the part whose failures set the stored backoff -goes on from it in the next run, and only that part's acceptance resets it. -The other parts count their own failures from 0. Before, every part -started at the stored backoff, so one failed part after a long backoff -sent the whole entry back for the next, longer wait. +within 30 s. It makes the new row before it cancels the old wake, and it +keeps a wake that the stored entry still needs. A return to the foreground +also makes the wake again (KEEP) for a queued entry whose wait stays and +that has no WorkManager row at all. + +Android: a call that failed while it connected, with no earlier try inside +the same call, now waits at most 15 minutes. Such a call sent no request. +The failures are `ConnectException`, `UnknownHostException`, +`NoRouteToHostException`, `PortUnreachableException`, +`SSLHandshakeException` and `SSLPeerUnverifiedException`, with no +suppressed exception (OkHttp attaches the earlier tries of a call as +suppressed, and an earlier try may have sent the body). Every attempt +starts on a network that Android reports as validated. Every other +failure, a timeout included, keeps the retry policy's `maxMs` (2 hours by +default), because the server may have the request. + +Android: in a chunked entry, the part that releases a run for a long wait +takes the stored backoff with its own count. It goes on from it in the +next run, and only that part's acceptance resets it. The other parts count +their own failures from 0. Before, every part started at the stored +backoff, so one failed part after a long backoff sent the whole entry back +for the next, longer wait. ## 11.3.0 diff --git a/README.md b/README.md index 8609f62f..3156fcbb 100644 --- a/README.md +++ b/README.md @@ -358,7 +358,7 @@ Retry classes: | Attempt outcome | Action | | --- | --- | | 2xx, or an `accept` rule matches | settle `completed` | -| network failure, 5xx, 408, 429 | exponential backoff (base 1 s, max 2 h, jitter 0.2) until `expiresAt`. On Android a failure in the connect phase (the request never reached the server) waits at most 15 minutes | +| network failure, 5xx, 408, 429 | exponential backoff (base 1 s, max 2 h, jitter 0.2) until `expiresAt`. On Android a call that failed while it connected, with no earlier try inside the call, waits at most 15 minutes | | 401, 403 | park as `awaiting-auth`; resume on `updateHeaders()` | | other 4xx matching a `retry.transient` rule (`status`, optional `bodyIncludes`), under the rule's `maxRetries` | as transient. The first matching rule in the list wins. `maxRetries: 3` retries three matching responses; the fourth settles. No `maxRetries` means no cap. See "Transient rule counts" below. A rule for 401, 403, 408 or 429 is rejected at the call | | other 4xx in `retry.terminalHttp.exempt` (default `[404]`), when no rule matched | as transient, with no cap | @@ -490,8 +490,10 @@ kept. A paused entry past `expiresAt` settles `error` with `errorKind: 'expired'` when it is resumed. On Android, `resume()` also ends some retry backoff waits of the queued -entries in its scope, and they run at once. A resumed entry keeps at most 5 -backoff steps. See "Backoff and the app" under Android platform notes. +entries in its scope, and they run at once. `pause()` keeps an entry's +wait, and `resume()` gives back what is left of it under the same rule. A +resumed entry keeps at most 5 backoff steps. See "Backoff and the app" +under Android platform notes. Neither call rejects because the settings cannot be saved. On Android, either call can still reject `E_STORAGE` when an entry file cannot be saved. From bbcb1c193f32d4de294c429f1e869a576ab71e59 Mon Sep 17 00:00:00 2001 From: Dylan Murphy Date: Tue, 17 Nov 2026 19:14:22 -0500 Subject: [PATCH 09/10] Android clear-wait: several refused addresses, kept streak, wake beside a held main row From the third check of this branch: - reachedNoServer accepts suppressed exceptions when each one, and each one nested in it, is a connect failure too: every address of a host refused. Any other suppressed exception still rules the cap out. - toResumed keeps the streak when a wait is kept over the pause, and caps it at 5 only when the wait ended (resume() and the sweep's repair path). - retryNow makes the wake (KEEP) for a waiting entry that has no delayed wake row, also when a held-back main row exists. RowSummary.withWake lists entries with a delayed row; only a wake has an initial delay. Co-Authored-By: Claude Opus 5.5 --- .../ai/openspace/backgroundupload/EntryRun.kt | 4 +- .../backgroundupload/EntryTransitions.kt | 27 +++++++------ .../backgroundupload/QueueController.kt | 10 +++-- .../backgroundupload/RetryClassifier.kt | 38 ++++++++++--------- .../openspace/backgroundupload/Scheduler.kt | 7 ++-- .../backgroundupload/EntryTransitionsTest.kt | 4 ++ .../backgroundupload/MainChainTest.kt | 17 ++++++++- .../backgroundupload/QueueControllerTest.kt | 23 +++++++++++ .../backgroundupload/RetryClassifierTest.kt | 14 +++++++ .../openspace/backgroundupload/TestSupport.kt | 6 +-- 10 files changed, 107 insertions(+), 43 deletions(-) diff --git a/android/src/main/java/ai/openspace/backgroundupload/EntryRun.kt b/android/src/main/java/ai/openspace/backgroundupload/EntryRun.kt index fff032ec..cb048b9e 100644 --- a/android/src/main/java/ai/openspace/backgroundupload/EntryRun.kt +++ b/android/src/main/java/ai/openspace/backgroundupload/EntryRun.kt @@ -373,8 +373,8 @@ internal class EntryRun( AttemptResult.Terminal(verdict.errorKind, verdict.message, null) } else AttemptResult.Transient // The attempt started on a validated network (waitForNetwork). A call - // that failed while it connected, with no earlier try in the call, - // waits at most 15 minutes; any other failure keeps the entry's maximum. + // whose every try failed while it connected waits at most 15 minutes; + // any other failure keeps the entry's maximum. val waitPolicy = if (RetryClassifier.reachedNoServer(error)) RetryClassifier.transportFailurePolicy(policy) else policy return Attempt(entry, url, waitPolicy, result, WaitCause.NO_RESPONSE) } diff --git a/android/src/main/java/ai/openspace/backgroundupload/EntryTransitions.kt b/android/src/main/java/ai/openspace/backgroundupload/EntryTransitions.kt index 76979e4c..ce5d7ff9 100644 --- a/android/src/main/java/ai/openspace/backgroundupload/EntryTransitions.kt +++ b/android/src/main/java/ai/openspace/backgroundupload/EntryTransitions.kt @@ -105,23 +105,26 @@ object EntryTransitions { * updateHeaders() saves with the new headers. The settings value can be * older after a settings write that never landed. Back to queued, the * entry keeps what is left of its wait (resume() then applies - * [clearsWait]), and the streak keeps at most - * [RetryClassifier.RETRY_NOW_MAX_STREAK] steps, as after a cleared wait: - * a pause and resume in normal work must not give a failing entry a - * burst of attempts. + * [clearsWait]). A kept wait keeps its streak too, so the waits after it + * grow as they would with no pause. When the wait ended during the pause, + * the streak keeps at most [RetryClassifier.RETRY_NOW_MAX_STREAK] steps, + * as after a cleared wait: a pause and resume in normal work must not + * give a failing entry a burst of attempts. */ fun toResumed(e: QueueEntry, now: Long): QueueEntry = if (e.parkedGeneration != null && e.parkedGeneration == e.headerGeneration) { e.copy(state = EntryState.AWAITING_AUTH, updatedAt = now) } else { - e.copy( - state = EntryState.QUEUED, - parkedGeneration = null, - backoffStreak = min(e.backoffStreak, RetryClassifier.RETRY_NOW_MAX_STREAK), - // The wait kept over the pause; one that ended during it is gone. - nextAttemptAt = e.nextAttemptAt?.takeIf { it > now }, - updatedAt = now, - ) + // The wait kept over the pause; one that ended during it is gone. + e.nextAttemptAt?.takeIf { it > now }.let { left -> + e.copy( + state = EntryState.QUEUED, + parkedGeneration = null, + backoffStreak = if (left != null) e.backoffStreak else min(e.backoffStreak, RetryClassifier.RETRY_NOW_MAX_STREAK), + nextAttemptAt = left, + updatedAt = now, + ) + } } /** diff --git a/android/src/main/java/ai/openspace/backgroundupload/QueueController.kt b/android/src/main/java/ai/openspace/backgroundupload/QueueController.kt index 01283bf0..8cc44b37 100644 --- a/android/src/main/java/ai/openspace/backgroundupload/QueueController.kt +++ b/android/src/main/java/ai/openspace/backgroundupload/QueueController.kt @@ -250,7 +250,8 @@ class QueueController( * ([WorkScheduler.rowSummary], one query): that is not a retry, it only * makes sure a row exists, and a row WorkManager holds back after a stop * is replaced ([WorkScheduler.schedule]). A queued entry whose wait stays - * and that has no unfinished row at all gets its wake again (KEEP). Running, paused, parked and + * and that has no delayed wake row gets its wake again (KEEP), also when + * it has a main row: WorkManager may hold that row back for hours. Running, paused, parked and * settled entries are left alone. A queued entry sends nothing, and the * per-id gate holds any new row while a worker of the entry still winds * down, so this never starts a second transfer next to one in progress. @@ -293,9 +294,10 @@ class QueueController( val rows = runCatching { scheduler.rowSummary() }.onFailure(::failed).getOrNull() idle.filter { rows == null || it.id !in rows.startable } .forEach { e -> runCatching { scheduleRun(e) }.onFailure(::failed) } - // A wait that stays needs its wake. KEEP leaves one that exists. - waiting.filter { rows == null || it.id !in rows.withRow } - .forEach { e -> runCatching { scheduleRun(e, keepWake = true) }.onFailure(::failed) } + // A wait that stays needs its wake. KEEP leaves one that exists, so + // this is at most one enqueue per entry that lacks a wake. + waiting.filter { rows == null || it.id !in rows.withWake } + .forEach { e -> runCatching { scheduler.scheduleWake(e, e.nextAttemptAt!!, replace = false) }.onFailure(::failed) } } cleared.forEach { events.state(it.toRow()) } first?.let { Diag.error("retryNow: $failures entry saves or WorkManager calls failed; the next app start or foreground tries again", it) } diff --git a/android/src/main/java/ai/openspace/backgroundupload/RetryClassifier.kt b/android/src/main/java/ai/openspace/backgroundupload/RetryClassifier.kt index 552bd3c1..8f0b5e5f 100644 --- a/android/src/main/java/ai/openspace/backgroundupload/RetryClassifier.kt +++ b/android/src/main/java/ai/openspace/backgroundupload/RetryClassifier.kt @@ -41,8 +41,8 @@ object RetryClassifier { const val IN_WORKER_BACKOFF_MAX_MS = 30_000L /** - * The longest wait after a call that failed while it connected, with no - * earlier try in the call ([reachedNoServer]). Every attempt starts on a + * The longest wait after a call whose every try failed while it + * connected ([reachedNoServer]). Every attempt starts on a * network Android reports as validated, so such a failure says little * about the server, and a network change cannot wake a queued entry. * Every other failure keeps the policy's maxMs: a failure after the @@ -55,9 +55,9 @@ object RetryClassifier { const val CLEAR_MIN_AGE_MS = 30_000L /** - * Whether this call failed while it connected, with no earlier try inside - * the same call. Then this call sent no request. (An earlier attempt, in - * an earlier call, may still have reached the server.) + * Whether every try of this call failed while it connected. Then this + * call sent no request. (An earlier attempt, in an earlier call, may + * still have reached the server.) * * From OkHttp 4.12.0: RealConnection.connectSocket throws ConnectException * ("Failed to connect to"); the DNS lookup (Dns.SYSTEM) throws @@ -67,20 +67,24 @@ object RetryClassifier { * SSLPeerUnverifiedException. OkHttp retries inside one call: it throws * the last failure with the earlier tries it recovered from attached as * suppressed (Util.withSuppressed in RetryAndFollowUpInterceptor), and a - * RouteException adds later connect failures as suppressed too. An - * earlier try may have sent the body, so any suppressed exception rules - * the failure out. A SocketTimeoutException is left out: a connect - * timeout and a read or write timeout throw the same class. + * RouteException adds the failures of the host's other addresses as + * suppressed too. So each suppressed exception, and each one nested in + * it, must be a connect failure as well: then every try failed while it + * connected (for example, every address of the host refused). Any other + * suppressed exception is an earlier try that may have sent the body, + * and rules the failure out. A SocketTimeoutException is left out: a + * connect timeout and a read or write timeout throw the same class. */ fun reachedNoServer(error: Throwable): Boolean = - error.suppressed.isEmpty() && ( - error is java.net.ConnectException || - error is java.net.UnknownHostException || - error is java.net.NoRouteToHostException || - error is java.net.PortUnreachableException || - error is javax.net.ssl.SSLHandshakeException || - error is javax.net.ssl.SSLPeerUnverifiedException - ) + isConnectFailure(error) && error.suppressed.all(::reachedNoServer) + + private fun isConnectFailure(error: Throwable) = + error is java.net.ConnectException || + error is java.net.UnknownHostException || + error is java.net.NoRouteToHostException || + error is java.net.PortUnreachableException || + error is javax.net.ssl.SSLHandshakeException || + error is javax.net.ssl.SSLPeerUnverifiedException /** * The highest backoff streak an entry keeps when the app clears its wait. diff --git a/android/src/main/java/ai/openspace/backgroundupload/Scheduler.kt b/android/src/main/java/ai/openspace/backgroundupload/Scheduler.kt index aec92e8b..8e2761c8 100644 --- a/android/src/main/java/ai/openspace/backgroundupload/Scheduler.kt +++ b/android/src/main/java/ai/openspace/backgroundupload/Scheduler.kt @@ -129,9 +129,10 @@ interface WorkScheduler { /** * [startable]: the ids with a run in progress, or a row WorkManager may - * start now. [withRow]: the ids with any unfinished row, main chain or wake. + * start now. [withWake]: the ids with an unfinished delayed row. Only a wake + * has an initial delay; a main-chain row never does. */ -data class RowSummary(val startable: Set, val withRow: Set) +data class RowSummary(val startable: Set, val withWake: Set) /** * One unfinished WorkManager run. [entryId] comes from the run's id tag. @@ -249,7 +250,7 @@ open class RowScheduler( val open = rows.library(UNFINISHED) return RowSummary( startable = open.filter { it.state == WorkInfo.State.RUNNING || it.startsBy(now) }.mapNotNull { it.entryId }.toSet(), - withRow = open.mapNotNull { it.entryId }.toSet(), + withWake = open.filter { it.delayed }.mapNotNull { it.entryId }.toSet(), ) } } diff --git a/android/src/test/java/ai/openspace/backgroundupload/EntryTransitionsTest.kt b/android/src/test/java/ai/openspace/backgroundupload/EntryTransitionsTest.kt index 24a97fea..b65ffd8f 100644 --- a/android/src/test/java/ai/openspace/backgroundupload/EntryTransitionsTest.kt +++ b/android/src/test/java/ai/openspace/backgroundupload/EntryTransitionsTest.kt @@ -59,6 +59,10 @@ class EntryTransitionsTest { assertEquals(3, EntryTransitions.toResumed(paused.copy(backoffStreak = 3), 9).backoffStreak) val parked = paused.copy(parkedGeneration = 3, headerGeneration = 3) assertEquals(9, EntryTransitions.toResumed(parked, 9).backoffStreak) + // A wait kept over the pause keeps its streak; one that ended does not. + val waiting = paused.copy(nextAttemptAt = 500) + assertEquals(9, EntryTransitions.toResumed(waiting, 9).backoffStreak) + assertEquals(5, EntryTransitions.toResumed(waiting, 500).backoffStreak) } @Test diff --git a/android/src/test/java/ai/openspace/backgroundupload/MainChainTest.kt b/android/src/test/java/ai/openspace/backgroundupload/MainChainTest.kt index 5da7c87d..a88230f8 100644 --- a/android/src/test/java/ai/openspace/backgroundupload/MainChainTest.kt +++ b/android/src/test/java/ai/openspace/backgroundupload/MainChainTest.kt @@ -226,7 +226,7 @@ class MainChainTest { } @Test - fun `rowSummary lists the entries that can start, and the entries with any unfinished row`() { + fun `rowSummary lists the entries that can start, and the entries with a delayed wake row`() { work.add("due") work.add("held", runAttemptCount = 8, lastEnqueueTime = work.now - minute) work.add("run", state = RUNNING, runAttemptCount = 1) @@ -235,7 +235,7 @@ class MainChainTest { work.add("gone", state = CANCELLED) val rows = scheduler.rowSummary() assertEquals(setOf("due", "run"), rows.startable) - assertEquals(setOf("due", "held", "run", "later", "blocked"), rows.withRow) + assertEquals(setOf("later"), rows.withWake) // One query, and it asks WorkManager for unfinished rows only. assertEquals(listOf(listOf(ENQUEUED, RUNNING, BLOCKED)), work.libraryQueries) } @@ -309,6 +309,19 @@ class MainChainTest { assertEquals(1, work.rows.size) } + @Test + fun `a return to the foreground makes the wake for a wait that stays when only a held-back main row exists`() { + // Probe Q4: a 10-minute wait, no wake, and a main row WorkManager holds + // for hours. Before, that main row counted as "a row", so the entry + // waited for it. + store.save(transportWait(streak = 9).copy(waitCause = WaitCause.RESPONSE, nextAttemptAt = work.now + 10 * minute)) + work.add("e1", runAttemptCount = 10, lastEnqueueTime = work.now - minute) + controller.retryNow() + assertEquals(listOf("enqueue:e1#wake:KEEP"), rows.log) + val wake = work.rows.single { it.name == "e1#wake" } + assertEquals(work.now + 10 * minute, work.nextScheduleAt(wake)) + } + @Test fun `a stale snapshot does not cancel the wake a worker made meanwhile`() { // Probe P7. retryNow read the entry queued with no wait. Then, before diff --git a/android/src/test/java/ai/openspace/backgroundupload/QueueControllerTest.kt b/android/src/test/java/ai/openspace/backgroundupload/QueueControllerTest.kt index 85c088e4..6d2ac596 100644 --- a/android/src/test/java/ai/openspace/backgroundupload/QueueControllerTest.kt +++ b/android/src/test/java/ai/openspace/backgroundupload/QueueControllerTest.kt @@ -1323,6 +1323,29 @@ class QueueControllerTest { assertEquals(5, store.load("e1")!!.backoffStreak) } + @Test + fun `resume keeps the streak of a wait after a server response that stays`() { + val at = now + 3_600_000 + store.save(waitAfter("e1", WaitCause.RESPONSE, age = 60_000, at = at)) + controller.pause() + controller.resume() + assertEquals(at, store.load("e1")!!.nextAttemptAt) + assertEquals(9, store.load("e1")!!.backoffStreak) + } + + @Test + fun `the sweep's resume repair keeps the streak of a wait that stays`() { + // A death partway through resume(): the row is paused, the settings are not. + val at = now + 3_600_000 + store.save(waitAfter("e1", WaitCause.RESPONSE, age = 60_000, at = at).copy(state = EntryState.PAUSED)) + controller.sweep() + val e = store.load("e1")!! + assertEquals(EntryState.QUEUED, e.state) + assertEquals(at, e.nextAttemptAt) + assertEquals(9, e.backoffStreak) + assertEquals(listOf("e1" to at), scheduler.wakes) + } + @Test fun `retryNow writes no entry file whose wait stays`() { store.save(waitAfter("r", WaitCause.RESPONSE, age = 3_600_000)) diff --git a/android/src/test/java/ai/openspace/backgroundupload/RetryClassifierTest.kt b/android/src/test/java/ai/openspace/backgroundupload/RetryClassifierTest.kt index 11acb893..b840bd17 100644 --- a/android/src/test/java/ai/openspace/backgroundupload/RetryClassifierTest.kt +++ b/android/src/test/java/ai/openspace/backgroundupload/RetryClassifierTest.kt @@ -198,6 +198,20 @@ class RetryClassifierTest { assertFalse(RetryClassifier.reachedNoServer(javax.net.ssl.SSLException("Read error"))) } + @Test + fun `every address of the host refused counts, also when OkHttp nests the failures`() { + val twoAddresses = java.net.ConnectException("Failed to connect to /10.0.0.2") + twoAddresses.addSuppressed(java.net.ConnectException("Failed to connect to /10.0.0.1")) + assertTrue(RetryClassifier.reachedNoServer(twoAddresses)) + val nested = java.net.ConnectException("Failed to connect to /10.0.0.3") + nested.addSuppressed(java.net.UnknownHostException("a").apply { addSuppressed(java.net.ConnectException("b")) }) + assertTrue(RetryClassifier.reachedNoServer(nested)) + // A sent try nested inside a connect failure still rules it out. + val hidden = java.net.ConnectException("Failed to connect to /10.0.0.3") + hidden.addSuppressed(java.net.ConnectException("b").apply { addSuppressed(IOException("unexpected end of stream")) }) + assertFalse(RetryClassifier.reachedNoServer(hidden)) + } + @Test fun `a connect failure after an earlier try in the same call does not count`() { // Probe P10. OkHttp attaches the earlier tries of a call as suppressed; diff --git a/android/src/test/java/ai/openspace/backgroundupload/TestSupport.kt b/android/src/test/java/ai/openspace/backgroundupload/TestSupport.kt index 1c1138ed..8f48131b 100644 --- a/android/src/test/java/ai/openspace/backgroundupload/TestSupport.kt +++ b/android/src/test/java/ai/openspace/backgroundupload/TestSupport.kt @@ -124,8 +124,8 @@ internal class FakeScheduler : WorkScheduler { val runs = mutableListOf() /** What rowSummary() returns as startable. */ val startable = mutableSetOf() - /** What rowSummary() returns as withRow, besides the startable ids. */ - val withRow = mutableSetOf() + /** What rowSummary() returns as withWake. */ + val withWake = mutableSetOf() /** How many rowSummary() calls were made. */ var startableQueries = 0 @@ -154,7 +154,7 @@ internal class FakeScheduler : WorkScheduler { override fun rowSummary(): RowSummary { startableQueries++ - return RowSummary(startable.toSet(), startable + withRow) + return RowSummary(startable.toSet(), withWake.toSet()) } } From df29f7390fe11befb8ddd6f2a2876e72d3ca4bc9 Mon Sep 17 00:00:00 2001 From: Dylan Murphy Date: Tue, 17 Nov 2026 19:14:27 -0500 Subject: [PATCH 10/10] Docs: a kept wait keeps its streak, several refused addresses, wake beside a held row Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 25 ++++++++++++++----------- README.md | 7 ++++--- 2 files changed, 18 insertions(+), 14 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2b6e3ce9..edd27a85 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -71,8 +71,9 @@ last failure; entries saved by an older version read as unknown. its cause. `resume()` gives the entry back what is left of it, ended only by the rule above. Before, `pause()` cleared the wait, so a pause and a resume sent the request again at once, also after a server response. A -paused row still does not show `nextAttemptAt`. A paused entry that -`resume()` moves back to `queued` keeps at most 5 backoff steps. +paused row still does not show `nextAttemptAt`. An entry whose wait stays +keeps its backoff steps, so the waits after it grow as with no pause. When +the wait ended or is cleared, it keeps at most 5 steps. Android: the boot sweep, `resume()`, a return to the foreground, `updateHeaders()` and a same-id `mutate()` now replace a WorkManager row @@ -85,15 +86,17 @@ of the entry's main chain runs, and keeps a row that WorkManager starts within 30 s. It makes the new row before it cancels the old wake, and it keeps a wake that the stored entry still needs. A return to the foreground also makes the wake again (KEEP) for a queued entry whose wait stays and -that has no WorkManager row at all. - -Android: a call that failed while it connected, with no earlier try inside -the same call, now waits at most 15 minutes. Such a call sent no request. -The failures are `ConnectException`, `UnknownHostException`, -`NoRouteToHostException`, `PortUnreachableException`, -`SSLHandshakeException` and `SSLPeerUnverifiedException`, with no -suppressed exception (OkHttp attaches the earlier tries of a call as -suppressed, and an earlier try may have sent the body). Every attempt +that has no wake row, also when it has a main row that WorkManager holds +back. + +Android: a call whose every try failed while it connected now waits at +most 15 minutes. Such a call sent no request. The failures are +`ConnectException`, `UnknownHostException`, `NoRouteToHostException`, +`PortUnreachableException`, `SSLHandshakeException` and +`SSLPeerUnverifiedException`. OkHttp attaches the earlier tries of a call +as suppressed exceptions; each of them must be one of these too, as when +every address of a host refuses. Any other earlier try may have sent the +body, and then the cap does not apply. Every attempt starts on a network that Android reports as validated. Every other failure, a timeout included, keeps the retry policy's `maxMs` (2 hours by default), because the server may have the request. diff --git a/README.md b/README.md index 3156fcbb..ec20d263 100644 --- a/README.md +++ b/README.md @@ -358,7 +358,7 @@ Retry classes: | Attempt outcome | Action | | --- | --- | | 2xx, or an `accept` rule matches | settle `completed` | -| network failure, 5xx, 408, 429 | exponential backoff (base 1 s, max 2 h, jitter 0.2) until `expiresAt`. On Android a call that failed while it connected, with no earlier try inside the call, waits at most 15 minutes | +| network failure, 5xx, 408, 429 | exponential backoff (base 1 s, max 2 h, jitter 0.2) until `expiresAt`. On Android a call whose every try failed while it connected waits at most 15 minutes | | 401, 403 | park as `awaiting-auth`; resume on `updateHeaders()` | | other 4xx matching a `retry.transient` rule (`status`, optional `bodyIncludes`), under the rule's `maxRetries` | as transient. The first matching rule in the list wins. `maxRetries: 3` retries three matching responses; the fourth settles. No `maxRetries` means no cap. See "Transient rule counts" below. A rule for 401, 403, 408 or 429 is rejected at the call | | other 4xx in `retry.terminalHttp.exempt` (default `[404]`), when no rule matched | as transient, with no cap | @@ -492,8 +492,9 @@ kept. A paused entry past `expiresAt` settles `error` with On Android, `resume()` also ends some retry backoff waits of the queued entries in its scope, and they run at once. `pause()` keeps an entry's wait, and `resume()` gives back what is left of it under the same rule. A -resumed entry keeps at most 5 backoff steps. See "Backoff and the app" -under Android platform notes. +kept wait keeps its backoff steps; when the wait ended or is cleared, the +entry keeps at most 5. See "Backoff and the app" under Android platform +notes. Neither call rejects because the settings cannot be saved. On Android, either call can still reject `E_STORAGE` when an entry file cannot be saved.