diff --git a/CHANGELOG.md b/CHANGELOG.md index 574795a4..604c09a9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,51 @@ +## Unreleased + +Android: a system stop now counts as a failed attempt only when it cut a +request. Before, every stop at the job's time limit added one backoff step, +up to 2 hours, also for a run that only waited for the network or for a free +transmit slot and sent nothing. Now a run that only waited goes back to +`queued` with no backoff step. A time-limit stop that cut a request still +takes one backoff step. The foreground-service time limit (Android 15 and +later) now counts as a time-limit stop. + +After every time-limit stop, the library schedules a fresh WorkManager row +for the entry: at once after a run that only waited, or when its backoff +ends, also for a short backoff. WorkManager runs the stopped row again too, +but only after its own backoff, which doubles with each start of that row. +By the WorkManager 2.12.0 code that wait is 64 minutes at the 8th start and +128 minutes at the 9th, so it passes an hour after about 9 starts. A fresh +row does not have that wait. Stops for other reasons go back to `queued` +with no new row, as before: the stopped row runs again when the system +allows it and after WorkManager's own backoff. + +A run that finds its entry waiting longer than it can sleep out now makes +sure the entry's wake row exists, and keeps one that does. Before, a wake +lost to a failed enqueue or a process death left the entry waiting for the +next boot sweep, `resume()` or `mutate()` of the same id. + +A chunked run no longer settles `completed` while a part is not accepted. +If a part ends without an answer and without a failure, the run goes back +to `queued` with no outcome and the worker is retried. + +Stops for quota, standby bucket, background restriction, device state and +the user write a warning to `takeDiagnostics()`, only when the run had +already taken the entry. There is one record per reason, for example +`run stopped by the system: QUOTA`. Its `data` holds the latest `entryId` +and a `count` of such stops since the last `takeDiagnostics()`. So many +stops cannot push older warnings and errors out of the log. Android 10 and +11 (API 29 and 30) cannot read stop reasons, so every stop there goes back +to `queued` with no backoff step and no warning, as before. + +A short backoff that a run waits out in place now saves its backoff step at +once. A stop during that wait keeps the step. A stopped entry whose +`nextAttemptAt` is already past no longer shows it. + +**Behavior change:** the attempt is now written ahead after the request +gets a transmit slot, not before. A run that waits for a slot and is stopped +counts no attempt. So `attempts`, and the `attempt` field of later attempt +events, are lower for uploads that only waited. The number of attempt +events does not change. That is the correct count. + ## 11.3.0 Added `useRequests(filter?)` and `useRequestsWithProgress(filter?)`, React diff --git a/android/src/main/java/ai/openspace/backgroundupload/ChunkedUploadWorker.kt b/android/src/main/java/ai/openspace/backgroundupload/ChunkedUploadWorker.kt index d236ec9a..74d19acd 100644 --- a/android/src/main/java/ai/openspace/backgroundupload/ChunkedUploadWorker.kt +++ b/android/src/main/java/ai/openspace/backgroundupload/ChunkedUploadWorker.kt @@ -55,7 +55,16 @@ internal class ChunkedTransfer(private val run: EntryRun) { return failed.settlement } // Every executor returned, and an executor returns only when its part was - // accepted: the server's auto-publish condition. + // accepted: the server's auto-publish condition. A part that ended with a + // CancellationException while the run was not cancelled also returns, so + // check before completing. + var missing = pending.filterNot { it in acceptedHere } + if (missing.isNotEmpty()) { + // A part can also return when the store already shows it accepted. + val stored = run.ops.latest(run.entryId, run.generation).descriptor!!.parts!! + missing = missing.filterNot { stored[it].accepted } + if (missing.isNotEmpty()) throw EntryRun.PartsNotAcceptedException(missing) + } return Settlement.Completed(null, lastAcceptedUrl ?: d0.reportUrl, d0.method) } diff --git a/android/src/main/java/ai/openspace/backgroundupload/Diag.kt b/android/src/main/java/ai/openspace/backgroundupload/Diag.kt index f347eda4..2b4a7c05 100644 --- a/android/src/main/java/ai/openspace/backgroundupload/Diag.kt +++ b/android/src/main/java/ai/openspace/backgroundupload/Diag.kt @@ -42,6 +42,18 @@ internal object Diag { record(DiagnosticLog.INFO, message, null, data, replaceUntaken) } + /** + * A system stop of a run, as a warning. One record per [kind]: it holds + * the latest entry id and how many such stops happened since JS last + * took the log. + */ + fun countStop(kind: String, entryId: String) { + val message = "run stopped by the system: $kind" + runCatching { Log.w(TAG, "$message ($entryId)") } + val target = log ?: return + runCatching { target.count(DiagnosticLog.WARN, message, clock(), mapOf("entryId" to entryId)) } + } + private fun record( level: String, message: String, diff --git a/android/src/main/java/ai/openspace/backgroundupload/DiagnosticLog.kt b/android/src/main/java/ai/openspace/backgroundupload/DiagnosticLog.kt index 1acc922e..af6cc4ef 100644 --- a/android/src/main/java/ai/openspace/backgroundupload/DiagnosticLog.kt +++ b/android/src/main/java/ai/openspace/backgroundupload/DiagnosticLog.kt @@ -37,6 +37,7 @@ class DiagnosticLog(private val file: File) { const val INFO = "info" const val WARN = "warn" const val ERROR = "error" + const val COUNT_KEY = "count" private val gson = Gson() private val listType = object : TypeToken>() {}.type @@ -88,6 +89,21 @@ class DiagnosticLog(private val file: File) { } } + /** + * A repeating event in one slot, as [replace] does, with a count. The new + * record carries [data], plus `count`: 1 more than the untaken record of + * the same level and message, or 1. So many events cannot push older + * records out of the log. + */ + @Synchronized + fun count(level: String, message: String, at: Long, data: Map) { + runCatching { + val earlier = records().lastOrNull { it.level == level && it.message == message } + val count = ((earlier?.data?.get(COUNT_KEY) as? Number)?.toInt() ?: 0) + 1 + replace(Record(at, level, message, null, data + (COUNT_KEY to count))) + } + } + /** Every record, oldest first, then an empty log. */ @Synchronized fun drain(): List { diff --git a/android/src/main/java/ai/openspace/backgroundupload/EntryRun.kt b/android/src/main/java/ai/openspace/backgroundupload/EntryRun.kt index 27301058..fbb6c2c1 100644 --- a/android/src/main/java/ai/openspace/backgroundupload/EntryRun.kt +++ b/android/src/main/java/ai/openspace/backgroundupload/EntryRun.kt @@ -35,6 +35,14 @@ internal class EntryRun( /** The entry was paused (whole queue or its key) between the module's pause and the work cancel reaching us. */ class PausedException : Exception("paused") + /** + * A chunked run ended with [indexes] not accepted, and no part failed. + * An IOException, so the run takes the store-failure path: back to queued + * with no outcome, and the worker returns retry. + */ + class PartsNotAcceptedException(val indexes: List) : + IOException("chunked run ended with parts $indexes not accepted") + /** How one attempt ended, after the retry table. */ sealed class AttemptResult { data class Accepted(val response: UploadResponse) : AttemptResult() @@ -79,6 +87,10 @@ internal class EntryRun( var liveBytes = 0L private set + /** A cancel reached a request after its write-ahead. Chunked parts set it from their own coroutines. */ + @Volatile + private var stopCutRequest = false + /** * Returns when the run is over. Throws a CancellationException after it * handled a stop, or an IOException after a store write failed (the @@ -105,6 +117,8 @@ internal class EntryRun( var first = true while (true) { try { + // A part cancelled by a sibling's park in the last pass does not count. + stopCutRequest = false if (!first) current = ops.latest(entryId, generation) first = false host.foreground(current) @@ -134,13 +148,14 @@ internal class EntryRun( // A system stop moves a running entry back to queued. A pause or a // cancel already moved it; then this does nothing. endProgress(completed = false) - if (host.stoppedByTimeout()) releaseAfterTimeout() else ops.stopped(entryId, generation) + afterStop(host.stopKind()) throw error } catch (error: IOException) { - // A store write failed (disk full, directory briefly unwritable). - // The attempt step classifies every network IOException itself, so - // one that lands here is storage: transient, no outcome. Back to - // queued; the caller returns retry. + // A store write failed (disk full, directory briefly unwritable), or + // a chunked run ended with a part not accepted. The attempt step + // classifies every network IOException itself, so one that lands here + // is not a network failure: transient, no outcome. Back to queued; + // the caller returns retry. endProgress(completed = false) ops.stopped(entryId, generation) throw error @@ -183,11 +198,33 @@ internal class EntryRun( ) /** - * The system stopped the run at its time limit. That happens to a - * headless run whose foreground start was denied (API 31+), after about - * 10 minutes. Starting again at once would send the body from byte 0 on - * every run, so this is one more transient failure: the next backoff - * step, then a wake. + * A system stop. Only a time-limit stop that cut a request is a failed + * attempt. A run that only waited (for the network, a slot, or a short + * backoff) sent nothing, so it goes back to queued with no backoff step. + * + * After any time-limit stop a fresh wake row starts the entry again. + * WorkManager also runs the stopped row again, but only after its own + * backoff, which grows with every start of that row. The other stop + * reasons say nothing about the server, and the system decides when their + * row runs again. + * + * The state change is saved first, then the diagnostic record. + */ + private fun afterStop(kind: StopKind) { + when { + kind == StopKind.TIME_LIMIT && stopCutRequest -> releaseAfterTimeout() + kind == StopKind.TIME_LIMIT -> ops.stopped(entryId, generation, wake = true) + else -> ops.stopped(entryId, generation) + } + if (kind.logged) Diag.countStop(kind.name, entryId) + } + + /** + * The system stopped the run at its time limit while a request was on its + * way. That happens to a run with no foreground mode after about 10 + * minutes, or to a foreground run whose service type used up its time. + * Starting again at once would send the body from byte 0 on every run, so + * this is one more transient failure: the next backoff step, then a wake. */ private fun releaseAfterTimeout() { runCatching { @@ -197,12 +234,20 @@ internal class EntryRun( val now = host.now() val backoff = RetryClassifier.backoffMs(policy(e), streak) ops.release(entryId, generation, RetryClassifier.nextAttemptAt(now, backoff, e.expiresAt), streak) - }.onFailure { Diag.error("could not release '$entryId' after a timeout stop", it) } + }.onFailure { + if (it is WorkerOps.WakeNotScheduledException) { + Diag.error("released '$entryId' after a timeout stop, but could not schedule its wake", it.cause) + } else { + Diag.error("could not release '$entryId' after a timeout stop", it) + } + } } /** * Sleeps out a short backoff remainder. False when the wait is long (the - * wake run comes back for it) or the entry is no longer queued. + * wake run comes back for it) or the entry is no longer queued. For a long + * wait it first makes sure a wake is pending, so a lost wake does not + * leave the entry waiting for the next boot sweep. */ private suspend fun waitUntilDue(initial: QueueEntry): Boolean { var e = initial @@ -210,7 +255,10 @@ internal class EntryRun( val at = e.nextAttemptAt ?: return true val remaining = at - host.now() if (remaining <= 0) return true - if (remaining > RetryClassifier.IN_WORKER_BACKOFF_MAX_MS) return false + if (remaining > RetryClassifier.IN_WORKER_BACKOFF_MAX_MS) { + if (e.state == EntryState.QUEUED) ops.ensureWake(e, at) + return false + } host.sleep(min(remaining, WAIT_POLL_MS)) e = store.load(entryId) ?: return false if (e.state != EntryState.QUEUED && e.state != EntryState.RUNNING) return false @@ -239,11 +287,15 @@ internal class EntryRun( RetryClassifier.policy(ops.settings().retry, entry.descriptor?.retry) /** - * One HTTP attempt, the step both transfers share: write the attempt - * ahead (attempts + 1, its X-Request-Id), send, emit the attempt event, - * and classify. [partIndex] is null for a simple entry. [body] builds the - * request body from the entry the attempt runs under. [fileExists] tells a - * missing payload from a network failure. + * One HTTP attempt, the step both transfers share: take a transmit slot, + * write the attempt ahead (attempts + 1, its X-Request-Id), send, emit the + * attempt event, and classify. [partIndex] is null for a simple entry. + * [body] builds the request body from the entry the attempt runs under. + * [fileExists] tells a missing payload from a network failure. + * + * The write-ahead happens inside the slot. A run that only waits for a + * slot sends nothing, so it counts no attempt, and the request takes the + * headers that are current when it starts to send. */ suspend fun attempt( partIndex: Int?, @@ -252,7 +304,29 @@ internal class EntryRun( fileExists: () -> Boolean, ): Attempt { val requestId = UUID.randomUUID().toString() - val entry = ops.recordAttempt(entryId, generation, requestId) + val highPriority = ops.latest(entryId, generation).descriptor!!.highPriority + var written: QueueEntry? = null + var failure: Throwable? = null + val response = try { + host.withSlot(highPriority) { + val e = ops.recordAttempt(entryId, generation, requestId) + written = e + val d = e.descriptor!! + val part = partIndex?.let { d.parts!![it] } + host.send(TransferRequest(part?.url ?: d.url!!, d.method, headersFor(d, part, requestId), body(d, part)), onProgress) + } + } catch (error: CancellationException) { + // A stop after the write-ahead cut a request that was on its way. + if (written != null) stopCutRequest = true + throw error + } catch (error: Throwable) { + // With no write-ahead, the store write failed: not a network attempt. + if (written == null) throw error + failure = error + null + } + + val entry = written!! // The generation of the headers this attempt sends: both come from the same entry. val headerGeneration = entry.headerGeneration val d = entry.descriptor!! @@ -260,11 +334,8 @@ internal class EntryRun( val url = part?.url ?: d.url!! val policy = policy(entry) - val response = try { - host.send(TransferRequest(url, d.method, headersFor(d, part, requestId), body(d, part), d.highPriority), onProgress) - } catch (error: CancellationException) { - throw error - } catch (error: Throwable) { + if (response == null) { + val error = failure!! // A failed probe reads as present, so a flaky check is a retryable // network error and not a terminal "file gone". val verdict = RetryClassifier.classifyFailure(error, runCatching(fileExists).getOrDefault(true)) @@ -316,7 +387,7 @@ internal class EntryRun( val now = host.now() if (backoff <= RetryClassifier.IN_WORKER_BACKOFF_MAX_MS) { val wait = min(backoff, max(0L, expiresAt - now)) - ops.backingOff(entryId, generation, now + wait) + ops.backingOff(entryId, generation, now + wait, streak) host.sleep(wait) return } diff --git a/android/src/main/java/ai/openspace/backgroundupload/EntryTransitions.kt b/android/src/main/java/ai/openspace/backgroundupload/EntryTransitions.kt index 7b1784c5..60a7d423 100644 --- a/android/src/main/java/ai/openspace/backgroundupload/EntryTransitions.kt +++ b/android/src/main/java/ai/openspace/backgroundupload/EntryTransitions.kt @@ -1,5 +1,7 @@ package ai.openspace.backgroundupload +import kotlin.math.max + /** * The state changes of one entry, as pure functions. [QueueController] and * [WorkerOps] apply them inside `QueueStore.compute`, so each one is atomic @@ -21,10 +23,15 @@ object EntryTransitions { /** * A short backoff that the worker waits out itself. The row stays running - * and shows when the next attempt is due. + * and shows when the next attempt is due. The failure's step is saved now, + * not at the next release. Chunked parts keep their own streaks, so the + * entry keeps the highest; an accepted part resets it to 0. */ - fun toBackingOff(e: QueueEntry, nextAttemptAt: Long, now: Long) = - e.copy(nextAttemptAt = nextAttemptAt, updatedAt = now) + fun toBackingOff(e: QueueEntry, nextAttemptAt: Long, streak: Int, now: Long) = e.copy( + nextAttemptAt = nextAttemptAt, + backoffStreak = max(e.backoffStreak, streak), + updatedAt = now, + ) /** One attempt starts: attempts + 1, its X-Request-Id, and no pending backoff. */ fun toAttempt(e: QueueEntry, requestId: String, now: Long) = e.copy( @@ -71,8 +78,15 @@ object EntryTransitions { e.copy(state = EntryState.QUEUED, parkedGeneration = null, updatedAt = now) } - /** A system stop of a running worker. WorkManager runs the row again. No outcome. */ - fun toStopped(e: QueueEntry, now: Long) = e.copy(state = EntryState.QUEUED, updatedAt = now) + /** + * A system stop of a running worker. No outcome. A nextAttemptAt that is + * already past is cleared: a queued row does not show a due time behind it. + */ + fun toStopped(e: QueueEntry, now: Long) = e.copy( + state = EntryState.QUEUED, + nextAttemptAt = e.nextAttemptAt?.takeIf { it > now }, + updatedAt = now, + ) /** updateHeaders() on a parked entry. */ fun toUnparked(e: QueueEntry, paused: Boolean, now: Long) = e.copy( diff --git a/android/src/main/java/ai/openspace/backgroundupload/EntryWorker.kt b/android/src/main/java/ai/openspace/backgroundupload/EntryWorker.kt index 13b98832..f74fae4f 100644 --- a/android/src/main/java/ai/openspace/backgroundupload/EntryWorker.kt +++ b/android/src/main/java/ai/openspace/backgroundupload/EntryWorker.kt @@ -2,9 +2,9 @@ package ai.openspace.backgroundupload import android.app.NotificationManager import android.content.Context +import android.os.Build import androidx.work.CoroutineWorker import androidx.work.ForegroundInfo -import androidx.work.WorkInfo import androidx.work.WorkerParameters import kotlinx.coroutines.CancellationException import kotlinx.coroutines.Dispatchers @@ -77,8 +77,11 @@ open class EntryWorker(protected val context: Context, params: WorkerParameters) override suspend fun sleep(ms: Long) = delay(ms) + override suspend fun withSlot(highPriority: Boolean, block: suspend () -> T): T = + transferGate.withPermit(highPriority, block) + override suspend fun send(request: TransferRequest, onProgress: (Long) -> Unit): UploadResponse = - transferGate.withPermit(request.highPriority) { okhttpSend(uploadHttpClient, request, onProgress) } + okhttpSend(uploadHttpClient, request, onProgress) override fun connectivity(wifiOnly: Boolean): Connectivity { connectivity = validateConnectivity(context, wifiOnly) @@ -110,7 +113,10 @@ open class EntryWorker(protected val context: Context, params: WorkerParameters) override fun attempt(event: AttemptEvent) = EventReporter.attempt(event) - override fun stoppedByTimeout() = stopReason == WorkInfo.STOP_REASON_TIMEOUT + // getStopReason() needs API 31. Below that the reason is unknown, and + // every stop takes the plain path: back to queued, no backoff step. + override fun stopKind() = + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.S) StopKind.of(stopReason) else StopKind.OTHER } // MARK: - notification diff --git a/android/src/main/java/ai/openspace/backgroundupload/TransferHost.kt b/android/src/main/java/ai/openspace/backgroundupload/TransferHost.kt index d7556a3d..0b4a71c1 100644 --- a/android/src/main/java/ai/openspace/backgroundupload/TransferHost.kt +++ b/android/src/main/java/ai/openspace/backgroundupload/TransferHost.kt @@ -1,5 +1,7 @@ package ai.openspace.backgroundupload +import androidx.work.WorkInfo + /** * What a run needs from the platform: time, the network, progress, the * notification, and live attempt events. [EntryWorker] is the real one. The @@ -11,7 +13,13 @@ internal interface TransferHost { suspend fun sleep(ms: Long) - /** One request through the library-wide cap of 4. Throws on a transport failure. */ + /** + * Runs [block] while it holds one slot of the library-wide cap of 4. The + * wait for a free slot happens here, before [block] starts. + */ + suspend fun withSlot(highPriority: Boolean, block: suspend () -> T): T + + /** One request. Call it inside [withSlot]. Throws on a transport failure. */ suspend fun send(request: TransferRequest, onProgress: (Long) -> Unit): UploadResponse /** The network state for the queue's wifi-only setting. The notification shows it. */ @@ -29,6 +37,45 @@ internal interface TransferHost { fun attempt(event: AttemptEvent) - /** Whether the system stopped this run at its time limit (JobScheduler, about 10 minutes). */ - fun stoppedByTimeout(): Boolean + /** Why the system stopped this run. Read it only after the run was cancelled. */ + fun stopKind(): StopKind +} + +/** + * Why the system stopped a run. [logged] kinds say why a queue did not move, + * so the run writes one persisted warning that names the kind. + */ +internal enum class StopKind(val logged: Boolean) { + /** The job's time limit, or the foreground service's time limit. */ + TIME_LIMIT(false), + + /** A WorkManager network constraint stopped being met. */ + NETWORK(false), + + QUOTA(true), + APP_STANDBY(true), + BACKGROUND_RESTRICTION(true), + DEVICE_STATE(true), + USER(true), + + /** Every other reason, and every stop on API 29 and 30, which cannot read the reason. */ + OTHER(false); + + companion object { + /** + * Maps a WorkManager stop reason (WorkInfo.STOP_REASON_*). A cancel by the + * app, a preempt, the other constraints, NOT_STOPPED, UNKNOWN and any + * reason a later Android adds are OTHER. + */ + fun of(reason: Int): StopKind = when (reason) { + WorkInfo.STOP_REASON_TIMEOUT, WorkInfo.STOP_REASON_FOREGROUND_SERVICE_TIMEOUT -> TIME_LIMIT + WorkInfo.STOP_REASON_CONSTRAINT_CONNECTIVITY -> NETWORK + WorkInfo.STOP_REASON_QUOTA -> QUOTA + WorkInfo.STOP_REASON_APP_STANDBY -> APP_STANDBY + WorkInfo.STOP_REASON_BACKGROUND_RESTRICTION -> BACKGROUND_RESTRICTION + WorkInfo.STOP_REASON_DEVICE_STATE -> DEVICE_STATE + WorkInfo.STOP_REASON_USER -> USER + else -> OTHER + } + } } diff --git a/android/src/main/java/ai/openspace/backgroundupload/UploadUtils.kt b/android/src/main/java/ai/openspace/backgroundupload/UploadUtils.kt index e17c37ec..154cac08 100644 --- a/android/src/main/java/ai/openspace/backgroundupload/UploadUtils.kt +++ b/android/src/main/java/ai/openspace/backgroundupload/UploadUtils.kt @@ -40,8 +40,6 @@ data class TransferRequest( val method: String, val headers: Map, val body: RequestBody?, - /** Whether the request may use the slots kept for high priority. See [TransferGate]. */ - val highPriority: Boolean = false, ) /** Sends one request and reports bytes written. The headers are sent as they are. */ diff --git a/android/src/main/java/ai/openspace/backgroundupload/WorkerOps.kt b/android/src/main/java/ai/openspace/backgroundupload/WorkerOps.kt index cc1e8483..55e98dc1 100644 --- a/android/src/main/java/ai/openspace/backgroundupload/WorkerOps.kt +++ b/android/src/main/java/ai/openspace/backgroundupload/WorkerOps.kt @@ -114,16 +114,18 @@ class WorkerOps( /** * A short backoff the worker waits out in place: the row stays running - * and carries [nextAttemptAt]. Best effort; a lost write only hides the - * time. For a chunked entry, a sibling part's next attempt clears it. + * and carries [nextAttemptAt]. It also saves the failure's backoff + * [streak], so a stop during the wait keeps that step. Best effort; a + * lost write hides the time and loses the step. For a chunked entry, a + * sibling part's next attempt clears the time. */ - fun backingOff(id: String, generation: Int, nextAttemptAt: Long) { + fun backingOff(id: String, generation: Int, nextAttemptAt: Long, streak: Int) { val now = clock() var applied = false val next = store.update(id) { e -> if (EntryTransitions.isOwnedRun(e, generation)) { applied = true - EntryTransitions.toBackingOff(e, nextAttemptAt, now) + EntryTransitions.toBackingOff(e, nextAttemptAt, streak, now) } else e } if (applied && next != null) events.state(next.toRow()) @@ -328,12 +330,40 @@ class WorkerOps( } if (!applied || next == null) return false events.state(next.toRow()) - scheduler.scheduleWake(next, nextAttemptAt, replace = true) + try { + scheduler.scheduleWake(next, nextAttemptAt, replace = true) + } catch (error: Exception) { + throw WakeNotScheduledException(id, error) + } return true } - /** A system stop. A paused or cancelled entry is the module's, so only a running one moves. Never journals. */ - fun stopped(id: String, generation: Int) { + /** The release was saved, but its wake was not scheduled. A later run makes it again ([ensureWake]). */ + class WakeNotScheduledException(id: String, cause: Throwable) : + Exception("released '$id', but could not schedule its wake", cause) + + /** + * A run found [entry] queued with a wait too long to sleep out. Makes its + * wake at [at] if none is pending, and keeps one that is (KEEP). A wake + * goes missing when its enqueue threw, or the process died between the + * entry save and WorkManager's commit. Never throws. + */ + fun ensureWake(entry: QueueEntry, at: Long) { + runCatching { scheduler.scheduleWake(entry, at, replace = false) } + .onFailure { Diag.error("could not schedule a wake for '${entry.id}'", it) } + } + + /** + * A system stop. A paused or cancelled entry is the module's, so only a + * running one moves. Never journals. + * + * [wake] true also schedules a fresh wake row, at the entry's + * nextAttemptAt or now. A time-limit stop needs it: WorkManager runs the + * stopped row again only after its own backoff, which grows with every + * start of that row (up to 5 hours), and while that row waits, schedule() + * adds no row. A fresh row starts its count at 0. + */ + fun stopped(id: String, generation: Int, wake: Boolean = false) { val now = clock() var applied = false val next = runCatching { @@ -344,7 +374,14 @@ class WorkerOps( } else e } }.getOrNull() - if (applied && next != null) events.state(next.toRow()) + if (!applied || next == null) return + events.state(next.toRow()) + if (wake) { + // A stop must not throw past the run. Without the wake, the stopped + // row still runs again, after WorkManager's own backoff. + runCatching { scheduler.scheduleWake(next, next.nextAttemptAt ?: now, replace = true) } + .onFailure { Diag.error("could not schedule a wake for '$id' after a stop", it) } + } } fun settings(): QueueSettings = settings.load() diff --git a/android/src/test/java/ai/openspace/backgroundupload/EntryRunTest.kt b/android/src/test/java/ai/openspace/backgroundupload/EntryRunTest.kt index 82172bf7..306b7bf7 100644 --- a/android/src/test/java/ai/openspace/backgroundupload/EntryRunTest.kt +++ b/android/src/test/java/ai/openspace/backgroundupload/EntryRunTest.kt @@ -1,7 +1,14 @@ package ai.openspace.backgroundupload +import androidx.work.WorkInfo import kotlinx.coroutines.CancellationException +import kotlinx.coroutines.Job +import kotlinx.coroutines.awaitCancellation +import kotlinx.coroutines.currentCoroutineContext +import kotlinx.coroutines.ensureActive +import kotlinx.coroutines.launch import kotlinx.coroutines.runBlocking +import org.junit.After import org.junit.Assert.assertEquals import org.junit.Assert.assertFalse import org.junit.Assert.assertNull @@ -23,7 +30,12 @@ internal class FakeHost(var clock: Long = 10_000L) : TransferHost { var handler: suspend (TransferRequest, (Long) -> Unit) -> UploadResponse = { _, _ -> UploadResponse(200, "ok", mapOf()) } var onSleep: () -> Unit = {} var network = ArrayDeque() - var timeout = false + /** What stopKind() answers after a cancel. */ + var stop = StopKind.OTHER + /** Runs while a request waits for a transmit slot, before it holds one. */ + var beforeSlot: suspend () -> Unit = {} + /** The highPriority argument of every slot request, in order. */ + val slots = mutableListOf() var foregroundError: Throwable? = null /** The wifiOnly value of every connectivity check, in order. */ val wifiChecks = mutableListOf() @@ -34,6 +46,14 @@ internal class FakeHost(var clock: Long = 10_000L) : TransferHost { sleeps += ms clock += ms onSleep() + // A real delay throws here when the run's job was cancelled. + currentCoroutineContext().ensureActive() + } + + override suspend fun withSlot(highPriority: Boolean, block: suspend () -> T): T { + slots += highPriority + beforeSlot() + return block() } override suspend fun send(request: TransferRequest, onProgress: (Long) -> Unit): UploadResponse { @@ -66,7 +86,7 @@ internal class FakeHost(var clock: Long = 10_000L) : TransferHost { attempts += event } - override fun stoppedByTimeout() = timeout + override fun stopKind() = stop } class EntryRunTest { @@ -93,6 +113,13 @@ class EntryRunTest { controller = QueueController(store, journal, settings, events, scheduler, { false }, { host.clock }) controller.configureRetry(RetryDefaults(baseMs = 1_000, jitter = 0.0)) journal.drain(Any()) + Diag.log = null + } + + @After + fun tearDown() { + Diag.log = null + Diag.clock = System::currentTimeMillis } private fun run(id: String = "e1") = runBlocking { EntryRun(id, store, ops, host).run() } @@ -131,9 +158,11 @@ class EntryRunTest { @Test fun `a high-priority entry sends its request as high priority`() { controller.enqueue(parsed(descriptor = desc(dataJson = """{"a":1}""", highPriority = true))) - respond(200) + controller.enqueue(parsed(id = "normal")) + respond(200, 200) run() - assertTrue(host.requests.single().highPriority) + run("normal") + assertEquals(listOf(true, false), host.slots) } @Test @@ -371,14 +400,235 @@ class EntryRunTest { fun `a timeout stop takes one more backoff step instead of restarting at once`() { controller.enqueue(parsed()) store.save(store.load("e1")!!.copy(backoffStreak = 2)) - host.timeout = true - host.handler = { _, _ -> throw CancellationException("timeout") } + host.stop = StopKind.TIME_LIMIT + host.handler = { _, _ -> throw CancellationException("timeout") } // the stop cuts the request assertThrows(CancellationException::class.java) { run() } val e = store.load("e1")!! assertEquals(EntryState.QUEUED, e.state) assertEquals(3, e.backoffStreak) + assertEquals(1, e.attempts) assertEquals(host.clock + 4_000, e.nextAttemptAt) // base 1 s * 2^(3-1) + // A fresh wake row, even for a wait a worker could sleep out in place. assertEquals(listOf("e1" to host.clock + 4_000), scheduler.wakes) + assertEquals(listOf(true), scheduler.wakeReplaces) + } + + @Test + fun `a foreground-service timeout that cut a request takes one backoff step`() { + controller.enqueue(parsed()) + store.save(store.load("e1")!!.copy(backoffStreak = 6)) + host.stop = StopKind.of(WorkInfo.STOP_REASON_FOREGROUND_SERVICE_TIMEOUT) + host.handler = { _, _ -> throw CancellationException("fgs timeout") } + assertThrows(CancellationException::class.java) { run() } + val e = store.load("e1")!! + assertEquals(EntryState.QUEUED, e.state) + assertEquals(7, e.backoffStreak) + assertEquals(listOf("e1" to host.clock + 64_000), scheduler.wakes) // base 1 s * 2^(7-1) + } + + @Test + fun `a timeout stop while it waits for the network is not a failed attempt, and a fresh row runs it at once`() { + controller.enqueue(parsed()) + host.stop = StopKind.TIME_LIMIT + host.network = ArrayDeque(listOf(Connectivity.NoInternet)) + host.onSleep = { throw CancellationException("timeout") } + assertThrows(CancellationException::class.java) { run() } + val e = store.load("e1")!! + assertEquals(EntryState.QUEUED, e.state) + assertEquals(0, e.backoffStreak) + assertNull(e.nextAttemptAt) + assertEquals(0, e.attempts) + assertEquals(emptyList(), host.requests) + assertEquals(listOf("e1" to host.clock), scheduler.wakes) + assertEquals(listOf(true), scheduler.wakeReplaces) + } + + @Test + fun `a timeout stop while it waits for a transmit slot counts no attempt`() { + controller.enqueue(parsed()) + store.save(store.load("e1")!!.copy(backoffStreak = 2)) + host.stop = StopKind.TIME_LIMIT + host.beforeSlot = { throw CancellationException("timeout") } + assertThrows(CancellationException::class.java) { run() } + val e = store.load("e1")!! + assertEquals(EntryState.QUEUED, e.state) + assertEquals(0, e.attempts) + assertNull(e.lastRequestId) + assertEquals(2, e.backoffStreak) + assertNull(e.nextAttemptAt) + assertEquals(listOf("e1" to host.clock), scheduler.wakes) + assertEquals(emptyList(), host.attempts) + } + + @Test + fun `the attempt is written ahead inside the slot`() { + controller.enqueue(parsed()) + var attemptsWhileWaiting = -1 + host.beforeSlot = { attemptsWhileWaiting = store.load("e1")!!.attempts } + respond(200) + run() + assertEquals(0, attemptsWhileWaiting) + assertEquals(1, store.load("e1")!!.attempts) + } + + @Test + fun `a timeout stop during a short backoff keeps the failure's step and wakes at the due time`() { + controller.enqueue(parsed()) + respond(503) + host.stop = StopKind.TIME_LIMIT + val start = host.clock + // The stop lands 600 ms into the 1 s wait after the 503. + host.onSleep = { + host.clock -= 400 + throw CancellationException("timeout") + } + assertThrows(CancellationException::class.java) { run() } + val e = store.load("e1")!! + assertEquals(EntryState.QUEUED, e.state) + assertEquals(1, e.backoffStreak) // the 503's step, as if no stop had landed + assertEquals(1, e.attempts) + assertEquals(start + 1_000, e.nextAttemptAt) + assertEquals(listOf("e1" to start + 1_000), scheduler.wakes) + } + + @Test + fun `a stop after a short backoff ran out clears the past nextAttemptAt`() { + controller.enqueue(parsed()) + respond(503) + host.onSleep = { throw CancellationException("preempted") } // after the whole 1 s wait + assertThrows(CancellationException::class.java) { run() } + val e = store.load("e1")!! + assertEquals(EntryState.QUEUED, e.state) + assertNull(e.nextAttemptAt) + assertFalse(events.rows.last().toMap().containsKey("nextAttemptAt")) + assertEquals(1, e.backoffStreak) + assertEquals(emptyList>(), scheduler.wakes) // not a time-limit stop + } + + @Test + fun `a quota stop goes back to queued with no backoff step and no wake, and writes a counted warning`() { + val file = File(tmp.newFolder(), "diagnostics.json") + Diag.log = DiagnosticLog(file) + controller.enqueue(parsed()) + store.save(store.load("e1")!!.copy(backoffStreak = 2)) + host.stop = StopKind.QUOTA + host.handler = { _, _ -> throw CancellationException("quota") } // even when it cut a request + assertThrows(CancellationException::class.java) { run() } + val e = store.load("e1")!! + assertEquals(EntryState.QUEUED, e.state) + assertEquals(2, e.backoffStreak) + assertNull(e.nextAttemptAt) + assertEquals(emptyList>(), scheduler.wakes) + val warning = DiagnosticLog(file).drain().single() + assertEquals(DiagnosticLog.WARN, warning.level) + assertEquals("run stopped by the system: QUOTA", warning.message) + assertEquals(mapOf("entryId" to "e1", "count" to 1.0), warning.data) + } + + @Test + fun `50 stops keep one counted record per kind, and an older error survives`() { + val file = File(tmp.newFolder(), "diagnostics.json") + Diag.log = DiagnosticLog(file) + Diag.error("an older error") + host.handler = { _, _ -> throw CancellationException("stopped") } + for (n in 1..50) { + controller.enqueue(parsed(id = "e$n")) + host.stop = if (n % 2 == 0) StopKind.QUOTA else StopKind.APP_STANDBY + assertThrows(CancellationException::class.java) { run("e$n") } + } + val records = DiagnosticLog(file).drain() + assertEquals(listOf("an older error", "run stopped by the system: APP_STANDBY", "run stopped by the system: QUOTA"), records.map { it.message }) + assertEquals(mapOf("entryId" to "e49", "count" to 25.0), records[1].data) + assertEquals(mapOf("entryId" to "e50", "count" to 25.0), records[2].data) + } + + @Test + fun `the stop is saved before the warning is written`() { + val file = File(tmp.newFolder(), "diagnostics.json") + Diag.log = DiagnosticLog(file) + controller.enqueue(parsed()) + var stateAtWarning: EntryState? = null + Diag.clock = { + stateAtWarning = store.load("e1")!!.state + host.clock + } + host.stop = StopKind.DEVICE_STATE + host.handler = { _, _ -> throw CancellationException("doze") } + assertThrows(CancellationException::class.java) { run() } + assertEquals(EntryState.QUEUED, stateAtWarning) + } + + @Test + fun `a network-constraint stop is never a failure and writes no warning`() { + val file = File(tmp.newFolder(), "diagnostics.json") + Diag.log = DiagnosticLog(file) + controller.enqueue(parsed()) + host.stop = StopKind.NETWORK + host.handler = { _, _ -> throw CancellationException("network lost") } + assertThrows(CancellationException::class.java) { run() } + val e = store.load("e1")!! + assertEquals(EntryState.QUEUED, e.state) + assertEquals(0, e.backoffStreak) + assertNull(e.nextAttemptAt) + assertEquals(emptyList>(), scheduler.wakes) + assertEquals(emptyList(), DiagnosticLog(file).drain()) + } + + @Test + fun `a run that finds a long wait makes its wake again, and keeps a wake that exists`() { + controller.enqueue(parsed()) + store.save(store.load("e1")!!.copy(nextAttemptAt = host.clock + 60_000)) + scheduler.wakes.clear() + scheduler.wakeReplaces.clear() + run() + assertEquals(emptyList(), host.requests) + assertEquals(EntryState.QUEUED, store.load("e1")!!.state) + assertEquals(listOf("e1" to host.clock + 60_000), scheduler.wakes) + assertEquals(listOf(false), scheduler.wakeReplaces) // KEEP: a pending wake stays as it is + } + + @Test + fun `a wake lost after a timeout release is made again by the stopped row's re-run`() { + val file = File(tmp.newFolder(), "diagnostics.json") + Diag.log = DiagnosticLog(file) + controller.enqueue(parsed()) + store.save(store.load("e1")!!.copy(backoffStreak = 6)) + host.stop = StopKind.TIME_LIMIT + host.handler = { _, _ -> throw CancellationException("timeout") } + scheduler.wakeFailures = 1 + assertThrows(CancellationException::class.java) { run() } + val e = store.load("e1")!! + assertEquals(EntryState.QUEUED, e.state) + assertEquals(7, e.backoffStreak) + val at = host.clock + 64_000 + assertEquals(at, e.nextAttemptAt) + assertEquals(emptyList>(), scheduler.wakes) + val error = DiagnosticLog(file).drain().single() + assertEquals(DiagnosticLog.ERROR, error.level) + assertEquals("released 'e1' after a timeout stop, but could not schedule its wake", error.message) + + // WorkManager runs the stopped row again. It sees the long wait and makes the wake. + run() + assertEquals(1, host.requests.size) + assertEquals(listOf("e1" to at), scheduler.wakes) + assertEquals(listOf(false), scheduler.wakeReplaces) + } + + @Test + fun `a wake that fails after an idle timeout stop leaves the entry queued and writes one error`() { + val file = File(tmp.newFolder(), "diagnostics.json") + Diag.log = DiagnosticLog(file) + controller.enqueue(parsed()) + host.stop = StopKind.TIME_LIMIT + host.network = ArrayDeque(listOf(Connectivity.NoInternet)) + host.onSleep = { throw CancellationException("timeout") } + scheduler.wakeFailures = 1 + val thrown = assertThrows(CancellationException::class.java) { run() } + assertEquals("timeout", thrown.message) // the stop ends as a stop, not with the enqueue error + assertEquals(EntryState.QUEUED, store.load("e1")!!.state) + val error = DiagnosticLog(file).drain().single() + assertEquals(DiagnosticLog.ERROR, error.level) + assertEquals("could not schedule a wake for 'e1' after a stop", error.message) } @Test @@ -716,6 +966,107 @@ class EntryRunTest { assertEquals(EntryState.COMPLETED, store.load("e1")!!.state) } + @Test + fun `a part cut by a sibling's park does not make a later idle stop a failure`() { + chunked() + lateinit var job: Job + host.stop = StopKind.TIME_LIMIT + host.handler = { r, _ -> + if (r.url.endsWith("start=0")) { + try { + awaitCancellation() // in flight when part 1 parks the entry + } finally { + // New headers land before the park, so the run goes on for a second pass. + controller.updateHeaders(mapOf("Authorization" to "Bearer new")) + // In that pass the network is gone and the system stops the run at its time limit. + host.network = ArrayDeque(listOf(Connectivity.NoInternet)) + host.onSleep = { job.cancel(CancellationException("timeout")) } + } + } else UploadResponse(401, "", mapOf()) + } + runBlocking { + job = launch { EntryRun("e1", store, ops, host).run() } + job.join() + } + assertEquals(listOf(EntryRun.CONNECTIVITY_POLL_MS), host.sleeps) // the second pass waited for the network + val e = store.load("e1")!! + assertEquals(EntryState.QUEUED, e.state) + assertEquals(0, e.backoffStreak) + assertNull(e.nextAttemptAt) + assertEquals(listOf("e1" to host.clock), scheduler.wakes) // a fresh row, at once + } + + @Test + fun `a chunked part cut by a timeout stop takes one backoff step and keeps the accepted parts`() { + chunked() + store.save(store.load("e1")!!.copy(backoffStreak = 2)) + lateinit var job: Job + var cutRequestId: String? = null + host.stop = StopKind.TIME_LIMIT + host.handler = { r, _ -> + if (r.url.endsWith("start=20")) { + cutRequestId = r.headers["X-Request-Id"] + job.cancel(CancellationException("timeout")) // the stop lands while part 2 is on its way + awaitCancellation() + } else UploadResponse(200, "", mapOf()) + } + runBlocking { + job = launch { EntryRun("e1", store, ops, host).run() } + job.join() + } + val e = store.load("e1")!! + assertEquals(EntryState.QUEUED, e.state) + assertEquals(listOf(true, true, false), e.descriptor!!.parts!!.map { it.accepted }) + // markAccepted reset the streak to 0, so the cut part's step makes it 1. + assertEquals(1, e.backoffStreak) + assertEquals(3, e.attempts) + assertEquals(cutRequestId, e.lastRequestId) + assertEquals(listOf("e1" to host.clock + 1_000), scheduler.wakes) + assertEquals(emptyList(), journal.unacknowledged()) + } + + @Test + fun `a part that ends with a cancel of its own does not let the entry complete`() { + chunked() + host.handler = { r, _ -> + // Not the run's cancel: only this part's coroutine ends. + if (r.url.endsWith("start=10")) throw CancellationException("part only") + UploadResponse(200, "", mapOf()) + } + assertThrows(EntryRun.PartsNotAcceptedException::class.java) { run() } + val e = store.load("e1")!! + assertEquals(EntryState.QUEUED, e.state) + assertEquals(listOf(true, false, true), e.descriptor!!.parts!!.map { it.accepted }) + assertEquals(emptyList(), journal.unacknowledged()) + } + + @Test + fun `the entry keeps the highest backoff step of its concurrent parts`() { + chunked() + lateinit var job: Job + var sleeps = 0 + var part0Calls = 0 + host.stop = StopKind.TIME_LIMIT + host.handler = { r, _ -> + when { + // Part 0: two 503s (streak 2), then its third request is on its way at the stop. + r.url.endsWith("start=0") -> if (++part0Calls <= 2) UploadResponse(503, "", mapOf()) else awaitCancellation() + // Part 1: one 503 (streak 1), saved after part 0's streak 2. + r.url.endsWith("start=10") -> UploadResponse(503, "", mapOf()) + else -> UploadResponse(200, "", mapOf()) + } + } + // The stop lands in part 1's wait, the third wait of the run. + host.onSleep = { if (++sleeps == 3) job.cancel(CancellationException("timeout")) } + runBlocking { + job = launch { EntryRun("e1", store, ops, host).run() } + job.join() + } + assertEquals(listOf(1_000L, 2_000L, 1_000L), host.sleeps) + // Part 0's request was cut, so one step on top of the highest saved streak: 2 + 1. + assertEquals(3, store.load("e1")!!.backoffStreak) + } + @Test fun `a part 401 parks the whole entry and keeps the accepted parts`() { chunked() diff --git a/android/src/test/java/ai/openspace/backgroundupload/EntryTransitionsTest.kt b/android/src/test/java/ai/openspace/backgroundupload/EntryTransitionsTest.kt index 58ad3bbf..0d6eec64 100644 --- a/android/src/test/java/ai/openspace/backgroundupload/EntryTransitionsTest.kt +++ b/android/src/test/java/ai/openspace/backgroundupload/EntryTransitionsTest.kt @@ -86,7 +86,7 @@ class EntryTransitionsTest { @Test fun `a short backoff keeps the row running and shows the time, and the next attempt clears it`() { - val waiting = EntryTransitions.toBackingOff(entry(state = EntryState.RUNNING), 9_000, 5) + val waiting = EntryTransitions.toBackingOff(entry(state = EntryState.RUNNING), 9_000, 1, 5) assertEquals(EntryState.RUNNING, waiting.state) assertEquals(9_000L, waiting.nextAttemptAt) assertEquals(9_000.0, waiting.toRow().toMap()["nextAttemptAt"]) diff --git a/android/src/test/java/ai/openspace/backgroundupload/StopKindTest.kt b/android/src/test/java/ai/openspace/backgroundupload/StopKindTest.kt new file mode 100644 index 00000000..676f8c13 --- /dev/null +++ b/android/src/test/java/ai/openspace/backgroundupload/StopKindTest.kt @@ -0,0 +1,58 @@ +package ai.openspace.backgroundupload + +import androidx.work.WorkInfo +import org.junit.Assert.assertEquals +import org.junit.Test + +/** + * The stop-reason mapping. Each row pairs the WorkManager constant with the + * value the Android reference documents for it, so a changed constant fails + * here and not on a device. + */ +class StopKindTest { + private data class Row(val name: String, val constant: Int, val documented: Int, val kind: StopKind) + + private val table = listOf( + Row("TIMEOUT", WorkInfo.STOP_REASON_TIMEOUT, 3, StopKind.TIME_LIMIT), + Row("FOREGROUND_SERVICE_TIMEOUT", WorkInfo.STOP_REASON_FOREGROUND_SERVICE_TIMEOUT, -128, StopKind.TIME_LIMIT), + Row("CONSTRAINT_CONNECTIVITY", WorkInfo.STOP_REASON_CONSTRAINT_CONNECTIVITY, 7, StopKind.NETWORK), + Row("QUOTA", WorkInfo.STOP_REASON_QUOTA, 10, StopKind.QUOTA), + Row("APP_STANDBY", WorkInfo.STOP_REASON_APP_STANDBY, 12, StopKind.APP_STANDBY), + Row("BACKGROUND_RESTRICTION", WorkInfo.STOP_REASON_BACKGROUND_RESTRICTION, 11, StopKind.BACKGROUND_RESTRICTION), + Row("DEVICE_STATE", WorkInfo.STOP_REASON_DEVICE_STATE, 4, StopKind.DEVICE_STATE), + Row("USER", WorkInfo.STOP_REASON_USER, 13, StopKind.USER), + Row("CANCELLED_BY_APP", WorkInfo.STOP_REASON_CANCELLED_BY_APP, 1, StopKind.OTHER), + Row("PREEMPT", WorkInfo.STOP_REASON_PREEMPT, 2, StopKind.OTHER), + Row("CONSTRAINT_BATTERY_NOT_LOW", WorkInfo.STOP_REASON_CONSTRAINT_BATTERY_NOT_LOW, 5, StopKind.OTHER), + Row("CONSTRAINT_CHARGING", WorkInfo.STOP_REASON_CONSTRAINT_CHARGING, 6, StopKind.OTHER), + Row("CONSTRAINT_DEVICE_IDLE", WorkInfo.STOP_REASON_CONSTRAINT_DEVICE_IDLE, 8, StopKind.OTHER), + Row("CONSTRAINT_STORAGE_NOT_LOW", WorkInfo.STOP_REASON_CONSTRAINT_STORAGE_NOT_LOW, 9, StopKind.OTHER), + Row("SYSTEM_PROCESSING", WorkInfo.STOP_REASON_SYSTEM_PROCESSING, 14, StopKind.OTHER), + Row("ESTIMATED_APP_LAUNCH_TIME_CHANGED", WorkInfo.STOP_REASON_ESTIMATED_APP_LAUNCH_TIME_CHANGED, 15, StopKind.OTHER), + Row("NOT_STOPPED", WorkInfo.STOP_REASON_NOT_STOPPED, -256, StopKind.OTHER), + Row("UNKNOWN", WorkInfo.STOP_REASON_UNKNOWN, -512, StopKind.OTHER), + ) + + @Test + fun `each WorkManager constant has its documented value`() { + for (row in table) assertEquals(row.name, row.documented, row.constant) + } + + @Test + fun `each stop reason maps to its kind`() { + for (row in table) assertEquals(row.name, row.kind, StopKind.of(row.constant)) + } + + @Test + fun `a reason a later Android adds is OTHER`() { + assertEquals(StopKind.OTHER, StopKind.of(99)) + } + + @Test + fun `only quota, standby, background restriction, device state and user are logged`() { + assertEquals( + setOf(StopKind.QUOTA, StopKind.APP_STANDBY, StopKind.BACKGROUND_RESTRICTION, StopKind.DEVICE_STATE, StopKind.USER), + StopKind.values().filter { it.logged }.toSet(), + ) + } +} diff --git a/android/src/test/java/ai/openspace/backgroundupload/TestSupport.kt b/android/src/test/java/ai/openspace/backgroundupload/TestSupport.kt index bfc98d08..db56629e 100644 --- a/android/src/test/java/ai/openspace/backgroundupload/TestSupport.kt +++ b/android/src/test/java/ai/openspace/backgroundupload/TestSupport.kt @@ -111,6 +111,10 @@ internal class RecordingEvents : QueueEvents { internal class FakeScheduler : WorkScheduler { val scheduled = mutableListOf() val wakes = mutableListOf>() + /** The replace argument of every scheduleWake, in order. REPLACE makes a fresh row. */ + val wakeReplaces = mutableListOf() + /** How many of the next scheduleWake calls throw, as a failed WorkManager enqueue. */ + var wakeFailures = 0 val cancelled = mutableListOf() val cancelledWakes = mutableListOf() /** What libraryRuns() returns. */ @@ -121,7 +125,12 @@ internal class FakeScheduler : WorkScheduler { } override fun scheduleWake(entry: QueueEntry, at: Long, replace: Boolean) { + if (wakeFailures > 0) { + wakeFailures-- + throw IllegalStateException("enqueue failed") + } wakes += entry.id to at + wakeReplaces += replace } override fun cancel(id: String) { diff --git a/android/src/test/java/ai/openspace/backgroundupload/WorkerOpsTest.kt b/android/src/test/java/ai/openspace/backgroundupload/WorkerOpsTest.kt index 86d8c33d..4a3c2eea 100644 --- a/android/src/test/java/ai/openspace/backgroundupload/WorkerOpsTest.kt +++ b/android/src/test/java/ai/openspace/backgroundupload/WorkerOpsTest.kt @@ -438,7 +438,7 @@ class WorkerOpsTest { @Test fun `a short backoff shows nextAttemptAt on the running row until the next attempt`() { store.save(entry(state = EntryState.RUNNING)) - ops.backingOff("e1", 1, nextAttemptAt = 24_000) + ops.backingOff("e1", 1, nextAttemptAt = 24_000, streak = 1) val waiting = store.load("e1")!! assertEquals(EntryState.RUNNING, waiting.state) assertEquals(24_000L, waiting.nextAttemptAt) @@ -459,7 +459,7 @@ class WorkerOpsTest { @Test fun `a short backoff on an entry the run no longer owns changes nothing`() { store.save(entry(state = EntryState.PAUSED)) - ops.backingOff("e1", 1, nextAttemptAt = 24_000) + ops.backingOff("e1", 1, nextAttemptAt = 24_000, streak = 1) assertNull(store.load("e1")!!.nextAttemptAt) assertEquals(emptyList(), events.log) }