From 3b549482be0f047b5edd1a4e70792a2f3a18f2ed Mon Sep 17 00:00:00 2001 From: Dylan Murphy Date: Tue, 17 Nov 2026 19:30:24 -0500 Subject: [PATCH 01/12] Android: an upload with no network sleeps on a wake that waits for the network On API 30 and later, a run with no foreground mode that finds no usable network waits at most 60 s, then goes back to queued and sleeps on a wake row with a WorkManager network rule (CONNECTED, or a validated Wi-Fi request for a Wi-Fi-only entry). A run with foreground mode waits at most 30 minutes first. No attempt, backoff step, wait cause or nextAttemptAt is written. A run hands over only while none of its requests is sending. Backoff wakes and the wake after a time-limit stop also wait for a network; a parked entry's expiry wake does not. API 29 keeps the poll and unconstrained wakes. The sweep, resume() and a same-id enqueue keep a network wake; retryNow() adds a main row next to it only when the network fits now. setWifiOnly(false) runs the queued entries that follow it. The boot record counts entries by wake kind and reports wifiOnly. A Wi-Fi-only upload also waits while the process is bound to a network that is not Wi-Fi. Co-Authored-By: Claude Fable 5.2 --- .../backgroundupload/BootSnapshot.kt | 30 ++- .../ai/openspace/backgroundupload/EntryRun.kt | 97 ++++++++- .../openspace/backgroundupload/EntryWorker.kt | 21 +- .../backgroundupload/QueueController.kt | 67 ++++-- .../openspace/backgroundupload/Scheduler.kt | 173 ++++++++++++--- .../backgroundupload/TransferHost.kt | 14 +- .../backgroundupload/UploadNotification.kt | 36 ++-- .../backgroundupload/UploaderModule.kt | 5 +- .../openspace/backgroundupload/WorkerOps.kt | 45 +++- .../backgroundupload/EntryRunTest.kt | 201 +++++++++++++++++- .../backgroundupload/MainChainTest.kt | 175 ++++++++++++++- .../backgroundupload/QueueControllerTest.kt | 96 +++++++++ .../backgroundupload/RequiredNetworkTest.kt | 68 ++++++ .../openspace/backgroundupload/TestSupport.kt | 39 +++- 14 files changed, 973 insertions(+), 94 deletions(-) create mode 100644 android/src/test/java/ai/openspace/backgroundupload/RequiredNetworkTest.kt diff --git a/android/src/main/java/ai/openspace/backgroundupload/BootSnapshot.kt b/android/src/main/java/ai/openspace/backgroundupload/BootSnapshot.kt index 199bf22d..74dadc99 100644 --- a/android/src/main/java/ai/openspace/backgroundupload/BootSnapshot.kt +++ b/android/src/main/java/ai/openspace/backgroundupload/BootSnapshot.kt @@ -7,7 +7,9 @@ package ai.openspace.backgroundupload * A queued or running entry with no run at all shows in entriesWithoutRun. * v9Leftovers counts the v9 work still on disk, and orphans the * directories with no entry file that are not v9: what the boot cleanup - * left, and what it deleted. Zeros included. + * left, and what it deleted. Zeros included. wifiOnly is the queue's + * setting, and entriesByWake counts the queued entries by the wake they + * sleep on: a backoff wait, the network, or Wi-Fi. */ internal object BootSnapshot { const val MESSAGE = "boot sweep" @@ -22,6 +24,7 @@ internal object BootSnapshot { runs: List?, now: Long, leftovers: Leftovers, + wifiOnly: Boolean, ): Map { val base = mapOf( "entries" to entries.size, @@ -30,6 +33,7 @@ internal object BootSnapshot { "oldestUpdatedAt" to entries.minOfOrNull { it.updatedAt }, "v9Leftovers" to leftovers.v9.toMap(), "orphans" to leftovers.orphans.toMap(), + "wifiOnly" to wifiOnly, ) if (runs == null) return base + ("runsUnavailable" to true) val withRun = runs.mapNotNull { it.entryId }.toSet() @@ -44,6 +48,30 @@ internal object BootSnapshot { run.delayed && run.nextScheduleAt != null && run.nextScheduleAt - now > EXPIRY_WAKE_HORIZON_MS }, "maxRunAttemptCount" to runs.maxOfOrNull { it.runAttemptCount }, + "entriesByWake" to entriesByWake(entries, runs), ) } + + /** + * Each queued entry once, by its wake: "backoff" a delayed wake (a + * backoff wait), else "wifi" a wake that waits for Wi-Fi, else "network" + * one that waits for any network. An entry with none of them is not + * counted. Only a wake has a delay or a network rule. + */ + private fun entriesByWake(entries: List, runs: List): Map { + val byEntry = runs.filter { it.entryId != null }.groupBy { it.entryId } + val counts = linkedMapOf("backoff" to 0, "network" to 0, "wifi" to 0) + for (e in entries) { + if (e.state != EntryState.QUEUED) continue + val own = byEntry[e.id].orEmpty() + val kind = when { + own.any { it.delayed } -> "backoff" + own.any { it.network == RequiredNetwork.WIFI } -> "wifi" + own.any { it.network == RequiredNetwork.CONNECTED } -> "network" + else -> continue + } + counts[kind] = counts.getValue(kind) + 1 + } + return counts + } } diff --git a/android/src/main/java/ai/openspace/backgroundupload/EntryRun.kt b/android/src/main/java/ai/openspace/backgroundupload/EntryRun.kt index cb048b9e..c47e9a9c 100644 --- a/android/src/main/java/ai/openspace/backgroundupload/EntryRun.kt +++ b/android/src/main/java/ai/openspace/backgroundupload/EntryRun.kt @@ -39,6 +39,43 @@ internal class EntryRun( class ExpiredException : Exception("expired before completion") + /** + * The run waited long enough for the network and hands the wait to a + * WorkManager wake that waits for [network] ([WorkerOps.awaitNetwork]). + */ + class AwaitNetworkException(val network: RequiredNetwork) : Exception("waiting for the network") + + /** + * Keeps a hand-over away from a request on its way. Chunked parts run + * side by side, and a part that hands over stops its siblings. So a part + * hands over only when no request of the run is sending, and once it does, + * no part starts a request: [beginSend] throws instead. + */ + class SendGate { + private var sending = 0 + private var handOver: RequiredNetwork? = null + + /** Before the write-ahead of a request. Throws when a hand-over began. */ + @Synchronized + fun beginSend() { + handOver?.let { throw AwaitNetworkException(it) } + sending++ + } + + @Synchronized + fun endSend() { + sending-- + } + + /** True when no request is sending; from then on no request starts. */ + @Synchronized + fun tryHandOver(network: RequiredNetwork): Boolean { + if (sending > 0) return false + handOver = network + return true + } + } + /** The entry was paused (whole queue or its key) between the module's pause and the work cancel reaching us. */ class PausedException : Exception("paused") @@ -78,6 +115,19 @@ internal class EntryRun( /** The poll while the network is unusable (offline, or waiting for wifi). */ const val CONNECTIVITY_POLL_MS = 10_000L + /** + * How long a run with no foreground mode waits for the network before it + * hands over to a wake that waits for it. JobScheduler counts the wait + * against the app's job quota, and Android may stop the job meanwhile. + */ + const val HEADLESS_NETWORK_WAIT_MS = 60_000L + + /** + * The same for a run that holds foreground mode. On Android 15 and later + * each minute of it uses the dataSync budget of 6 hours in 24. + */ + const val FOREGROUND_NETWORK_WAIT_MS = 30 * 60_000L + /** The poll while a short backoff remainder runs out before the run starts. */ private const val WAIT_POLL_MS = 10_000L @@ -96,6 +146,12 @@ internal class EntryRun( var liveBytes = 0L private set + /** Whether the run holds foreground mode. Once it does, it keeps it until the worker ends. */ + @Volatile + private var holdsForeground = false + + private val sends = SendGate() + /** A cancel reached a request after its write-ahead. Chunked parts set it from their own coroutines. */ @Volatile private var stopCutRequest = false @@ -130,7 +186,7 @@ internal class EntryRun( stopCutRequest = false if (!first) current = ops.latest(entryId, generation) first = false - host.foreground(current) + holdsForeground = host.foreground(current) || holdsForeground val settlement = transfer(current) endProgress(completed = settlement is Settlement.Completed) ops.settle(entryId, generation, settlement) @@ -153,6 +209,12 @@ internal class EntryRun( } catch (error: PausedException) { endProgress(completed = false) return + } catch (wait: AwaitNetworkException) { + // No request is on its way (SendGate). A wake that fails to enqueue + // throws, and the worker returns retry. + endProgress(completed = false) + ops.awaitNetwork(entryId, generation, wait.network) + return } catch (error: CancellationException) { // A system stop moves a running entry back to queued. A pause or a // cancel already moved it; then this does nothing. @@ -295,14 +357,29 @@ internal class EntryRun( * Waits until the network fits the entry's Wi-Fi rule: its own wifiOnly, * else the queue's setting. Re-reads the settings and the entry at every * poll, so a setWifiOnly() toggle reaches an entry that follows it. + * + * On API 30 and later the run does not poll without end. After + * [HEADLESS_NETWORK_WAIT_MS] with no foreground mode, or + * [FOREGROUND_NETWORK_WAIT_MS] with it, it throws [AwaitNetworkException] + * and the entry sleeps on a wake that WorkManager starts once the network + * is up. It hands over only while no request of the run is sending + * ([SendGate]). API 29 keeps the poll: there WorkManager registers one + * network callback per constrained row. */ suspend fun waitForNetwork() { + val since = host.now() while (true) { val s = ops.settings() val entry = ops.latest(entryId, generation) if (s.isPaused(entry.key)) throw PausedException() if (RetryClassifier.isExpired(host.now(), entry.expiresAt)) throw ExpiredException() - if (host.connectivity(s.wifiOnlyFor(entry)) == Connectivity.Ok) return + val wifiOnly = s.wifiOnlyFor(entry) + if (host.connectivity(wifiOnly) == Connectivity.Ok) return + val limit = if (holdsForeground) FOREGROUND_NETWORK_WAIT_MS else HEADLESS_NETWORK_WAIT_MS + if (host.networkWakes() && host.now() - since >= limit) { + val network = if (wifiOnly) RequiredNetwork.WIFI else RequiredNetwork.CONNECTED + if (sends.tryHandOver(network)) throw AwaitNetworkException(network) + } host.sleep(CONNECTIVITY_POLL_MS) } } @@ -333,11 +410,17 @@ internal class EntryRun( var failure: Throwable? = null val response = try { host.withSlot(highPriority) { - val e = ops.recordAttempt(entryId, generation, requestId) - written = e - val d = e.descriptor!! - val part = partIndex?.let { d.parts!![it] } - host.send(TransferRequest(part?.url ?: d.url!!, d.method, headersFor(d, part, requestId), body(d, part)), onProgress) + // A sibling part that handed over meanwhile: no request starts. + sends.beginSend() + try { + val e = ops.recordAttempt(entryId, generation, requestId) + written = e + val d = e.descriptor!! + val part = partIndex?.let { d.parts!![it] } + host.send(TransferRequest(part?.url ?: d.url!!, d.method, headersFor(d, part, requestId), body(d, part)), onProgress) + } finally { + sends.endSend() + } } } catch (error: CancellationException) { // A stop after the write-ahead cut a request that was on its way. diff --git a/android/src/main/java/ai/openspace/backgroundupload/EntryWorker.kt b/android/src/main/java/ai/openspace/backgroundupload/EntryWorker.kt index 23cedff8..c0a981d3 100644 --- a/android/src/main/java/ai/openspace/backgroundupload/EntryWorker.kt +++ b/android/src/main/java/ai/openspace/backgroundupload/EntryWorker.kt @@ -50,6 +50,10 @@ open class EntryWorker(protected val context: Context, params: WorkerParameters) @Volatile private var showsNotification = false + /** setForeground succeeded in this run. WorkManager keeps the run in foreground mode until it ends. */ + @Volatile + private var inForeground = false + /** When this run began. WorkManager set the row RUNNING before it. */ private var startedAt = 0L @@ -93,11 +97,14 @@ open class EntryWorker(protected val context: Context, params: WorkerParameters) return connectivity } - override suspend fun foreground(entry: QueueEntry) { + override suspend fun foreground(entry: QueueEntry): Boolean { showsNotification = entry.descriptor?.noNotification == false - startForeground() + if (startForeground()) inForeground = true + return inForeground } + override fun networkWakes() = Build.VERSION.SDK_INT >= Build.VERSION_CODES.R + override fun progressStarted(id: String, total: Long, sent: Long) { UploadProgress.add(id, total) UploadProgress.set(id, sent) @@ -133,16 +140,18 @@ open class EntryWorker(protected val context: Context, params: WorkerParameters) // WorkManager relaunch) is not a failure; the transfer runs without // foreground priority, under JobScheduler's time limit (see // [EntryRun.releaseAfterTimeout]). Any other failure is logged and the run - // goes on. - private suspend fun startForeground() { - if (!showsNotification) return - try { + // goes on. Returns whether setForeground succeeded. + private suspend fun startForeground(): Boolean { + if (!showsNotification) return false + return try { ensureNotificationChannel(notificationManager, config) setForeground(getForegroundInfo()) + true } catch (error: CancellationException) { throw error } catch (error: Throwable) { if (!isForegroundStartDenied(error)) Diag.warn("foreground start failed; running without it", error) + false } } diff --git a/android/src/main/java/ai/openspace/backgroundupload/QueueController.kt b/android/src/main/java/ai/openspace/backgroundupload/QueueController.kt index 8cc44b37..300f202e 100644 --- a/android/src/main/java/ai/openspace/backgroundupload/QueueController.kt +++ b/android/src/main/java/ai/openspace/backgroundupload/QueueController.kt @@ -21,6 +21,8 @@ class QueueController( private val scheduler: WorkScheduler, private val isWorkerRunning: (id: String) -> Boolean = WorkerGate::isRunning, private val clock: () -> Long = System::currentTimeMillis, + /** Whether the network now fits the entry's Wi-Fi rule, as a run checks it. */ + private val networkFits: (QueueEntry) -> Boolean = { true }, ) { // MARK: - enqueue @@ -249,10 +251,14 @@ class QueueController( * failure, is scheduled only when no row of it can start now * ([WorkScheduler.rowSummary], one query): that is not a retry, it only * makes sure a row exists, and a row WorkManager holds back after a stop - * is replaced ([WorkScheduler.schedule]). A queued entry whose wait stays - * and that has no delayed wake row gets its wake again (KEEP), also when - * it has a main row: WorkManager may hold that row back for hours. Running, paused, parked and - * settled entries are left alone. A queued entry sends nothing, and the + * is replaced ([WorkScheduler.schedule]). An entry that sleeps on a wake + * that waits for the network gets a main row only when the network fits + * now ([networkFits]); offline, the wake stays its only row, so a + * foreground does not start a run that polls and hands over again. A + * queued entry whose wait stays and that has no delayed wake row gets its + * wake again (KEEP), also when it has a main row: WorkManager may hold + * that row back for hours. Running, paused, parked and settled entries + * are left alone. A queued entry sends nothing, and the * per-id gate holds any new row while a worker of the entry still winds * down, so this never starts a second transfer next to one in progress. * @@ -292,12 +298,17 @@ class QueueController( val waiting = queued.filter { it.nextAttemptAt != null && it.id !in clearedIds } if (idle.isNotEmpty() || waiting.isNotEmpty()) { val rows = runCatching { scheduler.rowSummary() }.onFailure(::failed).getOrNull() - idle.filter { rows == null || it.id !in rows.startable } + // An entry that sleeps on a network wake waits for WorkManager while + // the network does not fit. When it fits, a main row starts the entry + // now too, so a start never depends on WorkManager's constraint + // tracking alone. The per-id gate runs the two rows one after the + // other, and the second finds the entry no longer queued. + idle.filter { rows == null || (it.id !in rows.startable && (it.id !in rows.networkWait || networkFits(it))) } .forEach { e -> runCatching { scheduleRun(e) }.onFailure(::failed) } // A wait that stays needs its wake. KEEP leaves one that exists, so // this is at most one enqueue per entry that lacks a wake. waiting.filter { rows == null || it.id !in rows.withWake } - .forEach { e -> runCatching { scheduler.scheduleWake(e, e.nextAttemptAt!!, replace = false) }.onFailure(::failed) } + .forEach { e -> runCatching { scheduler.scheduleWake(e, e.nextAttemptAt!!, replace = false, requiredNetwork(e, settings.load())) }.onFailure(::failed) } } cleared.forEach { events.state(it.toRow()) } first?.let { Diag.error("retryNow: $failures entry saves or WorkManager calls failed; the next app start or foreground tries again", it) } @@ -406,9 +417,28 @@ class QueueController( * The settings calls (this one, [pause], [resume], [configureRetry], * [updateHeaders]) never throw for a failed settings write. See * [QueueSettingsStore.update]. + * + * A change to off runs now every queued entry that follows the setting + * and has no wait and no worker: such an entry may sleep on a wake that + * waits for Wi-Fi, which no longer fits. That wake is cancelled once the + * main row is made. A change to on needs no work: a run that starts on + * another network fails the check and hands over with the Wi-Fi rule. + * The same value schedules nothing. A failed WorkManager call is logged + * and the next entry goes on. */ fun setWifiOnly(enabled: Boolean) { - settings.update { it.copy(wifiOnly = enabled) } + val before = settings.load().wifiOnly + val after = settings.update { it.copy(wifiOnly = enabled) }.wifiOnly + if (!before || after) return + store.all() + .filter { + !it.legacy && it.state == EntryState.QUEUED && it.nextAttemptAt == null && + it.descriptor?.wifiOnly == null && !isWorkerRunning(it.id) + } + .forEach { e -> + runCatching { scheduleRun(e, keepNetworkWake = false) } + .onFailure { Diag.error("setWifiOnly: could not schedule '${e.id}'", it) } + } } fun configureRetry(defaults: RetryDefaults) { @@ -612,7 +642,7 @@ class QueueController( val removed = store.removeOrphans(counted.orphanDirs, now) Diag.info( BootSnapshot.MESSAGE, - BootSnapshot.of(found, runs, now, counted.afterCleanup(removed)), + BootSnapshot.of(found, runs, now, counted.afterCleanup(removed), s.wifiOnly), replaceUntaken = true, ) changed.forEach { events.state(it.toRow()) } @@ -632,26 +662,28 @@ class QueueController( * Runs [entry] when it is queued. A backoff longer than a worker waits goes * to the wake; a parked entry gets a wake at its expiry, so it settles * 'expired' on time, once that expiry is near (see [EXPIRY_WAKE_HORIZON_MS]). - * Any other wake of the entry is stale, so it is cancelled. + * Any other wake of the entry is stale, so it is cancelled, except a wake + * that waits for the network ([keepNetworkWake]): the entry may sleep on + * it, and it starts the entry when the network is back. */ - private fun scheduleRun(entry: QueueEntry, keepWake: Boolean = false) { + private fun scheduleRun(entry: QueueEntry, keepWake: Boolean = false, keepNetworkWake: Boolean = true) { val now = clock() when (entry.state) { EntryState.QUEUED -> { val at = entry.nextAttemptAt if (at != null && at - now > RetryClassifier.IN_WORKER_BACKOFF_MAX_MS) { - scheduler.scheduleWake(entry, at, replace = !keepWake) + scheduler.scheduleWake(entry, at, replace = !keepWake, requiredNetwork(entry, settings.load())) } else { // The main row first: these are two separate WorkManager calls, // and a process death between them must not leave the entry // with neither a wake nor a main row. scheduler.schedule(entry) - cancelStaleWake(entry.id) + cancelStaleWake(entry.id, keepNetworkWake) } } EntryState.AWAITING_AUTH -> if (expiryWakeIsNear(entry.expiresAt, now)) { - scheduler.scheduleWake(entry, entry.expiresAt, replace = !keepWake) + scheduler.scheduleWake(entry, entry.expiresAt, replace = !keepWake, requiredNetwork(entry, settings.load())) } else { cancelStaleWake(entry.id) } @@ -665,13 +697,16 @@ class QueueController( // // The caller's entry can be a stale snapshot: a worker may have released // the entry with a long wait and made its wake since. So the stored entry - // is read again, and a wake it still needs is kept. - private fun cancelStaleWake(id: String) { + // is read again, and a wake it still needs is kept. A queued entry with no + // long wait may sleep on a wake that waits for the network (a run handed + // over, or a worker did that since the snapshot): that wake is kept too, + // unless [keepNetworkWake] is false. + private fun cancelStaleWake(id: String, keepNetworkWake: Boolean = true) { if (isWorkerRunning(id)) return val stored = runCatching { store.load(id) }.getOrNull() val at = stored?.nextAttemptAt if (stored?.state == EntryState.QUEUED && at != null && at - clock() > RetryClassifier.IN_WORKER_BACKOFF_MAX_MS) return - scheduler.cancelWake(id) + scheduler.cancelWake(id, keepNetworkWake = keepNetworkWake && stored?.state == EntryState.QUEUED) } /** The entries [transformAll] saved, and the saves that failed. */ diff --git a/android/src/main/java/ai/openspace/backgroundupload/Scheduler.kt b/android/src/main/java/ai/openspace/backgroundupload/Scheduler.kt index 8e2761c8..7c982745 100644 --- a/android/src/main/java/ai/openspace/backgroundupload/Scheduler.kt +++ b/android/src/main/java/ai/openspace/backgroundupload/Scheduler.kt @@ -1,7 +1,12 @@ package ai.openspace.backgroundupload import android.content.Context +import android.net.NetworkCapabilities +import android.net.NetworkRequest +import android.os.Build +import androidx.work.Constraints import androidx.work.ExistingWorkPolicy +import androidx.work.NetworkType import androidx.work.OneTimeWorkRequest import androidx.work.OneTimeWorkRequestBuilder import androidx.work.WorkInfo @@ -51,6 +56,34 @@ internal object WorkManagerBackoff { } } +/** + * The network a wake row waits for, as a WorkManager constraint. Both + * network rules need a validated network, as the in-worker check does. + * + * - NONE: no constraint. + * - CONNECTED: NetworkType.CONNECTED. WorkManager 2.12.0 turns it into a + * request with INTERNET and VALIDATED (NetworkTypeCompatKt.toNetworkRequest). + * - WIFI: a NetworkRequest with TRANSPORT_WIFI, INTERNET and VALIDATED. + */ +enum class RequiredNetwork { NONE, CONNECTED, WIFI } + +/** + * The network rule of a wake row for [entry], as saved. + * + * - A parked entry's wake only settles its expiry: NONE. + * - A backoff wait (nextAttemptAt set): CONNECTED. The run checks the Wi-Fi + * rule and hands over again when it must, so a backoff wake never keeps a + * stale Wi-Fi rule. + * - Otherwise the entry waits for the network its run needs: WIFI when its + * own wifiOnly, else the queue's setting, says so; else CONNECTED. + */ +internal fun requiredNetwork(entry: QueueEntry, settings: QueueSettings): RequiredNetwork = when { + entry.state == EntryState.AWAITING_AUTH -> RequiredNetwork.NONE + entry.nextAttemptAt != null -> RequiredNetwork.CONNECTED + settings.wifiOnlyFor(entry) -> RequiredNetwork.WIFI + else -> RequiredNetwork.CONNECTED +} + /** What [RowScheduler.schedule] does to an entry's main chain. */ internal enum class MainChainStep { NONE, APPEND, REPLACE } @@ -108,17 +141,22 @@ interface WorkScheduler { fun schedule(entry: QueueEntry) /** - * A run at [at] under a second unique name. Used for long backoffs and for - * the expiry of a parked entry. [replace] false keeps a wake that already - * exists (the boot sweep). + * A run at [at] under a second unique name. Used for long backoffs, for + * the expiry of a parked entry, and for an entry that waits for the + * network ([network], see [requiredNetwork]). [replace] false keeps a wake + * that already exists (the boot sweep). */ - fun scheduleWake(entry: QueueEntry, at: Long, replace: Boolean) + fun scheduleWake(entry: QueueEntry, at: Long, replace: Boolean, network: RequiredNetwork) /** Cancels the main chain and the wake. */ fun cancel(id: String) - /** Cancels the wake only. A wake that is no longer needed still holds a WorkManager slot. */ - fun cancelWake(id: String) + /** + * Cancels the wake only. A wake that is no longer needed still holds a + * WorkManager slot. [keepNetworkWake] keeps a wake with no delay that + * waits for a network: the wake of an entry that waits for the network. + */ + fun cancelWake(id: String, keepNetworkWake: Boolean = false) /** Every unfinished WorkManager run of the library, main chains and wakes. For the boot snapshot. */ fun libraryRuns(): List @@ -128,16 +166,24 @@ interface WorkScheduler { } /** - * [startable]: the ids with a run in progress, or a row WorkManager may - * start now. [withWake]: the ids with an unfinished delayed row. Only a wake - * has an initial delay; a main-chain row never does. + * [startable]: the ids with a run in progress, or a row with no network rule + * that WorkManager may start now. [networkWait]: the ids with a due wake + * that waits for a network ([isNetworkWake]); WorkManager starts it once + * its network is up, which the summary cannot see. [withWake]: the ids with + * an unfinished delayed row or network wake. Only a wake has an initial + * delay or a network constraint; a main-chain row has neither. */ -data class RowSummary(val startable: Set, val withWake: Set) +data class RowSummary( + val startable: Set, + val withWake: Set, + val networkWait: Set = emptySet(), +) /** * One unfinished WorkManager run. [entryId] comes from the run's id tag. * [delayed] is a run enqueued with an initial delay, which only a wake has. - * [nextScheduleAt] is null when WorkManager reports none. + * [nextScheduleAt] is null when WorkManager reports none. [network] is its + * network constraint; only a wake has one. */ data class PendingRun( val entryId: String?, @@ -145,12 +191,16 @@ data class PendingRun( val runAttemptCount: Int, val nextScheduleAt: Long?, val delayed: Boolean, + val network: RequiredNetwork = RequiredNetwork.NONE, ) /** * One unfinished WorkManager row. [nextScheduleAt] is the earliest time * WorkManager starts it (`WorkInfo.nextScheduleTimeMillis`): Long.MAX_VALUE - * for a row that is not ENQUEUED. + * for a row that is not ENQUEUED. WorkManager 2.12.0 leaves the network + * constraint out of that time (WorkSpec.WorkInfoPojo.calculateNextRunTimeMillis): + * a constrained row reports its enqueue time plus its delay, and starts + * once that time has come and its [network] is up. */ data class WorkRow( val entryId: String?, @@ -158,8 +208,12 @@ data class WorkRow( val runAttemptCount: Int, val nextScheduleAt: Long, val delayed: Boolean, + val network: RequiredNetwork = RequiredNetwork.NONE, ) +/** A wake with no delay that waits for a network: an entry that waits for the network sleeps on it. */ +internal val WorkRow.isNetworkWake get() = !delayed && network != RequiredNetwork.NONE + /** The WorkManager calls [RowScheduler] makes. Tests run the same rules against a fake. */ interface WorkRows { /** The unfinished rows of one unique name. */ @@ -168,7 +222,7 @@ interface WorkRows { /** Every row of the library, main chains and wakes, in one of [states]. One query. */ fun library(states: List): List - fun enqueue(name: String, policy: ExistingWorkPolicy, entry: QueueEntry, delayMs: Long) + fun enqueue(name: String, policy: ExistingWorkPolicy, entry: QueueEntry, delayMs: Long, network: RequiredNetwork) fun cancel(name: String) } @@ -187,8 +241,22 @@ interface WorkRows { * back every later "run now" appended behind it. They use the wake name * (`#wake`) with an initial delay instead. * - * No WorkManager Constraints: connectivity and wifi-only are checked inside - * the worker, as v9, because the constraint path was unreliable. + * Main-chain rows carry no WorkManager constraint. A constrained row does + * not start while the app is visible and offline, so it could not take + * foreground mode then; its constraint is fixed when it is made, so a + * setWifiOnly() change would leave stale rows; and on API 29 WorkManager + * registers one network callback per constrained row. Only a wake row + * carries a network rule ([RequiredNetwork]): a backoff wake, and the wake + * of an entry whose run found no network ([EntryRun.waitForNetwork]). The + * worker checks the network itself before every attempt, also on a row a + * constraint started. + * + * Earlier versions used no constraint at all, because the constraint path + * was unreliable. The WorkManager release notes for 2.10.1 to 2.12.0 list + * fixes for network constraints that were not met while the network was + * up, a stale capability set, an identical second constraint, and a + * blocked network (Android 15) that passed the constraint. The library is + * on 2.12.0. */ open class RowScheduler( private val rows: WorkRows, @@ -205,18 +273,19 @@ open class RowScheduler( override fun schedule(entry: QueueEntry) { when (mainChainStep(rows.unfinished(entry.id), clock())) { MainChainStep.NONE -> Unit - MainChainStep.APPEND -> rows.enqueue(entry.id, ExistingWorkPolicy.APPEND_OR_REPLACE, entry, 0L) - MainChainStep.REPLACE -> rows.enqueue(entry.id, ExistingWorkPolicy.REPLACE, entry, 0L) + MainChainStep.APPEND -> rows.enqueue(entry.id, ExistingWorkPolicy.APPEND_OR_REPLACE, entry, 0L, RequiredNetwork.NONE) + MainChainStep.REPLACE -> rows.enqueue(entry.id, ExistingWorkPolicy.REPLACE, entry, 0L, RequiredNetwork.NONE) } } - override fun scheduleWake(entry: QueueEntry, at: Long, replace: Boolean) { + override fun scheduleWake(entry: QueueEntry, at: Long, replace: Boolean, network: RequiredNetwork) { val delay = WorkManagerScheduler.initialDelayMs(at, clock()) rows.enqueue( WorkManagerScheduler.wakeName(entry.id), if (replace) ExistingWorkPolicy.REPLACE else ExistingWorkPolicy.KEEP, entry, delay, + network, ) } @@ -227,9 +296,12 @@ open class RowScheduler( // A cancel reschedules all pending work even when the name has none, and // the boot sweep calls this for every queued entry. - override fun cancelWake(id: String) { + override fun cancelWake(id: String, keepNetworkWake: Boolean) { val name = WorkManagerScheduler.wakeName(id) - if (rows.unfinished(name).isNotEmpty()) rows.cancel(name) + val open = rows.unfinished(name) + if (open.isEmpty()) return + if (keepNetworkWake && open.any { it.isNetworkWake }) return + rows.cancel(name) } // One query for every row, not one per entry: the boot sweep runs this on @@ -242,6 +314,7 @@ open class RowScheduler( runAttemptCount = row.runAttemptCount, nextScheduleAt = row.nextScheduleAt.takeIf { it != Long.MAX_VALUE }, delayed = row.delayed, + network = row.network, ) } @@ -249,8 +322,10 @@ open class RowScheduler( val now = clock() val open = rows.library(UNFINISHED) return RowSummary( - startable = open.filter { it.state == WorkInfo.State.RUNNING || it.startsBy(now) }.mapNotNull { it.entryId }.toSet(), - withWake = open.filter { it.delayed }.mapNotNull { it.entryId }.toSet(), + startable = open.filter { it.state == WorkInfo.State.RUNNING || (it.startsBy(now) && !it.isNetworkWake) } + .mapNotNull { it.entryId }.toSet(), + withWake = open.filter { it.delayed || it.network != RequiredNetwork.NONE }.mapNotNull { it.entryId }.toSet(), + networkWait = open.filter { it.isNetworkWake && it.startsBy(now) }.mapNotNull { it.entryId }.toSet(), ) } } @@ -296,8 +371,8 @@ private class WorkManagerRows(private val workManager: WorkManager) : WorkRows { WorkQuery.Builder.fromTags(listOf(WorkManagerScheduler.WORK_TAG)).addStates(states).build(), ).get().map(::row) - override fun enqueue(name: String, policy: ExistingWorkPolicy, entry: QueueEntry, delayMs: Long) { - workManager.beginUniqueWork(name, policy, request(entry, delayMs)).enqueue() + override fun enqueue(name: String, policy: ExistingWorkPolicy, entry: QueueEntry, delayMs: Long, network: RequiredNetwork) { + workManager.beginUniqueWork(name, policy, request(entry, delayMs, network)).enqueue() } override fun cancel(name: String) { @@ -311,9 +386,21 @@ private class WorkManagerRows(private val workManager: WorkManager) : WorkRows { runAttemptCount = info.runAttemptCount, nextScheduleAt = info.nextScheduleTimeMillis, delayed = info.initialDelayMillis > 0, + network = networkOf(info.constraints), ) - private fun request(entry: QueueEntry, delayMs: Long): OneTimeWorkRequest { + // On API 28 and later setRequiredNetworkRequest stores the request and + // sets the type to NOT_REQUIRED (Constraints.Builder in 2.12.0). + private fun networkOf(c: Constraints): RequiredNetwork { + val request = c.requiredNetworkRequest + return when { + request != null && request.hasTransport(NetworkCapabilities.TRANSPORT_WIFI) -> RequiredNetwork.WIFI + request != null || c.requiredNetworkType != NetworkType.NOT_REQUIRED -> RequiredNetwork.CONNECTED + else -> RequiredNetwork.NONE + } + } + + private fun request(entry: QueueEntry, delayMs: Long, network: RequiredNetwork): OneTimeWorkRequest { val builder = if (entry.body?.kind == StagedBody.CHUNKED) { OneTimeWorkRequestBuilder() } else { @@ -327,6 +414,42 @@ private class WorkManagerRows(private val workManager: WorkManager) : WorkRows { .addTag(WorkManagerScheduler.ID_TAG_PREFIX + entry.id) .setInputData(workDataOf(WorkManagerScheduler.ENTRY_ID_KEY to entry.id)) .setInitialDelay(delayMs, TimeUnit.MILLISECONDS) + .setConstraints(constraints(network)) .build() } + + /** + * The Wi-Fi request is the in-worker check's rule: a validated Wi-Fi + * network. WorkManager uses the request on API 28 and later (minSdk is + * 29); UNMETERED is only its fallback for older API levels. A new + * NetworkRequest.Builder also asks for NOT_VPN and NOT_RESTRICTED; + * WorkManager 2.12.0 removes both from the request it builds for + * CONNECTED (NetworkTypeCompatKt.toNetworkRequest), and so does this one. + * + * API 29 gets no constraint, as before: there WorkManager tracks each + * constrained row with its own network callback (IndividualNetworkCallback), + * and the app's callbacks share a limit of 100. Its runs keep the + * in-worker poll ([TransferHost.networkWakes]). + */ + private fun constraints(network: RequiredNetwork): Constraints { + if (Build.VERSION.SDK_INT < Build.VERSION_CODES.R) return Constraints.NONE + return when (network) { + RequiredNetwork.NONE -> Constraints.NONE + RequiredNetwork.CONNECTED -> Constraints.Builder().setRequiredNetworkType(NetworkType.CONNECTED).build() + RequiredNetwork.WIFI -> Constraints.Builder() + .setRequiredNetworkRequest( + NetworkRequest.Builder() + .addTransportType(NetworkCapabilities.TRANSPORT_WIFI) + .addCapability(NetworkCapabilities.NET_CAPABILITY_INTERNET) + .addCapability(NetworkCapabilities.NET_CAPABILITY_VALIDATED) + // As WorkManager's own CONNECTED request: a VPN over Wi-Fi counts. + .removeCapability(NetworkCapabilities.NET_CAPABILITY_NOT_VPN) + .removeCapability(NetworkCapabilities.NET_CAPABILITY_NOT_RESTRICTED) + .build(), + NetworkType.UNMETERED, + ) + .build() + } + } + } diff --git a/android/src/main/java/ai/openspace/backgroundupload/TransferHost.kt b/android/src/main/java/ai/openspace/backgroundupload/TransferHost.kt index 5cd57653..8db21dfa 100644 --- a/android/src/main/java/ai/openspace/backgroundupload/TransferHost.kt +++ b/android/src/main/java/ai/openspace/backgroundupload/TransferHost.kt @@ -25,8 +25,18 @@ internal interface TransferHost { /** The network state for the queue's wifi-only setting. The notification shows it. */ fun connectivity(wifiOnly: Boolean): Connectivity - /** Foreground mode for an entry that shows the notification. Never throws. */ - suspend fun foreground(entry: QueueEntry) + /** + * Foreground mode for an entry that shows the notification. Never throws. + * Returns whether the run holds foreground mode. + */ + suspend fun foreground(entry: QueueEntry): Boolean + + /** + * Whether a run that finds no network may hand the wait to a wake row + * with a network constraint: API 30 and later. On API 29 WorkManager + * tracks each constrained row with its own network callback. + */ + fun networkWakes(): Boolean fun progressStarted(id: String, total: Long, sent: Long) diff --git a/android/src/main/java/ai/openspace/backgroundupload/UploadNotification.kt b/android/src/main/java/ai/openspace/backgroundupload/UploadNotification.kt index f1b6b90b..4a31a3e8 100644 --- a/android/src/main/java/ai/openspace/backgroundupload/UploadNotification.kt +++ b/android/src/main/java/ai/openspace/backgroundupload/UploadNotification.kt @@ -26,19 +26,31 @@ internal enum class Connectivity { NoWifi, NoInternet, Ok } @Synchronized internal fun validateConnectivity(context: Context, wifiOnly: Boolean): Connectivity { val manager = context.getSystemService(Context.CONNECTIVITY_SERVICE) as ConnectivityManager - val network = manager.activeNetwork - val capabilities = manager.getNetworkCapabilities(network) - - val hasInternet = capabilities?.hasCapability(NET_CAPABILITY_VALIDATED) == true - - // not wifiOnly, return early - if (!wifiOnly) return if (hasInternet) Connectivity.Ok else Connectivity.NoInternet + // activeNetwork is the system default network, not a network the process + // is bound to (ConnectivityManager docs). + val capabilities = manager.getNetworkCapabilities(manager.activeNetwork) + // While the app binds the process to a network (bindProcessToNetwork), new + // sockets and DNS use that network, also OkHttp's. + val bound = manager.boundNetworkForProcess + return connectivityOf( + wifiOnly = wifiOnly, + validated = capabilities?.hasCapability(NET_CAPABILITY_VALIDATED) == true, + defaultWifi = capabilities?.hasTransport(TRANSPORT_WIFI) == true, + boundWifi = bound?.let { manager.getNetworkCapabilities(it)?.hasTransport(TRANSPORT_WIFI) == true }, + ) +} - // handle wifiOnly - return if (hasInternet && capabilities?.hasTransport(TRANSPORT_WIFI) == true) - Connectivity.Ok - else - Connectivity.NoWifi // don't return NoInternet here, more direct to request to join wifi +/** + * The network check. The default network must be validated. Wi-Fi only + * also needs it to be Wi-Fi, and a process bound to a network that is not + * Wi-Fi fails it: its sockets would send over that network. [boundWifi] is + * null when the process is not bound. Wi-Fi only answers NoWifi, not + * NoInternet: asking the user to join Wi-Fi is the more direct hint. + */ +internal fun connectivityOf(wifiOnly: Boolean, validated: Boolean, defaultWifi: Boolean, boundWifi: Boolean?): Connectivity = when { + !wifiOnly -> if (validated) Connectivity.Ok else Connectivity.NoInternet + validated && defaultWifi && boundWifi != false -> Connectivity.Ok + else -> Connectivity.NoWifi } // Makes sure that the channel for the foreground notification exists. It makes diff --git a/android/src/main/java/ai/openspace/backgroundupload/UploaderModule.kt b/android/src/main/java/ai/openspace/backgroundupload/UploaderModule.kt index 1f50951f..38f990e6 100644 --- a/android/src/main/java/ai/openspace/backgroundupload/UploaderModule.kt +++ b/android/src/main/java/ai/openspace/backgroundupload/UploaderModule.kt @@ -53,7 +53,10 @@ class UploaderModule(context: ReactApplicationContext) : private val journal = EventJournal.get(context) private val settings = QueueSettingsStore.get(context) private val scheduler = WorkManagerScheduler(context) - private val controller = QueueController(store, journal, settings, EventReporter, scheduler) + private val controller = QueueController( + store, journal, settings, EventReporter, scheduler, + networkFits = { validateConnectivity(context, settings.load().wifiOnlyFor(it)) == Connectivity.Ok }, + ) // Before init: the lifecycle listener it registers can fire at once. private val foreground = ForegroundTask(queueExecutor, { settings.saveIfBehind() }, { controller.retryNow() }) diff --git a/android/src/main/java/ai/openspace/backgroundupload/WorkerOps.kt b/android/src/main/java/ai/openspace/backgroundupload/WorkerOps.kt index a2c7d0bc..7c4cba81 100644 --- a/android/src/main/java/ai/openspace/backgroundupload/WorkerOps.kt +++ b/android/src/main/java/ai/openspace/backgroundupload/WorkerOps.kt @@ -329,7 +329,7 @@ class WorkerOps( if (result == ParkResult.PARKED && next != null) { events.state(next.toRow()) // A parked entry still expires on time. A far expiry waits for a later boot sweep. - if (expiryWakeIsNear(next.expiresAt, now)) scheduler.scheduleWake(next, next.expiresAt, replace = true) + if (expiryWakeIsNear(next.expiresAt, now)) scheduler.scheduleWake(next, next.expiresAt, replace = true, requiredNetwork(next, settings())) } return result } @@ -361,7 +361,7 @@ class WorkerOps( events.state(next.toRow()) if (!wake) return true try { - scheduler.scheduleWake(next, nextAttemptAt, replace = true) + scheduler.scheduleWake(next, nextAttemptAt, replace = true, requiredNetwork(next, settings())) } catch (error: Exception) { throw WakeNotScheduledException(id, error) } @@ -379,7 +379,7 @@ class WorkerOps( * entry save and WorkManager's commit. Never throws. */ fun ensureWake(entry: QueueEntry, at: Long) { - runCatching { scheduler.scheduleWake(entry, at, replace = false) } + runCatching { scheduler.scheduleWake(entry, at, replace = false, requiredNetwork(entry, settings())) } .onFailure { Diag.error("could not schedule a wake for '${entry.id}'", it) } } @@ -391,7 +391,9 @@ class WorkerOps( * nextAttemptAt or now. A time-limit stop needs it when WorkManager holds * the stopped row back: it runs that row again only after its own * backoff, which grows with every start of that row (up to 5 hours). A - * fresh row starts its count at 0. + * fresh row starts its count at 0. The wake waits for the network the + * entry needs ([requiredNetwork]), so a run stopped while it waited for + * the network does not start again offline. */ fun stopped(id: String, generation: Int, wake: Boolean = false) { val now = clock() @@ -409,10 +411,43 @@ class WorkerOps( if (wake) { // A stop must not throw past the run. Without the wake, the stopped // row still runs again, after WorkManager's own backoff. - runCatching { scheduler.scheduleWake(next, next.nextAttemptAt ?: now, replace = true) } + runCatching { scheduler.scheduleWake(next, next.nextAttemptAt ?: now, replace = true, requiredNetwork(next, settings())) } .onFailure { Diag.error("could not schedule a wake for '$id' after a stop", it) } } } + /** + * The run found no usable network and hands the wait to WorkManager: + * back to queued, and a wake now that waits for [network]. Not a failure: + * no attempt, no backoff step, no wait cause, no nextAttemptAt + * ([EntryTransitions.toStopped] keeps what the last failure saved). The + * wake replaces any other wake of the entry. False when the run no + * longer owns the entry (a pause, cancel or replace landed first); then + * nothing is scheduled. + * + * Throws [WakeNotScheduledException] when the wake could not be + * enqueued. The entry is saved queued. The caller's worker then returns + * retry, so WorkManager runs its row again; the boot sweep and a return + * to the foreground also start a queued entry with no row that can start. + */ + fun awaitNetwork(id: String, generation: Int, network: RequiredNetwork): Boolean { + val now = clock() + var applied = false + val next = store.compute(id) { e -> + if (EntryTransitions.isOwnedRun(e, generation)) { + applied = true + EntryTransitions.toStopped(e!!, now) + } else e + } + if (!applied || next == null) return false + events.state(next.toRow()) + try { + scheduler.scheduleWake(next, now, replace = true, network) + } catch (error: Exception) { + throw WakeNotScheduledException(id, error) + } + return true + } + fun settings(): QueueSettings = settings.load() } diff --git a/android/src/test/java/ai/openspace/backgroundupload/EntryRunTest.kt b/android/src/test/java/ai/openspace/backgroundupload/EntryRunTest.kt index 0fb8d86f..0749d6f2 100644 --- a/android/src/test/java/ai/openspace/backgroundupload/EntryRunTest.kt +++ b/android/src/test/java/ai/openspace/backgroundupload/EntryRunTest.kt @@ -2,12 +2,14 @@ package ai.openspace.backgroundupload import androidx.work.WorkInfo import kotlinx.coroutines.CancellationException +import kotlinx.coroutines.CompletableDeferred import kotlinx.coroutines.Job import kotlinx.coroutines.awaitCancellation import kotlinx.coroutines.currentCoroutineContext import kotlinx.coroutines.ensureActive import kotlinx.coroutines.launch import kotlinx.coroutines.runBlocking +import kotlinx.coroutines.yield import org.junit.After import org.junit.Assert.assertEquals import org.junit.Assert.assertFalse @@ -37,6 +39,12 @@ internal class FakeHost(var clock: Long = 10_000L) : TransferHost { /** The highPriority argument of every slot request, in order. */ val slots = mutableListOf() var foregroundError: Throwable? = null + /** What foreground() answers: whether the run holds foreground mode. */ + var foregroundGranted = true + /** What networkWakes() answers: false is API 29. */ + var networkWake = true + /** sleep() also yields, so a sibling part coroutine can run. */ + var yieldOnSleep = false /** The wifiOnly value of every connectivity check, in order. */ val wifiChecks = mutableListOf() @@ -48,6 +56,7 @@ internal class FakeHost(var clock: Long = 10_000L) : TransferHost { onSleep() // A real delay throws here when the run's job was cancelled. currentCoroutineContext().ensureActive() + if (yieldOnSleep) yield() } override suspend fun withSlot(highPriority: Boolean, block: suspend () -> T): T { @@ -66,10 +75,13 @@ internal class FakeHost(var clock: Long = 10_000L) : TransferHost { return network.removeFirstOrNull() ?: Connectivity.Ok } - override suspend fun foreground(entry: QueueEntry) { + override suspend fun foreground(entry: QueueEntry): Boolean { foregroundError?.let { throw it } + return foregroundGranted } + override fun networkWakes() = networkWake + override fun progressStarted(id: String, total: Long, sent: Long) { progress += "start:$total:$sent" } @@ -385,6 +397,8 @@ class EntryRunTest { run() assertEquals(EntryState.AWAITING_AUTH, store.load("e1")!!.state) assertEquals(listOf("e1" to 90_000L), scheduler.wakes) + // The expiry wake settles the entry; that needs no network. + assertEquals(listOf(RequiredNetwork.NONE), scheduler.wakeNetworks) assertEquals(listOf("error"), host.attempts.map { it.outcome }) assertEquals(emptyList(), journal.unacknowledged()) } @@ -546,6 +560,8 @@ class EntryRunTest { // A fresh wake row, even for a wait a worker could sleep out in place. assertEquals(listOf("e1" to host.clock + 4_000), scheduler.wakes) assertEquals(listOf(true), scheduler.wakeReplaces) + // A backoff wake waits for a network too, so it does not start a run offline. + assertEquals(listOf(RequiredNetwork.CONNECTED), scheduler.wakeNetworks) } @Test @@ -576,6 +592,20 @@ class EntryRunTest { assertEquals(emptyList(), host.requests) assertEquals(listOf("e1" to host.clock), scheduler.wakes) assertEquals(listOf(true), scheduler.wakeReplaces) + // The fresh row waits for the network, so an offline entry does not start, poll and stop again. + assertEquals(listOf(RequiredNetwork.CONNECTED), scheduler.wakeNetworks) + } + + @Test + fun `a timeout stop while a Wi-Fi-only entry waits for Wi-Fi wakes it on Wi-Fi`() { + controller.setWifiOnly(true) + controller.enqueue(parsed()) + host.stop = StopKind.TIME_LIMIT + host.network = ArrayDeque(listOf(Connectivity.NoWifi)) + host.onSleep = { throw CancellationException("timeout") } + assertThrows(CancellationException::class.java) { run() } + assertEquals(listOf("e1" to host.clock), scheduler.wakes) + assertEquals(listOf(RequiredNetwork.WIFI), scheduler.wakeNetworks) } @Test @@ -873,6 +903,151 @@ class EntryRunTest { assertEquals(EntryState.COMPLETED, store.load("e1")!!.state) } + // MARK: - waiting for the network on a constrained wake + + @Test + fun `a run with no foreground mode and no network for 60 s goes back to queued and waits on a network wake`() { + controller.enqueue(parsed()) + host.foregroundGranted = false + host.network = ArrayDeque(List(7) { Connectivity.NoInternet }) + run() + val e = store.load("e1")!! + assertEquals(EntryState.QUEUED, e.state) + assertEquals(0, e.attempts) + assertEquals(0, e.backoffStreak) + assertNull(e.nextAttemptAt) + assertNull(e.waitCause) + assertNull(e.failedAt) + assertEquals(emptyList(), host.requests) + assertEquals(emptyList(), host.attempts) + assertEquals(emptyList(), journal.unacknowledged()) + // Seven checks, 10 s apart: 60 s in all. + assertEquals(7, host.wifiChecks.size) + assertEquals(EntryRun.HEADLESS_NETWORK_WAIT_MS, host.sleeps.sum()) + assertEquals(listOf("e1" to host.clock), scheduler.wakes) + assertEquals(listOf(true), scheduler.wakeReplaces) + assertEquals(listOf(RequiredNetwork.CONNECTED), scheduler.wakeNetworks) + assertEquals("state:e1:queued", events.log.last()) + assertEquals("end:false", host.progress.last()) + } + + @Test + fun `a run with no foreground mode that finds the network within 60 s sends`() { + controller.enqueue(parsed()) + host.foregroundGranted = false + host.network = ArrayDeque(List(6) { Connectivity.NoInternet }) + respond(200) + run() + assertEquals(EntryState.COMPLETED, store.load("e1")!!.state) + assertEquals(emptyList>(), scheduler.wakes) + } + + @Test + fun `the hand-over keeps the attempts, streak and cause of the last failure, and writes no wait`() { + controller.enqueue(parsed()) + store.save(store.load("e1")!!.copy(attempts = 2, backoffStreak = 3, waitCause = WaitCause.NO_RESPONSE, failedAt = 5_000)) + host.foregroundGranted = false + host.network = ArrayDeque(List(7) { Connectivity.NoInternet }) + run() + val e = store.load("e1")!! + assertEquals(EntryState.QUEUED, e.state) + assertEquals(2, e.attempts) + assertEquals(3, e.backoffStreak) + assertEquals(WaitCause.NO_RESPONSE, e.waitCause) + assertEquals(5_000L, e.failedAt) + assertNull(e.nextAttemptAt) + } + + @Test + fun `a Wi-Fi-only entry hands over with the Wi-Fi constraint`() { + controller.setWifiOnly(true) + controller.enqueue(parsed(id = "follow")) + controller.enqueue(parsed(id = "own", descriptor = desc(dataJson = """{"a":1}""", wifiOnly = true))) + host.foregroundGranted = false + host.network = ArrayDeque(List(7) { Connectivity.NoWifi }) + run("follow") + host.network = ArrayDeque(List(7) { Connectivity.NoWifi }) + controller.setWifiOnly(false) + run("own") + assertEquals(listOf(RequiredNetwork.WIFI, RequiredNetwork.WIFI), scheduler.wakeNetworks) + assertEquals(EntryState.QUEUED, store.load("own")!!.state) + } + + @Test + fun `a run that holds foreground mode keeps the poll for 30 minutes`() { + controller.enqueue(parsed()) + host.foregroundGranted = true + host.network = ArrayDeque(List(180) { Connectivity.NoInternet }) + respond(200) + run() + assertEquals(EntryState.COMPLETED, store.load("e1")!!.state) + assertEquals(EntryRun.FOREGROUND_NETWORK_WAIT_MS, host.sleeps.sum()) + assertEquals(emptyList>(), scheduler.wakes) + } + + @Test + fun `a run that holds foreground mode hands over after 30 minutes with no network`() { + controller.enqueue(parsed()) + host.foregroundGranted = true + host.network = ArrayDeque(List(181) { Connectivity.NoInternet }) + respond(200) + run() + assertEquals(EntryState.QUEUED, store.load("e1")!!.state) + assertEquals(EntryRun.FOREGROUND_NETWORK_WAIT_MS, host.sleeps.sum()) + assertEquals(listOf(RequiredNetwork.CONNECTED), scheduler.wakeNetworks) + assertEquals(emptyList(), host.requests) + } + + @Test + fun `on API 29 a run with no foreground mode keeps the poll`() { + controller.enqueue(parsed()) + host.foregroundGranted = false + host.networkWake = false + host.network = ArrayDeque(List(20) { Connectivity.NoInternet }) + respond(200) + run() + assertEquals(EntryState.COMPLETED, store.load("e1")!!.state) + assertEquals(200_000L, host.sleeps.sum()) + assertEquals(emptyList>(), scheduler.wakes) + } + + @Test + fun `a hand-over whose wake can not be scheduled leaves the entry queued and fails the run, so the worker retries`() { + val file = File(tmp.newFolder(), "diagnostics.json") + Diag.log = DiagnosticLog(file) + controller.enqueue(parsed()) + host.foregroundGranted = false + host.network = ArrayDeque(List(7) { Connectivity.NoInternet }) + scheduler.wakeFailures = 1 + assertThrows(WorkerOps.WakeNotScheduledException::class.java) { run() } + val e = store.load("e1")!! + assertEquals(EntryState.QUEUED, e.state) + assertEquals(0, e.attempts) + assertEquals(emptyList>(), scheduler.wakes) + } + + @Test + fun `a pause while it waits to hand over stops the run with no wake`() { + controller.enqueue(parsed()) + host.foregroundGranted = false + host.network = ArrayDeque(List(7) { Connectivity.NoInternet }) + host.onSleep = { if (host.sleeps.size == 5) controller.pause() } + run() + assertEquals(EntryState.PAUSED, store.load("e1")!!.state) + assertEquals(emptyList>(), scheduler.wakes) + } + + @Test + fun `the send gate refuses a hand-over while a request is sending, and a send once a hand-over began`() { + val gate = EntryRun.SendGate() + gate.beginSend() + assertFalse(gate.tryHandOver(RequiredNetwork.WIFI)) + gate.endSend() + assertTrue(gate.tryHandOver(RequiredNetwork.WIFI)) + val refused = assertThrows(EntryRun.AwaitNetworkException::class.java) { gate.beginSend() } + assertEquals(RequiredNetwork.WIFI, refused.network) + } + // MARK: - transient rules private val bodyStopped = UploadOutcome.TransientRule(400, "Bad Request", maxRetries = 2) @@ -1406,4 +1581,28 @@ class EntryRunTest { run() assertEquals(20, outcome().bytesSent) } + + @Test + fun `a chunked hand-over waits for a part that is sending, and never cuts it`() { + val src = File(tmp.newFolder(), "video.bin").apply { writeBytes(ByteArray(20) { it.toByte() }) } + controller.enqueue(parsed(descriptor = desc(url = null, method = "PUT", file = src.path, + parts = listOf(part(0, 10), part(10, 20))))) + host.foregroundGranted = false + host.yieldOnSleep = true + // Part 0 finds the network and sends; part 1 then finds none. + host.network = ArrayDeque(listOf(Connectivity.Ok) + List(30) { Connectivity.NoInternet }) + val answer = CompletableDeferred() + host.handler = { _, _ -> answer.await() } + val start = host.clock + host.onSleep = { if (host.clock - start >= 120_000 && !answer.isCompleted) answer.complete(UploadResponse(200, "", mapOf())) } + run() + val e = store.load("e1")!! + assertEquals(EntryState.QUEUED, e.state) + assertEquals(listOf(true, false), e.descriptor!!.parts!!.map { it.accepted }) + assertEquals(1, e.attempts) + assertEquals(listOf("completed"), host.attempts.map { it.outcome }) + // Part 1 waited past 60 s while part 0 was sending, then handed over. + assertTrue(host.sleeps.sum() >= 120_000) + assertEquals(listOf(RequiredNetwork.CONNECTED), scheduler.wakeNetworks) + } } diff --git a/android/src/test/java/ai/openspace/backgroundupload/MainChainTest.kt b/android/src/test/java/ai/openspace/backgroundupload/MainChainTest.kt index a88230f8..eb08ff73 100644 --- a/android/src/test/java/ai/openspace/backgroundupload/MainChainTest.kt +++ b/android/src/test/java/ai/openspace/backgroundupload/MainChainTest.kt @@ -7,6 +7,7 @@ import androidx.work.WorkInfo.State.ENQUEUED import androidx.work.WorkInfo.State.RUNNING import org.junit.Assert.assertEquals import org.junit.Assert.assertNull +import org.junit.Assert.assertThrows import org.junit.Assert.assertTrue import org.junit.Before import org.junit.Rule @@ -39,10 +40,10 @@ class MainChainTest { val log = mutableListOf() var enqueueFails = false - override fun enqueue(name: String, policy: ExistingWorkPolicy, entry: QueueEntry, delayMs: Long) { + override fun enqueue(name: String, policy: ExistingWorkPolicy, entry: QueueEntry, delayMs: Long, network: RequiredNetwork) { if (enqueueFails) throw IllegalStateException("the process died before the enqueue") - log += "enqueue:$name:$policy" - work.enqueue(name, policy, entry, delayMs) + log += "enqueue:$name:$policy" + if (network == RequiredNetwork.NONE) "" else ":$network" + work.enqueue(name, policy, entry, delayMs, network) } override fun cancel(name: String) { @@ -60,13 +61,17 @@ class MainChainTest { journal = EventJournal(tmp.newFolder("journal"), retryLater = { _, _ -> }) settings = QueueSettingsStore(File(root, "settings.json")) scheduler = RowScheduler(rows) { work.now } - controller = QueueController(store, journal, settings, events, scheduler, { it in running }, { work.now }) + controller = QueueController(store, journal, settings, events, scheduler, { it in running }, { work.now }, ::networkFits) } /** A queued entry that waits after a transport failure 10 minutes ago. */ private fun transportWait(id: String = "e1", streak: Int = 12) = entry(id = id, state = EntryState.QUEUED, nextAttemptAt = work.now + 60 * minute) .copy(backoffStreak = streak, waitCause = WaitCause.NO_RESPONSE, failedAt = work.now - 10 * minute) + /** The in-worker check, on the fake's network. */ + private fun networkFits(e: QueueEntry) = + work.network.ordinal >= (if (settings.load().wifiOnlyFor(e)) RequiredNetwork.WIFI else RequiredNetwork.CONNECTED).ordinal + /** The rows of [id] WorkManager may start now, main chain and wake. */ private fun startable(id: String = "e1") = work.startableNow().filter { it.entryId == id } @@ -218,7 +223,7 @@ class MainChainTest { work.add("e1#wake", state = RUNNING, runAttemptCount = 1) running += "e1" controller.retryNow() - scheduler.scheduleWake(store.load("e1")!!, work.now + 60 * minute, replace = true) + scheduler.scheduleWake(store.load("e1")!!, work.now + 60 * minute, replace = true, RequiredNetwork.CONNECTED) running -= "e1" assertEquals(CANCELLED, held.state) assertNull(store.load("e1")!!.nextAttemptAt) @@ -254,6 +259,158 @@ class MainChainTest { assertTrue(WorkManagerBackoff.restartBy(startedAt, 4) < startedAt + 10 * minute) } + // MARK: - an offline entry sleeps on a network wake + + /** + * A run of e1 under [rowName] waited for [network] and handed over: the + * entry is queued, the wake waits for the network, and the run's own row + * ends (a REPLACE of the wake name cancels a wake row that runs). + */ + private fun handedOver(rowName: String = "e1", network: RequiredNetwork = RequiredNetwork.CONNECTED) { + store.save(entry(state = EntryState.RUNNING)) + val own = work.add(rowName, state = RUNNING, runAttemptCount = 1) + running += "e1" + val ops = WorkerOps(store, journal, settings, events, scheduler) { work.now } + assertTrue(ops.awaitNetwork("e1", 1, network)) + running -= "e1" + if (own.state == RUNNING) own.state = androidx.work.WorkInfo.State.SUCCEEDED + rows.log.clear() + } + + private fun openRows() = work.rows.filter { !it.state.isFinished } + + @Test + fun `a constrained row reports its enqueue time as the next schedule time, and starts only when its network is up`() { + val row = work.add("e1#wake", network = RequiredNetwork.WIFI) + assertEquals(work.now, work.nextScheduleAt(row)) + work.network = RequiredNetwork.NONE + assertEquals(emptyList(), work.startableNow()) + work.network = RequiredNetwork.CONNECTED + assertEquals(emptyList(), work.startableNow()) + work.network = RequiredNetwork.WIFI + assertEquals(listOf(row), work.startableNow()) + } + + @Test + fun `a hand-over leaves one row, a wake that waits for the network`() { + work.network = RequiredNetwork.NONE + handedOver() + assertEquals(EntryState.QUEUED, store.load("e1")!!.state) + val wake = openRows().single() + assertEquals("e1#wake", wake.name) + assertEquals(RequiredNetwork.CONNECTED, wake.network) + assertEquals(0L, wake.initialDelay) + assertEquals(emptyList(), startable()) + // The network comes back: WorkManager starts the wake. + work.network = RequiredNetwork.CONNECTED + assertEquals(listOf(wake), startable()) + } + + @Test + fun `a hand-over from a run under the wake name replaces that wake`() { + work.network = RequiredNetwork.NONE + handedOver(rowName = "e1#wake", network = RequiredNetwork.WIFI) + assertEquals(listOf(CANCELLED, ENQUEUED), work.rows.map { it.state }) + assertEquals(RequiredNetwork.WIFI, openRows().single().network) + } + + @Test + fun `a return to the foreground while offline adds no row next to a network wake`() { + work.network = RequiredNetwork.NONE + handedOver() + controller.retryNow() + controller.retryNow() + assertEquals(emptyList(), rows.log) + assertEquals(listOf("e1#wake"), openRows().map { it.name }) + } + + @Test + fun `opening the app with a network starts an entry on a network wake, also when WorkManager has not started the wake`() { + work.network = RequiredNetwork.NONE + handedOver() + work.network = RequiredNetwork.CONNECTED + controller.retryNow() + // A main row starts now; the wake stays. The per-id gate runs them one after the other. + assertEquals(listOf("enqueue:e1:APPEND_OR_REPLACE"), rows.log) + assertEquals(listOf("e1#wake", "e1"), startable().map { it.name }) + // The next foreground finds a row that starts now and adds nothing. + rows.log.clear() + controller.retryNow() + assertEquals(emptyList(), rows.log) + } + + @Test + fun `a return to the foreground on cellular adds no row next to a Wi-Fi wake`() { + controller.setWifiOnly(true) + work.network = RequiredNetwork.NONE + handedOver(network = RequiredNetwork.WIFI) + work.network = RequiredNetwork.CONNECTED + controller.retryNow() + assertEquals(emptyList(), rows.log) + assertEquals(emptyList(), startable()) + } + + @Test + fun `rowSummary lists a network wake as the entry's wake and as a network wait, not as a row that starts now`() { + work.network = RequiredNetwork.NONE + work.add("n#wake", network = RequiredNetwork.CONNECTED) + work.add("w#wake", network = RequiredNetwork.WIFI) + work.add("b#wake", initialDelay = minute, network = RequiredNetwork.CONNECTED) + val summary = scheduler.rowSummary() + assertEquals(emptySet(), summary.startable) + assertEquals(setOf("n", "w", "b"), summary.withWake) + assertEquals(setOf("n", "w"), summary.networkWait) + } + + @Test + fun `a waiting entry with a short wait and a network wake gets no second wake`() { + work.network = RequiredNetwork.NONE + handedOver() + // A chunked sibling's short backoff was still pending at the hand-over. + store.save(store.load("e1")!!.copy(nextAttemptAt = work.now + 20_000, waitCause = WaitCause.RESPONSE, failedAt = work.now)) + controller.retryNow() + assertEquals(listOf("e1#wake"), openRows().map { it.name }) + } + + @Test + fun `the boot sweep adds a main row for an entry on a network wake and keeps the wake`() { + // The sweep is the repair path: a main row runs whatever WorkManager's constraint tracking does. + work.network = RequiredNetwork.NONE + handedOver() + controller.sweep() + assertEquals(listOf("enqueue:e1:APPEND_OR_REPLACE"), rows.log) + assertEquals(listOf("e1#wake", "e1"), openRows().map { it.name }) + } + + @Test + fun `setWifiOnly(false) starts an entry on a Wi-Fi wake with a main row and cancels the Wi-Fi wake`() { + controller.setWifiOnly(true) + work.network = RequiredNetwork.CONNECTED + handedOver(network = RequiredNetwork.WIFI) + assertEquals(emptyList(), startable()) + controller.setWifiOnly(false) + assertEquals(listOf("enqueue:e1:APPEND_OR_REPLACE", "cancel:e1#wake"), rows.log) + assertEquals(listOf("e1"), startable().map { it.name }) + } + + @Test + fun `a process death between the hand-over save and the wake enqueue leaves the run's own row to start again`() { + work.network = RequiredNetwork.NONE + store.save(entry(state = EntryState.RUNNING)) + val own = work.add("e1", state = RUNNING, runAttemptCount = 1, lastEnqueueTime = work.now - 10 * minute) + val ops = WorkerOps(store, journal, settings, events, scheduler) { work.now } + rows.enqueueFails = true + assertThrows(WorkerOps.WakeNotScheduledException::class.java) { ops.awaitNetwork("e1", 1, RequiredNetwork.CONNECTED) } + assertEquals(EntryState.QUEUED, store.load("e1")!!.state) + // The process dies. At the next start WorkManager sets the RUNNING row back to ENQUEUED (ForceStopRunnable). + own.state = ENQUEUED + rows.enqueueFails = false + controller.retryNow() + assertEquals(listOf(own), work.rows.filter { !it.state.isFinished }) + // It has no constraint, so it starts now, waits for the network and hands over again. + assertEquals(listOf(own), startable()) + } + // MARK: - order, and cost @Test @@ -302,7 +459,7 @@ class MainChainTest { fun `a return to the foreground makes the wake again for a wait that stays and has no row`() { store.save(transportWait().copy(waitCause = WaitCause.RESPONSE)) controller.retryNow() - assertEquals(listOf("enqueue:e1#wake:KEEP"), rows.log) + assertEquals(listOf("enqueue:e1#wake:KEEP:CONNECTED"), rows.log) assertEquals(listOf("e1#wake"), work.rows.filter { !it.state.isFinished }.map { it.name }) // With a wake there, KEEP adds nothing. controller.retryNow() @@ -317,7 +474,7 @@ class MainChainTest { store.save(transportWait(streak = 9).copy(waitCause = WaitCause.RESPONSE, nextAttemptAt = work.now + 10 * minute)) work.add("e1", runAttemptCount = 10, lastEnqueueTime = work.now - minute) controller.retryNow() - assertEquals(listOf("enqueue:e1#wake:KEEP"), rows.log) + assertEquals(listOf("enqueue:e1#wake:KEEP:CONNECTED"), rows.log) val wake = work.rows.single { it.name == "e1#wake" } assertEquals(work.now + 10 * minute, work.nextScheduleAt(wake)) } @@ -341,8 +498,8 @@ class MainChainTest { return work.library(states) } - override fun enqueue(name: String, policy: ExistingWorkPolicy, entry: QueueEntry, delayMs: Long) { - rows.enqueue(name, policy, entry, delayMs) + override fun enqueue(name: String, policy: ExistingWorkPolicy, entry: QueueEntry, delayMs: Long, network: RequiredNetwork) { + rows.enqueue(name, policy, entry, delayMs, network) if (name == "e1") { val row = work.rows.last() row.state = RUNNING diff --git a/android/src/test/java/ai/openspace/backgroundupload/QueueControllerTest.kt b/android/src/test/java/ai/openspace/backgroundupload/QueueControllerTest.kt index 6d2ac596..7747c641 100644 --- a/android/src/test/java/ai/openspace/backgroundupload/QueueControllerTest.kt +++ b/android/src/test/java/ai/openspace/backgroundupload/QueueControllerTest.kt @@ -824,6 +824,36 @@ class QueueControllerTest { assertTrue(QueueSettingsStore(file).load().wifiOnly) } + @Test + fun `setWifiOnly(false) runs the queued entries that follow the setting`() { + controller.setWifiOnly(true) + store.save(entry(id = "follow", state = EntryState.QUEUED)) + store.save(entry(id = "own", state = EntryState.QUEUED, descriptor = desc(dataJson = """{"a":1}""", wifiOnly = true))) + store.save(entry(id = "cell", state = EntryState.QUEUED, descriptor = desc(dataJson = """{"a":1}""", wifiOnly = false))) + store.save(entry(id = "waits", state = EntryState.QUEUED, nextAttemptAt = now + 3_600_000).copy(waitCause = WaitCause.RESPONSE, failedAt = now)) + store.save(entry(id = "busy", state = EntryState.QUEUED)) + store.save(entry(id = "paused", state = EntryState.PAUSED)) + store.save(entry(id = "running", state = EntryState.RUNNING)) + running += "busy" + controller.setWifiOnly(false) + assertEquals(listOf("follow"), scheduler.scheduled) + // Its Wi-Fi wake no longer fits the setting. + assertEquals(listOf("follow"), scheduler.cancelledWakes) + assertEquals(listOf(false), scheduler.cancelWakeKeeps) + assertEquals(emptyList>(), scheduler.wakes) + } + + @Test + fun `setWifiOnly with the same value schedules nothing, and turning it on schedules nothing`() { + store.save(entry(id = "follow", state = EntryState.QUEUED)) + controller.setWifiOnly(false) + // A run that starts on cellular fails the check and hands over with the Wi-Fi constraint. + controller.setWifiOnly(true) + controller.setWifiOnly(true) + assertEquals(emptyList(), scheduler.scheduled) + assertEquals(emptyList(), scheduler.cancelledWakes) + } + @Test fun `setWifiOnly with the same value does not fail when the settings can not be written`() { controller.setWifiOnly(true) @@ -1525,6 +1555,72 @@ class QueueControllerTest { assertEquals(7.0, data["maxRunAttemptCount"]) } + @Test + fun `sweep counts the entries by the wake they sleep on, and reports the wifiOnly setting`() { + val file = File(tmp.newFolder(), "diagnostics.json") + Diag.log = DiagnosticLog(file) + controller.setWifiOnly(true) + store.save(entry(id = "b", state = EntryState.QUEUED, nextAttemptAt = now + 3_600_000).copy(waitCause = WaitCause.RESPONSE, failedAt = now)) + store.save(entry(id = "n", state = EntryState.QUEUED)) + store.save(entry(id = "w", state = EntryState.QUEUED)) + store.save(entry(id = "m", state = EntryState.QUEUED)) + store.save(entry(id = "p", state = EntryState.AWAITING_AUTH, expiresAt = now + 1_000)) + scheduler.runs += PendingRun("b", "ENQUEUED", 0, now + 3_600_000, delayed = true, network = RequiredNetwork.CONNECTED) + scheduler.runs += PendingRun("n", "ENQUEUED", 0, now - 5_000, delayed = false, network = RequiredNetwork.CONNECTED) + scheduler.runs += PendingRun("w", "ENQUEUED", 0, now - 5_000, delayed = false, network = RequiredNetwork.WIFI) + // A main-chain row and a parked entry's expiry wake are not waits for a network or a backoff. + scheduler.runs += PendingRun("m", "ENQUEUED", 0, now, delayed = false) + scheduler.runs += PendingRun("p", "ENQUEUED", 0, now + 1_000, delayed = true) + controller.sweep() + val data = DiagnosticLog(file).drain().single().data!! + assertEquals(mapOf("backoff" to 1.0, "network" to 1.0, "wifi" to 1.0), data["entriesByWake"]) + assertEquals(true, data["wifiOnly"]) + } + + @Test + fun `sweep reports the wifiOnly setting when WorkManager can not be read`() { + val file = File(tmp.newFolder(), "diagnostics.json") + Diag.log = DiagnosticLog(file) + val broken = object : WorkScheduler by scheduler { + override fun libraryRuns(): List = throw IllegalStateException("no WorkManager") + } + QueueController(store, journal, settings, events, broken, { false }, { now }).sweep() + val data = DiagnosticLog(file).drain().single().data!! + assertEquals(false, data["wifiOnly"]) + assertEquals(true, data["runsUnavailable"]) + assertFalse(data.containsKey("entriesByWake")) + } + + @Test + fun `the wakes the controller makes wait for the network a run needs`() { + controller.setWifiOnly(true) + // A backoff wait after a server response stays at the boot sweep: its wake waits for any network. + store.save(entry(id = "b", state = EntryState.QUEUED, nextAttemptAt = now + 3_600_000).copy(waitCause = WaitCause.RESPONSE, failedAt = now)) + // A parked entry's expiry wake settles it, and needs no network. + store.save(entry(id = "p", state = EntryState.AWAITING_AUTH, expiresAt = now + 1_000)) + controller.sweep() + assertEquals( + mapOf("b" to RequiredNetwork.CONNECTED, "p" to RequiredNetwork.NONE), + scheduler.wakes.map { it.first }.zip(scheduler.wakeNetworks).toMap(), + ) + // A return to the foreground makes a missing backoff wake again, with the same rule. + scheduler.wakes.clear() + scheduler.wakeNetworks.clear() + controller.retryNow() + assertEquals(listOf("b" to now + 3_600_000), scheduler.wakes) + assertEquals(listOf(RequiredNetwork.CONNECTED), scheduler.wakeNetworks) + } + + @Test + fun `the sweep, resume and a same-id enqueue keep the network wake of a queued entry with no wait`() { + controller.enqueue(parsed()) + scheduler.cancelWakeKeeps.clear() + controller.sweep() + controller.resume() + controller.enqueue(parsed()) + assertEquals(listOf(true, true, true), scheduler.cancelWakeKeeps) + } + @Test fun `sweep records an entry that was running as it found it`() { val file = File(tmp.newFolder(), "diagnostics.json") diff --git a/android/src/test/java/ai/openspace/backgroundupload/RequiredNetworkTest.kt b/android/src/test/java/ai/openspace/backgroundupload/RequiredNetworkTest.kt new file mode 100644 index 00000000..5fee84f6 --- /dev/null +++ b/android/src/test/java/ai/openspace/backgroundupload/RequiredNetworkTest.kt @@ -0,0 +1,68 @@ +package ai.openspace.backgroundupload + +import org.junit.Assert.assertEquals +import org.junit.Test + +/** The network rule of a wake row, and what the in-worker network check accepts. */ +class RequiredNetworkTest { + private val now = 10_000L + + @Test + fun `the network a wake waits for, for each entry and setting`() { + // state, nextAttemptAt, the entry's own wifiOnly, the queue's wifiOnly → the rule. + val table = listOf( + Row(EntryState.QUEUED, null, null, false, RequiredNetwork.CONNECTED), + Row(EntryState.QUEUED, null, null, true, RequiredNetwork.WIFI), + Row(EntryState.QUEUED, null, true, false, RequiredNetwork.WIFI), + Row(EntryState.QUEUED, null, false, true, RequiredNetwork.CONNECTED), + // A backoff wake waits for any network: the run checks Wi-Fi and hands over if it must. + Row(EntryState.QUEUED, now + 60_000, null, true, RequiredNetwork.CONNECTED), + Row(EntryState.QUEUED, now + 60_000, true, false, RequiredNetwork.CONNECTED), + // A parked entry's wake only settles its expiry. + Row(EntryState.AWAITING_AUTH, null, null, true, RequiredNetwork.NONE), + Row(EntryState.AWAITING_AUTH, null, true, false, RequiredNetwork.NONE), + Row(EntryState.RUNNING, null, null, true, RequiredNetwork.WIFI), + ) + for (r in table) { + val e = entry(state = r.state, nextAttemptAt = r.nextAttemptAt, descriptor = desc(dataJson = """{"a":1}""", wifiOnly = r.own)) + assertEquals(r.toString(), r.expected, requiredNetwork(e, QueueSettings(wifiOnly = r.queue))) + } + } + + private data class Row( + val state: EntryState, + val nextAttemptAt: Long?, + val own: Boolean?, + val queue: Boolean, + val expected: RequiredNetwork, + ) + + @Test + fun `the network check, also while the process is bound to a network`() { + // wifiOnly, default validated, default is Wi-Fi, the bound network is Wi-Fi (null: not bound) → result. + val table = listOf( + Check(false, true, false, null, Connectivity.Ok), + Check(false, false, true, null, Connectivity.NoInternet), + Check(true, true, true, null, Connectivity.Ok), + Check(true, true, false, null, Connectivity.NoWifi), + Check(true, false, true, null, Connectivity.NoWifi), + // The process is bound to cellular: its sockets use cellular, so a Wi-Fi-only upload must not start. + Check(true, true, true, false, Connectivity.NoWifi), + Check(true, true, true, true, Connectivity.Ok), + // Not Wi-Fi only: the bind does not change the answer. + Check(false, true, true, false, Connectivity.Ok), + Check(false, false, true, false, Connectivity.NoInternet), + ) + for (c in table) { + assertEquals(c.toString(), c.expected, connectivityOf(c.wifiOnly, c.validated, c.defaultWifi, c.boundWifi)) + } + } + + private data class Check( + val wifiOnly: Boolean, + val validated: Boolean, + val defaultWifi: Boolean, + val boundWifi: Boolean?, + val expected: Connectivity, + ) +} diff --git a/android/src/test/java/ai/openspace/backgroundupload/TestSupport.kt b/android/src/test/java/ai/openspace/backgroundupload/TestSupport.kt index 8f48131b..ebc2a9b8 100644 --- a/android/src/test/java/ai/openspace/backgroundupload/TestSupport.kt +++ b/android/src/test/java/ai/openspace/backgroundupload/TestSupport.kt @@ -116,16 +116,22 @@ internal class FakeScheduler : WorkScheduler { val wakes = mutableListOf>() /** The replace argument of every scheduleWake, in order. REPLACE makes a fresh row. */ val wakeReplaces = mutableListOf() + /** The network argument of every scheduleWake, in order. */ + val wakeNetworks = mutableListOf() /** How many of the next scheduleWake calls throw, as a failed WorkManager enqueue. */ var wakeFailures = 0 val cancelled = mutableListOf() val cancelledWakes = mutableListOf() + /** The keepNetworkWake argument of every cancelWake, in order. */ + val cancelWakeKeeps = mutableListOf() /** What libraryRuns() returns. */ val runs = mutableListOf() /** What rowSummary() returns as startable. */ val startable = mutableSetOf() /** What rowSummary() returns as withWake. */ val withWake = mutableSetOf() + /** What rowSummary() returns as networkWait. */ + val networkWait = mutableSetOf() /** How many rowSummary() calls were made. */ var startableQueries = 0 @@ -133,28 +139,30 @@ internal class FakeScheduler : WorkScheduler { scheduled += entry.id } - override fun scheduleWake(entry: QueueEntry, at: Long, replace: Boolean) { + override fun scheduleWake(entry: QueueEntry, at: Long, replace: Boolean, network: RequiredNetwork) { if (wakeFailures > 0) { wakeFailures-- throw IllegalStateException("enqueue failed") } wakes += entry.id to at wakeReplaces += replace + wakeNetworks += network } override fun cancel(id: String) { cancelled += id } - override fun cancelWake(id: String) { + override fun cancelWake(id: String, keepNetworkWake: Boolean) { cancelledWakes += id + cancelWakeKeeps += keepNetworkWake } override fun libraryRuns(): List = runs.toList() override fun rowSummary(): RowSummary { startableQueries++ - return RowSummary(startable.toSet(), withWake.toSet()) + return RowSummary(startable.toSet(), withWake.toSet(), networkWait.toSet()) } } @@ -172,6 +180,11 @@ internal class FakeScheduler : WorkScheduler { * lastEnqueueTime + initialDelay. * - When a row finishes, the rows blocked behind it become ENQUEUED with a * new lastEnqueueTime. + * - A row with a network constraint is ENQUEUED like any other, and + * nextScheduleTimeMillis does not look at the constraint: it is + * lastEnqueueTime + initialDelay (WorkSpec.WorkInfoPojo.calculateNextRunTimeMillis + * passes no constraint to calculateNextRunTime). WorkManager starts the + * row only once that time has come and its network is up. */ internal class FakeWorkManager(var now: Long = 10_000L) : WorkRows { class Row( @@ -182,9 +195,16 @@ internal class FakeWorkManager(var now: Long = 10_000L) : WorkRows { var runAttemptCount: Int, var lastEnqueueTime: Long, val initialDelay: Long, + val network: RequiredNetwork = RequiredNetwork.NONE, ) val rows = mutableListOf() + + /** The best network up now: NONE offline, CONNECTED a validated network that is not Wi-Fi, WIFI a validated Wi-Fi network. */ + var network = RequiredNetwork.WIFI + + /** Whether [r]'s network constraint holds now. */ + fun constraintMet(r: Row) = r.network.ordinal <= network.ordinal private var serial = 0 fun nextScheduleAt(r: Row): Long = when { @@ -203,14 +223,15 @@ internal class FakeWorkManager(var now: Long = 10_000L) : WorkRows { runAttemptCount: Int = 0, lastEnqueueTime: Long = now, initialDelay: Long = 0, - ): Row = Row(serial++, name, name.substringBefore("#"), state, runAttemptCount, lastEnqueueTime, initialDelay) + network: RequiredNetwork = RequiredNetwork.NONE, + ): Row = Row(serial++, name, name.substringBefore("#"), state, runAttemptCount, lastEnqueueTime, initialDelay, network) .also { rows += it } fun open(name: String) = rows.filter { it.name == name && !it.state.isFinished } - /** The rows WorkManager may start now. */ + /** The rows WorkManager may start now: due, and their network is up. */ fun startableNow(): List = - rows.filter { it.state == WorkInfo.State.ENQUEUED && nextScheduleAt(it) <= now } + rows.filter { it.state == WorkInfo.State.ENQUEUED && nextScheduleAt(it) <= now && constraintMet(it) } override fun unfinished(name: String): List { nameQueries++ @@ -228,7 +249,7 @@ internal class FakeWorkManager(var now: Long = 10_000L) : WorkRows { return rows.filter { it.state in states }.map(::view) } - override fun enqueue(name: String, policy: ExistingWorkPolicy, entry: QueueEntry, delayMs: Long) { + override fun enqueue(name: String, policy: ExistingWorkPolicy, entry: QueueEntry, delayMs: Long, network: RequiredNetwork) { val open = open(name) val state = when (policy) { ExistingWorkPolicy.KEEP -> if (open.isNotEmpty()) return else WorkInfo.State.ENQUEUED @@ -241,7 +262,7 @@ internal class FakeWorkManager(var now: Long = 10_000L) : WorkRows { if (open.isNotEmpty()) WorkInfo.State.BLOCKED else WorkInfo.State.ENQUEUED else -> error("unused policy $policy") } - rows += Row(serial++, name, entry.id, state, 0, now, delayMs) + rows += Row(serial++, name, entry.id, state, 0, now, delayMs, network) } override fun cancel(name: String) { @@ -259,5 +280,5 @@ internal class FakeWorkManager(var now: Long = 10_000L) : WorkRows { row.state = WorkInfo.State.ENQUEUED } - private fun view(r: Row) = WorkRow(r.entryId, r.state, r.runAttemptCount, nextScheduleAt(r), r.initialDelay > 0) + private fun view(r: Row) = WorkRow(r.entryId, r.state, r.runAttemptCount, nextScheduleAt(r), r.initialDelay > 0, r.network) } From f132994d2382e6ca53913e408826f2cf121de7c1 Mon Sep 17 00:00:00 2001 From: Dylan Murphy Date: Tue, 17 Nov 2026 19:30:34 -0500 Subject: [PATCH 02/12] Docs: an Android upload with no network sleeps until a network is up CHANGELOG (Unreleased, behavior change), README Android notes, setWifiOnly and the boot record fields, and the BootWakeCounts type for entriesByWake and wifiOnly, with type tests. Co-Authored-By: Claude Fable 5.2 --- CHANGELOG.md | 51 +++++++++++++++++++++++++++++++++++++ README.md | 24 ++++++++++++++--- src/__typetests__/define.ts | 34 +++++++++++++++++++++++++ src/types.ts | 19 ++++++++++++-- 4 files changed, 123 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index edd27a85..49ddcd1b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -108,6 +108,57 @@ their own failures from 0. Before, every part started at the stored backoff, so one failed part after a long backoff sent the whole entry back for the next, longer wait. +**Behavior change:** on Android 11 and later (API 30), an upload that finds +no usable network no longer polls inside a running job until the network +returns. A run with no foreground mode checks every 10 s for at most +60 s. Then the entry goes back to `queued` and sleeps on a WorkManager +wake row that waits for a validated network, or a validated Wi-Fi network +when Wi-Fi only applies to it. Android starts that row when the network is +up. This is not a failure: `attempts`, the backoff step and the last +failure's cause stay as they were, and the row shows no `nextAttemptAt`. A +run that holds foreground mode polls for at most 30 minutes, then hands +over the same way, because on Android 15 and later each minute of it uses +the `dataSync` budget of 6 hours in 24. A run hands over only while none +of its requests is on its way: a chunked part that waits for the network +waits for a sibling part to finish sending, and once it hands over, no +part starts a request. Before, the job used the app's job quota while it +waited, Android stopped it at its time limit, and in the `rare` standby +bucket it could wait without a network for the whole run. Android 10 (API +29) keeps the poll, and its wake rows have no network rule, as before: +there WorkManager registers one network callback per constrained row. + +Android: backoff wakes (API 30 and later) now also wait for a validated +network, so a wake does not start a run offline. A wake after a time-limit +stop waits for the network the entry needs. A parked entry's expiry wake +has no network rule. Main-chain rows still have none, and every run still +checks the network itself before each attempt. One effect: an entry whose +`expiresAt` passes while it sleeps on such a wake settles `expired` when +the network returns or the app starts, not at that time. + +Android: `setWifiOnly(false)` now runs at once each queued entry that +follows the setting and has no wait and no worker, and cancels its +Wi-Fi wake. The same value schedules nothing, and `setWifiOnly(true)` +schedules nothing: a run that starts on another network hands over with +the Wi-Fi rule. + +Android: an entry that sleeps on a network wake keeps it through the boot +sweep, `resume()` and a same-id `mutate()`. The boot sweep and `resume()` +still add a main row too, so a start never depends on WorkManager's +constraint tracking alone; offline, that row waits up to 60 s and hands +over again. A return to the foreground adds a main row only when the +network fits the entry now, so it does not start a run offline. + +Android: a Wi-Fi-only upload now also waits while the app binds its +process to a network that is not Wi-Fi (`bindProcessToNetwork`). Its +sockets would use that network. The network check reads the system +default network, which a bind does not change. + +Android: the boot record (`takeDiagnostics()`, message `boot sweep`) now +has `wifiOnly`, the queue's setting, and `entriesByWake` +(`{ backoff, network, wifi }`): the queued entries by the wake row they +sleep on. `entriesByWake` is left out when WorkManager could not be read. +The `Diagnostic` type has both fields. + ## 11.3.0 Added `useRequests(filter?)` and `useRequestsWithProgress(filter?)`, React diff --git a/README.md b/README.md index ec20d263..e780bcf5 100644 --- a/README.md +++ b/README.md @@ -220,6 +220,18 @@ at the next run, after a growing backoff. A body that needs more than 10 minutes headless cannot finish that way. Use `parts` for large bodies: accepted parts are kept across runs. iOS has no equal limit. +**No network.** On API 30 and later a run that finds no usable network +checks every 10 s for at most 60 s, or 30 minutes when it holds foreground +mode. Then the entry goes back to `queued` and sleeps on a WorkManager row +that waits for a validated network (a validated Wi-Fi network when Wi-Fi +only applies). Android starts it when the network is up, within the +limits of the app's standby bucket and Doze. This is not a failed attempt and adds no backoff. Backoff wakes wait for a +network too. The boot sweep, `resume()`, and a return to the foreground +when the network fits, also start the entry from a plain row, so a start +does not depend on that rule alone. A Wi-Fi-only upload also waits while +the app binds its process to a network that is not Wi-Fi. API 29 keeps the +10 s check for as long as the run lasts. + **Backoff and the app.** When the app starts, calls `resume()`, or comes to the foreground (every Activity resume, at most once every 30 s), the library ends the retry backoff of a queued entry and runs it at once, when @@ -519,8 +531,9 @@ again. The queue's Wi-Fi setting. Persisted natively. Applies to queued and future entries whose descriptor does not set `wifiOnly`. An entry that sets it keeps its own value. Both are checked before each attempt, so a toggle moves the -queued entries that follow the setting. A call with the current value -resolves without a write. +queued entries that follow the setting. On Android, turning it off also +starts at once the queued entries that follow it and have no backoff wait. +A call with the current value resolves without a write. It does not reject. The new value applies at once, also when the disk is full. See [Settings on a full disk](#settings-on-a-full-disk). @@ -774,7 +787,12 @@ entry that was running when its process died counts as running. It also describes every unfinished WorkManager run the library had before the start changed them: the count by state, how many are delayed wakes, how many of those are due more than 24 hours out, the highest run attempt count, and how -many queued or running entries had no run at all. +many queued or running entries had no run at all. `entriesByWake` +(`{ backoff, network, wifi }`) counts the queued entries by the wake they +sleep on: the end of a backoff wait, any validated network, or Wi-Fi. It +is left out when WorkManager could not be read. `wifiOnly` is the queue's +Wi-Fi setting. Together they tell "waits for the network or for Wi-Fi by +choice" from "waits for a backoff". On iOS the boot record has only `v9Leftovers` and `orphans`. The other iOS records are the warnings of the count and of the orphan cleanup, and a diff --git a/src/__typetests__/define.ts b/src/__typetests__/define.ts index 0f2351e4..103ae858 100644 --- a/src/__typetests__/define.ts +++ b/src/__typetests__/define.ts @@ -2,6 +2,7 @@ // nothing here runs. Each @ts-expect-error line fails the build when the // error it expects goes away. import type { + BootWakeCounts, Diagnostic, Meta, OutcomeError, @@ -348,6 +349,39 @@ if (orphaned) { expectType(orphaned.removedDirs); expectType(orphaned.removedBytes); } +const wakes = record.data?.entriesByWake; +assertEqual(true); +if (wakes) { + expectType(wakes.backoff); + expectType(wakes.network); + expectType(wakes.wifi); +} +const wifiOnlySetting = record.data?.wifiOnly; +assertEqual(true); +expectType(wifiOnlySetting); +const androidBoot: Diagnostic = { + at: 1, + level: 'info', + message: 'boot sweep', + data: { wifiOnly: true, entriesByWake: { backoff: 1, network: 2, wifi: 0 } }, +}; +expectType(androidBoot); +const wakesMissingWifi: Diagnostic = { + at: 1, + level: 'info', + message: 'boot sweep', + // @ts-expect-error every wake count is required + data: { entriesByWake: { backoff: 1, network: 2 } }, +}; +expectType(wakesMissingWifi); +const wifiOnlyText: Diagnostic = { + at: 1, + level: 'info', + message: 'boot sweep', + // @ts-expect-error wifiOnly is a boolean + data: { wifiOnly: 'yes' }, +}; +expectType(wifiOnlyText); assertEqual['entries'], unknown>(true); const zero: Diagnostic = { at: 1, diff --git a/src/types.ts b/src/types.ts index 4bd96fc9..a35c074f 100644 --- a/src/types.ts +++ b/src/types.ts @@ -480,14 +480,29 @@ export type QueueOrphans = { removedBytes: number; }; +/** + * Android boot snapshot: the queued entries by the wake row they sleep on. + * Each entry counts once. `backoff`: a wake at the end of a backoff wait. + * `wifi`: a wake that waits for Wi-Fi. `network`: a wake that waits for any + * validated network. An entry with none of these is not counted. + */ +export type BootWakeCounts = { + backoff: number; + network: number; + wifi: number; +}; + /** * A diagnostic record's structured fields. The boot snapshot (message - * `'boot sweep'`) always has `v9Leftovers` and `orphans`; its other fields - * are untyped. + * `'boot sweep'`) always has `v9Leftovers` and `orphans`. On Android it also + * has `wifiOnly` (the queue's setting) and, when WorkManager could be read, + * `entriesByWake`. Its other fields are untyped. */ export type DiagnosticData = Record & { v9Leftovers?: V9Leftovers; orphans?: QueueOrphans; + wifiOnly?: boolean; + entriesByWake?: BootWakeCounts; }; /** From dd1cea2d6867e64f40266f427d366f8da5a9d6ab Mon Sep 17 00:00:00 2001 From: Dylan Murphy Date: Tue, 17 Nov 2026 20:26:02 -0500 Subject: [PATCH 03/12] Android: an entry with no network waits on a shared doorbell, not a constrained row The review found that rows with a network rule let WorkManager stop an upload mid-request when the rule is lost, and then hold the stopped row back for hours; that the repair paths left two rows per entry; and that many sleeping constrained wakes could fill WorkManager's slots. Now no row that runs an upload has a WorkManager constraint. A run that hands over goes back to queued with no row of its own and makes sure one of two doorbell rows exists (any validated network, or validated Wi-Fi). The doorbell's worker sends nothing: it starts each waiting entry whose network fits on an ordinary main row, and keeps a doorbell for the rest. Its enqueue is awaited. Backoff and expiry wakes have no rule again. The boot sweep, resume(), a foreground, a same-id enqueue and setWifiOnly() start a waiting entry only when the network fits it now, else they make sure its doorbell exists: one row per entry, and an offline app start starts nothing. A failed doorbell falls back to the main row. The network check never throws and is read at most twice per pass. A Wi-Fi-only run bound to cellular keeps waiting instead of handing over. The 30-minute foreground cap applies on API 35 and later only. The send gate is new for every pass. The boot record reports the doorbells. Co-Authored-By: Claude Fable 5.2 --- .../backgroundupload/BootSnapshot.kt | 34 +- .../backgroundupload/DoorbellWorker.kt | 37 ++ .../ai/openspace/backgroundupload/EntryRun.kt | 71 ++- .../openspace/backgroundupload/EntryWorker.kt | 2 +- .../backgroundupload/QueueController.kt | 166 ++++-- .../openspace/backgroundupload/Scheduler.kt | 284 +++++----- .../backgroundupload/TransferHost.kt | 8 +- .../backgroundupload/UploadNotification.kt | 46 +- .../backgroundupload/UploaderModule.kt | 4 +- .../openspace/backgroundupload/WorkerOps.kt | 43 +- .../backgroundupload/EntryRunTest.kt | 160 ++++-- .../backgroundupload/MainChainTest.kt | 175 +----- .../backgroundupload/NetworkDoorbellTest.kt | 515 ++++++++++++++++++ .../backgroundupload/QueueControllerTest.kt | 135 +++-- .../backgroundupload/RequiredNetworkTest.kt | 35 +- .../openspace/backgroundupload/TestSupport.kt | 105 +++- 16 files changed, 1261 insertions(+), 559 deletions(-) create mode 100644 android/src/main/java/ai/openspace/backgroundupload/DoorbellWorker.kt create mode 100644 android/src/test/java/ai/openspace/backgroundupload/NetworkDoorbellTest.kt diff --git a/android/src/main/java/ai/openspace/backgroundupload/BootSnapshot.kt b/android/src/main/java/ai/openspace/backgroundupload/BootSnapshot.kt index 74dadc99..ada2a7d7 100644 --- a/android/src/main/java/ai/openspace/backgroundupload/BootSnapshot.kt +++ b/android/src/main/java/ai/openspace/backgroundupload/BootSnapshot.kt @@ -8,8 +8,9 @@ package ai.openspace.backgroundupload * v9Leftovers counts the v9 work still on disk, and orphans the * directories with no entry file that are not v9: what the boot cleanup * left, and what it deleted. Zeros included. wifiOnly is the queue's - * setting, and entriesByWake counts the queued entries by the wake they - * sleep on: a backoff wait, the network, or Wi-Fi. + * setting. entriesByWake counts the queued entries by what starts them: + * their backoff wake, or the network or Wi-Fi doorbell; doorbells says + * which doorbell rows exist. */ internal object BootSnapshot { const val MESSAGE = "boot sweep" @@ -24,7 +25,7 @@ internal object BootSnapshot { runs: List?, now: Long, leftovers: Leftovers, - wifiOnly: Boolean, + settings: QueueSettings, ): Map { val base = mapOf( "entries" to entries.size, @@ -33,7 +34,7 @@ internal object BootSnapshot { "oldestUpdatedAt" to entries.minOfOrNull { it.updatedAt }, "v9Leftovers" to leftovers.v9.toMap(), "orphans" to leftovers.orphans.toMap(), - "wifiOnly" to wifiOnly, + "wifiOnly" to settings.wifiOnly, ) if (runs == null) return base + ("runsUnavailable" to true) val withRun = runs.mapNotNull { it.entryId }.toSet() @@ -48,17 +49,21 @@ internal object BootSnapshot { run.delayed && run.nextScheduleAt != null && run.nextScheduleAt - now > EXPIRY_WAKE_HORIZON_MS }, "maxRunAttemptCount" to runs.maxOfOrNull { it.runAttemptCount }, - "entriesByWake" to entriesByWake(entries, runs), + "entriesByWake" to entriesByWake(entries, runs, now, settings), + "doorbells" to mapOf( + "network" to runs.any { it.doorbell == RequiredNetwork.CONNECTED }, + "wifi" to runs.any { it.doorbell == RequiredNetwork.WIFI }, + ), ) } /** - * Each queued entry once, by its wake: "backoff" a delayed wake (a - * backoff wait), else "wifi" a wake that waits for Wi-Fi, else "network" - * one that waits for any network. An entry with none of them is not - * counted. Only a wake has a delay or a network rule. + * Each queued entry once. "backoff": it has a delayed wake (a backoff + * wait). "network" and "wifi": it has no run of its own and no long + * wait, so the doorbell of that rule starts it ([requiredNetwork]). An + * entry with a main-chain row is not counted. */ - private fun entriesByWake(entries: List, runs: List): Map { + private fun entriesByWake(entries: List, runs: List, now: Long, settings: QueueSettings): Map { val byEntry = runs.filter { it.entryId != null }.groupBy { it.entryId } val counts = linkedMapOf("backoff" to 0, "network" to 0, "wifi" to 0) for (e in entries) { @@ -66,9 +71,12 @@ internal object BootSnapshot { val own = byEntry[e.id].orEmpty() val kind = when { own.any { it.delayed } -> "backoff" - own.any { it.network == RequiredNetwork.WIFI } -> "wifi" - own.any { it.network == RequiredNetwork.CONNECTED } -> "network" - else -> continue + own.isNotEmpty() -> continue + else -> when (requiredNetwork(e, settings, now)) { + RequiredNetwork.CONNECTED -> "network" + RequiredNetwork.WIFI -> "wifi" + RequiredNetwork.NONE -> continue + } } counts[kind] = counts.getValue(kind) + 1 } diff --git a/android/src/main/java/ai/openspace/backgroundupload/DoorbellWorker.kt b/android/src/main/java/ai/openspace/backgroundupload/DoorbellWorker.kt new file mode 100644 index 00000000..5d538615 --- /dev/null +++ b/android/src/main/java/ai/openspace/backgroundupload/DoorbellWorker.kt @@ -0,0 +1,37 @@ +package ai.openspace.backgroundupload + +import android.content.Context +import androidx.work.CoroutineWorker +import androidx.work.WorkerParameters +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.withContext + +/** + * The worker of a doorbell row ([WorkScheduler.ensureDoorbell]). WorkManager + * runs it when the row's network is up. It sends nothing: on the queue + * executor it starts every queued entry that waits for a network that fits + * now ([QueueController.answerDoorbell]), and makes sure a doorbell stays for + * the entries whose network does not fit yet. It also runs with no JS and + * no module, so it builds its own controller over the shared stores. + * + * Returns retry when a WorkManager call failed, so the row runs again. + */ +class DoorbellWorker(private val context: Context, params: WorkerParameters) : CoroutineWorker(context, params) { + + override suspend fun doWork(): Result = withContext(Dispatchers.IO) { + // A headless run has no module, so it opens the log that its Diag calls write to. + DiagnosticLog.get(context) + val settings = QueueSettingsStore.get(context) + val controller = QueueController( + QueueStore.get(context), + EventJournal.get(context), + settings, + EventReporter, + WorkManagerScheduler(context), + networkFits = { wifiOnly -> networkFitsNow(context, wifiOnly) }, + ) + // Every queue change runs on this one thread; a blocking wait here is off the main thread. + val answered = UploaderModule.queueExecutor.submit { controller.answerDoorbell() }.get() + if (answered) Result.success() else Result.retry() + } +} diff --git a/android/src/main/java/ai/openspace/backgroundupload/EntryRun.kt b/android/src/main/java/ai/openspace/backgroundupload/EntryRun.kt index c47e9a9c..812453d0 100644 --- a/android/src/main/java/ai/openspace/backgroundupload/EntryRun.kt +++ b/android/src/main/java/ai/openspace/backgroundupload/EntryRun.kt @@ -41,9 +41,9 @@ internal class EntryRun( /** * The run waited long enough for the network and hands the wait to a - * WorkManager wake that waits for [network] ([WorkerOps.awaitNetwork]). + * doorbell row ([WorkerOps.awaitNetwork]). */ - class AwaitNetworkException(val network: RequiredNetwork) : Exception("waiting for the network") + class AwaitNetworkException : Exception("waiting for the network") /** * Keeps a hand-over away from a request on its way. Chunked parts run @@ -53,12 +53,12 @@ internal class EntryRun( */ class SendGate { private var sending = 0 - private var handOver: RequiredNetwork? = null + private var handingOver = false /** Before the write-ahead of a request. Throws when a hand-over began. */ @Synchronized fun beginSend() { - handOver?.let { throw AwaitNetworkException(it) } + if (handingOver) throw AwaitNetworkException() sending++ } @@ -69,9 +69,9 @@ internal class EntryRun( /** True when no request is sending; from then on no request starts. */ @Synchronized - fun tryHandOver(network: RequiredNetwork): Boolean { + fun tryHandOver(): Boolean { if (sending > 0) return false - handOver = network + handingOver = true return true } } @@ -117,17 +117,24 @@ internal class EntryRun( /** * How long a run with no foreground mode waits for the network before it - * hands over to a wake that waits for it. JobScheduler counts the wait + * hands over to a doorbell. JobScheduler counts the wait * against the app's job quota, and Android may stop the job meanwhile. */ const val HEADLESS_NETWORK_WAIT_MS = 60_000L /** - * The same for a run that holds foreground mode. On Android 15 and later - * each minute of it uses the dataSync budget of 6 hours in 24. + * The same for a run that holds foreground mode, on Android 15 (API 35) + * and later only: there each minute of it uses the dataSync budget of 6 + * hours in 24. Below API 35 such a run polls for as long as it runs. */ const val FOREGROUND_NETWORK_WAIT_MS = 30 * 60_000L + /** The first API level with a network doorbell. API 29 keeps the poll. */ + const val DOORBELL_MIN_SDK = 30 + + /** The first API level whose dataSync foreground service has a daily budget (Android 15). */ + const val FOREGROUND_BUDGET_MIN_SDK = 35 + /** The poll while a short backoff remainder runs out before the run starts. */ private const val WAIT_POLL_MS = 10_000L @@ -150,7 +157,10 @@ internal class EntryRun( @Volatile private var holdsForeground = false - private val sends = SendGate() + /** New for every pass: a pass that ended in a re-issue does not carry a hand-over into the next. */ + @Volatile + internal var sends = SendGate() + private set /** A cancel reached a request after its write-ahead. Chunked parts set it from their own coroutines. */ @Volatile @@ -184,6 +194,7 @@ internal class EntryRun( try { // A part cancelled by a sibling's park in the last pass does not count. stopCutRequest = false + sends = SendGate() if (!first) current = ops.latest(entryId, generation) first = false holdsForeground = host.foreground(current) || holdsForeground @@ -210,10 +221,10 @@ internal class EntryRun( endProgress(completed = false) return } catch (wait: AwaitNetworkException) { - // No request is on its way (SendGate). A wake that fails to enqueue - // throws, and the worker returns retry. + // No request is on its way (SendGate). A doorbell that could not be + // stored throws, and the worker returns retry. endProgress(completed = false) - ops.awaitNetwork(entryId, generation, wait.network) + ops.awaitNetwork(entryId, generation) return } catch (error: CancellationException) { // A system stop moves a running entry back to queued. A pause or a @@ -358,32 +369,42 @@ internal class EntryRun( * else the queue's setting. Re-reads the settings and the entry at every * poll, so a setWifiOnly() toggle reaches an entry that follows it. * - * On API 30 and later the run does not poll without end. After - * [HEADLESS_NETWORK_WAIT_MS] with no foreground mode, or - * [FOREGROUND_NETWORK_WAIT_MS] with it, it throws [AwaitNetworkException] - * and the entry sleeps on a wake that WorkManager starts once the network - * is up. It hands over only while no request of the run is sending - * ([SendGate]). API 29 keeps the poll: there WorkManager registers one - * network callback per constrained row. + * On API 30 and later the run does not poll without end. With no + * foreground mode it waits [HEADLESS_NETWORK_WAIT_MS]; with foreground + * mode on API 35 and later [FOREGROUND_NETWORK_WAIT_MS]. Then it throws + * [AwaitNetworkException]: the entry goes back to queued and a doorbell + * starts it once the network is up. Below API 35 a run with foreground + * mode polls as before. It hands over only while no request of the run is + * sending ([SendGate]), and only when the default network itself fails + * the rule: while the app binds the process to another network + * ([Connectivity.BoundElsewhere]) the run keeps waiting, because the + * doorbell would ring at once. API 29 keeps the poll. + * + * A check that throws reads as no network, so the run waits. */ suspend fun waitForNetwork() { val since = host.now() + val sdk = host.sdkInt() while (true) { val s = ops.settings() val entry = ops.latest(entryId, generation) if (s.isPaused(entry.key)) throw PausedException() if (RetryClassifier.isExpired(host.now(), entry.expiresAt)) throw ExpiredException() val wifiOnly = s.wifiOnlyFor(entry) - if (host.connectivity(wifiOnly) == Connectivity.Ok) return - val limit = if (holdsForeground) FOREGROUND_NETWORK_WAIT_MS else HEADLESS_NETWORK_WAIT_MS - if (host.networkWakes() && host.now() - since >= limit) { - val network = if (wifiOnly) RequiredNetwork.WIFI else RequiredNetwork.CONNECTED - if (sends.tryHandOver(network)) throw AwaitNetworkException(network) + val network = safeConnectivity(wifiOnly) { host.connectivity(wifiOnly) } + if (network == Connectivity.Ok) return + val limit = when { + sdk < DOORBELL_MIN_SDK || network == Connectivity.BoundElsewhere -> null + !holdsForeground -> HEADLESS_NETWORK_WAIT_MS + sdk >= FOREGROUND_BUDGET_MIN_SDK -> FOREGROUND_NETWORK_WAIT_MS + else -> null } + if (limit != null && host.now() - since >= limit && sends.tryHandOver()) throw AwaitNetworkException() host.sleep(CONNECTIVITY_POLL_MS) } } + fun policy(entry: QueueEntry) = RetryClassifier.policy(ops.settings().retry, entry.descriptor?.retry) diff --git a/android/src/main/java/ai/openspace/backgroundupload/EntryWorker.kt b/android/src/main/java/ai/openspace/backgroundupload/EntryWorker.kt index c0a981d3..e0e3cc95 100644 --- a/android/src/main/java/ai/openspace/backgroundupload/EntryWorker.kt +++ b/android/src/main/java/ai/openspace/backgroundupload/EntryWorker.kt @@ -103,7 +103,7 @@ open class EntryWorker(protected val context: Context, params: WorkerParameters) return inForeground } - override fun networkWakes() = Build.VERSION.SDK_INT >= Build.VERSION_CODES.R + override fun sdkInt() = Build.VERSION.SDK_INT override fun progressStarted(id: String, total: Long, sent: Long) { UploadProgress.add(id, total) diff --git a/android/src/main/java/ai/openspace/backgroundupload/QueueController.kt b/android/src/main/java/ai/openspace/backgroundupload/QueueController.kt index 300f202e..e6b90d09 100644 --- a/android/src/main/java/ai/openspace/backgroundupload/QueueController.kt +++ b/android/src/main/java/ai/openspace/backgroundupload/QueueController.kt @@ -21,13 +21,19 @@ class QueueController( private val scheduler: WorkScheduler, private val isWorkerRunning: (id: String) -> Boolean = WorkerGate::isRunning, private val clock: () -> Long = System::currentTimeMillis, - /** Whether the network now fits the entry's Wi-Fi rule, as a run checks it. */ - private val networkFits: (QueueEntry) -> Boolean = { true }, + /** + * Whether the network now fits an upload with this Wi-Fi rule, as a run + * checks it. Called at most twice per pass ([NetworkProbe]). + */ + private val networkFits: (wifiOnly: Boolean) -> Boolean = { true }, + /** Whether an entry with no network may wait on a doorbell: API 30 and later. */ + private val doorbells: Boolean = true, ) { // MARK: - enqueue - private class Enqueued(val entry: QueueEntry?, val reEmit: EventJournal.SettledRecord?) + /** [repair]: the entry was queued already (a same-id enqueue of the same body). */ + private class Enqueued(val entry: QueueEntry?, val reEmit: EventJournal.SettledRecord?, val repair: Boolean = false) /** A body staged before the store lock was taken, and the generation it was staged for. */ private class PreStaged(val generation: Int, val staged: BodyStaging.Staged) @@ -58,7 +64,9 @@ class QueueController( } result.reEmit?.let { record -> journal.listener()?.let { events.settled(record, it) } } result.entry?.let { entry -> - scheduleRun(entry) + // New work starts a run, which shows the notification and may take + // foreground mode. A same-id enqueue of a queued entry is a repair. + scheduleRun(entry, probe = if (result.repair) NetworkProbe() else null) events.state(entry.toRow()) } return p.id @@ -100,7 +108,7 @@ class QueueController( EnqueueRules.Action.Resume -> { val next = EnqueueRules.resumed(existing!!, p, s.isPaused(p.key), s.headerGeneration, now) saveOrThrow(next) - Enqueued(next, null) + Enqueued(next, null, repair = existing.state == EntryState.QUEUED) } EnqueueRules.Action.Create -> { val staged = preStaged(1) ?: stageOrThrow(p.descriptor, dir, 1) @@ -237,7 +245,8 @@ class QueueController( } result.changed.forEach { events.state(it.toRow()) } // Every queued entry, not only the resumed ones: a run is idempotent. - store.all().forEach { scheduleRun(it) } + val probe = NetworkProbe() + store.all().forEach { scheduleRun(it, probe = probe) } result.throwIfFailed("resume") } @@ -251,14 +260,10 @@ class QueueController( * failure, is scheduled only when no row of it can start now * ([WorkScheduler.rowSummary], one query): that is not a retry, it only * makes sure a row exists, and a row WorkManager holds back after a stop - * is replaced ([WorkScheduler.schedule]). An entry that sleeps on a wake - * that waits for the network gets a main row only when the network fits - * now ([networkFits]); offline, the wake stays its only row, so a - * foreground does not start a run that polls and hands over again. A - * queued entry whose wait stays and that has no delayed wake row gets its - * wake again (KEEP), also when it has a main row: WorkManager may hold - * that row back for hours. Running, paused, parked and settled entries - * are left alone. A queued entry sends nothing, and the + * is replaced ([WorkScheduler.schedule]). A queued entry whose wait stays + * and that has no delayed wake row gets its wake again (KEEP), also when + * it has a main row: WorkManager may hold that row back for hours. Running, paused, parked and + * settled entries are left alone. A queued entry sends nothing, and the * per-id gate holds any new row while a worker of the entry still winds * down, so this never starts a second transfer next to one in progress. * @@ -292,23 +297,19 @@ class QueueController( } } } - cleared.forEach { e -> runCatching { scheduleRun(e) }.onFailure(::failed) } + val probe = NetworkProbe() + cleared.forEach { e -> runCatching { scheduleRun(e, probe = probe) }.onFailure(::failed) } val clearedIds = cleared.map { it.id }.toSet() val idle = queued.filter { it.nextAttemptAt == null } val waiting = queued.filter { it.nextAttemptAt != null && it.id !in clearedIds } if (idle.isNotEmpty() || waiting.isNotEmpty()) { val rows = runCatching { scheduler.rowSummary() }.onFailure(::failed).getOrNull() - // An entry that sleeps on a network wake waits for WorkManager while - // the network does not fit. When it fits, a main row starts the entry - // now too, so a start never depends on WorkManager's constraint - // tracking alone. The per-id gate runs the two rows one after the - // other, and the second finds the entry no longer queued. - idle.filter { rows == null || (it.id !in rows.startable && (it.id !in rows.networkWait || networkFits(it))) } - .forEach { e -> runCatching { scheduleRun(e) }.onFailure(::failed) } + idle.filter { rows == null || it.id !in rows.startable } + .forEach { e -> runCatching { scheduleRun(e, probe = probe) }.onFailure(::failed) } // A wait that stays needs its wake. KEEP leaves one that exists, so // this is at most one enqueue per entry that lacks a wake. waiting.filter { rows == null || it.id !in rows.withWake } - .forEach { e -> runCatching { scheduler.scheduleWake(e, e.nextAttemptAt!!, replace = false, requiredNetwork(e, settings.load())) }.onFailure(::failed) } + .forEach { e -> runCatching { scheduler.scheduleWake(e, e.nextAttemptAt!!, replace = false) }.onFailure(::failed) } } cleared.forEach { events.state(it.toRow()) } first?.let { Diag.error("retryNow: $failures entry saves or WorkManager calls failed; the next app start or foreground tries again", it) } @@ -316,6 +317,49 @@ class QueueController( private fun inScope(e: QueueEntry, keys: List?) = keys == null || e.key in keys + /** + * A doorbell row ran: WorkManager saw a network that fits its rule. Each + * queued entry that waits for a network ([requiredNetwork]), has no row + * that can start now and no worker, is started when the network fits it + * now; for the others the doorbell of their rule is made sure again (a + * doorbell that runs this gets one queued behind it). One WorkManager + * query, and at most two network checks. Returns false when an entry + * save, a WorkManager call or the query failed: the doorbell worker then + * returns retry, so its row runs again. + */ + fun answerDoorbell(): Boolean { + val now = clock() + val s = settings.load() + val waiting = store.all().filter { requiredNetwork(it, s, now) != RequiredNetwork.NONE && !isWorkerRunning(it.id) } + if (waiting.isEmpty()) return true + val rows = runCatching { scheduler.rowSummary() } + .onFailure { Diag.error("doorbell: could not read WorkManager; it rings again", it) } + .getOrNull() ?: return false + val probe = NetworkProbe() + var answered = true + waiting.filter { it.id !in rows.startable }.forEach { e -> + runCatching { scheduleRun(e, probe = probe) }.onFailure { + answered = false + Diag.error("doorbell: could not schedule '${e.id}'; it rings again", it) + } + } + return answered + } + + /** + * The network check for one pass, at most once per Wi-Fi rule. A check + * that throws reads as no network. + */ + private inner class NetworkProbe { + private val answers = mutableMapOf() + + /** The doorbells already made sure in this pass. */ + val ensured = mutableSetOf() + + fun fits(wifiOnly: Boolean): Boolean = + answers.getOrPut(wifiOnly) { runCatching { networkFits(wifiOnly) }.getOrDefault(false) } + } + /** * Live: journal a 'cancelled' (user) outcome, then forget after its ack. * Settled (or legacy): forget now, row, bytes, and its unacknowledged @@ -418,25 +462,26 @@ class QueueController( * [updateHeaders]) never throw for a failed settings write. See * [QueueSettingsStore.update]. * - * A change to off runs now every queued entry that follows the setting - * and has no wait and no worker: such an entry may sleep on a wake that - * waits for Wi-Fi, which no longer fits. That wake is cancelled once the - * main row is made. A change to on needs no work: a run that starts on - * another network fails the check and hands over with the Wi-Fi rule. - * The same value schedules nothing. A failed WorkManager call is logged - * and the next entry goes on. + * A change to off moves the queued entries that follow the setting from + * the Wi-Fi rule to any network (see below). The same value, or a change + * to on, schedules nothing. */ fun setWifiOnly(enabled: Boolean) { val before = settings.load().wifiOnly val after = settings.update { it.copy(wifiOnly = enabled) }.wifiOnly if (!before || after) return + // Off: the entries that follow the setting now wait for any network. + // Each one with no long wait and no worker starts when that fits, else + // it waits on the CONNECTED doorbell. A Wi-Fi doorbell left behind + // finds nothing to start and ends. On needs no work: a run that starts + // on another network fails the check and hands over to the Wi-Fi doorbell. + val now = clock() + val s = settings.load() + val probe = NetworkProbe() store.all() - .filter { - !it.legacy && it.state == EntryState.QUEUED && it.nextAttemptAt == null && - it.descriptor?.wifiOnly == null && !isWorkerRunning(it.id) - } + .filter { it.descriptor?.wifiOnly == null && requiredNetwork(it, s, now) != RequiredNetwork.NONE && !isWorkerRunning(it.id) } .forEach { e -> - runCatching { scheduleRun(e, keepNetworkWake = false) } + runCatching { scheduleRun(e, probe = probe) } .onFailure { Diag.error("setWifiOnly: could not schedule '${e.id}'", it) } } } @@ -632,7 +677,8 @@ class QueueController( // process left in WorkManager. val runs = runCatching { scheduler.libraryRuns() }.getOrNull() toForget.forEach { scheduler.cancel(it) } - toSchedule.forEach { scheduleRun(it, keepWake = true) } + val probe = NetworkProbe() + toSchedule.forEach { scheduleRun(it, keepWake = true, probe = probe) } // Every start, even an idle one: a v9 leftover count of zero is a value // the app reports. It replaces a snapshot JS has not taken yet, so // launches with no takeDiagnostics() do not push out real warnings. @@ -642,7 +688,7 @@ class QueueController( val removed = store.removeOrphans(counted.orphanDirs, now) Diag.info( BootSnapshot.MESSAGE, - BootSnapshot.of(found, runs, now, counted.afterCleanup(removed), s.wifiOnly), + BootSnapshot.of(found, runs, now, counted.afterCleanup(removed), s), replaceUntaken = true, ) changed.forEach { events.state(it.toRow()) } @@ -662,28 +708,36 @@ class QueueController( * Runs [entry] when it is queued. A backoff longer than a worker waits goes * to the wake; a parked entry gets a wake at its expiry, so it settles * 'expired' on time, once that expiry is near (see [EXPIRY_WAKE_HORIZON_MS]). - * Any other wake of the entry is stale, so it is cancelled, except a wake - * that waits for the network ([keepNetworkWake]): the entry may sleep on - * it, and it starts the entry when the network is back. + * Any other wake of the entry is stale, so it is cancelled. + * + * With a [probe] (a repair: the boot sweep, resume(), a return to the + * foreground, a same-id enqueue, setWifiOnly(), a doorbell), an entry + * whose network does not fit now gets no row: it waits on the doorbell of + * its rule, so an offline app start starts no run. When the doorbell + * cannot be stored, the entry gets its main row after all: that run waits + * for the network and hands over again. Without a probe (new work, new + * headers) the run starts in any case. API 29 has no doorbell. */ - private fun scheduleRun(entry: QueueEntry, keepWake: Boolean = false, keepNetworkWake: Boolean = true) { + private fun scheduleRun(entry: QueueEntry, keepWake: Boolean = false, probe: NetworkProbe? = null) { val now = clock() when (entry.state) { EntryState.QUEUED -> { val at = entry.nextAttemptAt if (at != null && at - now > RetryClassifier.IN_WORKER_BACKOFF_MAX_MS) { - scheduler.scheduleWake(entry, at, replace = !keepWake, requiredNetwork(entry, settings.load())) + scheduler.scheduleWake(entry, at, replace = !keepWake) + } else if (probe != null && waitsOnDoorbell(entry, probe, now)) { + cancelStaleWake(entry.id) } else { // The main row first: these are two separate WorkManager calls, // and a process death between them must not leave the entry // with neither a wake nor a main row. scheduler.schedule(entry) - cancelStaleWake(entry.id, keepNetworkWake) + cancelStaleWake(entry.id) } } EntryState.AWAITING_AUTH -> if (expiryWakeIsNear(entry.expiresAt, now)) { - scheduler.scheduleWake(entry, entry.expiresAt, replace = !keepWake, requiredNetwork(entry, settings.load())) + scheduler.scheduleWake(entry, entry.expiresAt, replace = !keepWake) } else { cancelStaleWake(entry.id) } @@ -691,22 +745,34 @@ class QueueController( } } + /** + * True when [entry]'s network does not fit now and its doorbell is + * stored. False when it fits, on API 29, or when the doorbell failed. + */ + private fun waitsOnDoorbell(entry: QueueEntry, probe: NetworkProbe, now: Long): Boolean { + if (!doorbells) return false + val network = requiredNetwork(entry, settings.load(), now) + if (network == RequiredNetwork.NONE || probe.fits(network == RequiredNetwork.WIFI)) return false + if (network in probe.ensured) return true + return runCatching { scheduler.ensureDoorbell(network) } + .onSuccess { probe.ensured += network } + .onFailure { Diag.error("could not store the ${network.name} doorbell for '${entry.id}'; it starts a run instead", it) } + .isSuccess + } + // A running wake is the entry's worker; it finishes its own run. // [isWorkerRunning] reads the in-process gate, so it assumes WorkManager // runs workers in the app's process. The library sets no android:process. // // The caller's entry can be a stale snapshot: a worker may have released // the entry with a long wait and made its wake since. So the stored entry - // is read again, and a wake it still needs is kept. A queued entry with no - // long wait may sleep on a wake that waits for the network (a run handed - // over, or a worker did that since the snapshot): that wake is kept too, - // unless [keepNetworkWake] is false. - private fun cancelStaleWake(id: String, keepNetworkWake: Boolean = true) { + // is read again, and a wake it still needs is kept. + private fun cancelStaleWake(id: String) { if (isWorkerRunning(id)) return val stored = runCatching { store.load(id) }.getOrNull() val at = stored?.nextAttemptAt if (stored?.state == EntryState.QUEUED && at != null && at - clock() > RetryClassifier.IN_WORKER_BACKOFF_MAX_MS) return - scheduler.cancelWake(id, keepNetworkWake = keepNetworkWake && stored?.state == EntryState.QUEUED) + scheduler.cancelWake(id) } /** The entries [transformAll] saved, and the saves that failed. */ diff --git a/android/src/main/java/ai/openspace/backgroundupload/Scheduler.kt b/android/src/main/java/ai/openspace/backgroundupload/Scheduler.kt index 7c982745..a57e65e3 100644 --- a/android/src/main/java/ai/openspace/backgroundupload/Scheduler.kt +++ b/android/src/main/java/ai/openspace/backgroundupload/Scheduler.kt @@ -3,7 +3,6 @@ package ai.openspace.backgroundupload import android.content.Context import android.net.NetworkCapabilities import android.net.NetworkRequest -import android.os.Build import androidx.work.Constraints import androidx.work.ExistingWorkPolicy import androidx.work.NetworkType @@ -57,31 +56,26 @@ internal object WorkManagerBackoff { } /** - * The network a wake row waits for, as a WorkManager constraint. Both - * network rules need a validated network, as the in-worker check does. - * - * - NONE: no constraint. - * - CONNECTED: NetworkType.CONNECTED. WorkManager 2.12.0 turns it into a - * request with INTERNET and VALIDATED (NetworkTypeCompatKt.toNetworkRequest). - * - WIFI: a NetworkRequest with TRANSPORT_WIFI, INTERNET and VALIDATED. + * The network a queued entry waits for, and the doorbell row that waits for + * it. NONE: the entry does not wait for a network (it is not queued, or it + * waits for a backoff). CONNECTED and WIFI need a validated network, as the + * in-worker check does. */ enum class RequiredNetwork { NONE, CONNECTED, WIFI } /** - * The network rule of a wake row for [entry], as saved. - * - * - A parked entry's wake only settles its expiry: NONE. - * - A backoff wait (nextAttemptAt set): CONNECTED. The run checks the Wi-Fi - * rule and hands over again when it must, so a backoff wake never keeps a - * stale Wi-Fi rule. - * - Otherwise the entry waits for the network its run needs: WIFI when its - * own wifiOnly, else the queue's setting, says so; else CONNECTED. + * The doorbell [entry] waits for when no run of it can start: WIFI when its + * own wifiOnly, else the queue's setting, says so; else CONNECTED. NONE for + * an entry that is not queued, and for one whose wait is longer than a + * worker sleeps out: its wake row starts it ([RetryClassifier.IN_WORKER_BACKOFF_MAX_MS]). + * A short wait, such as a chunked sibling's backoff pending at a hand-over, + * does not change the rule. */ -internal fun requiredNetwork(entry: QueueEntry, settings: QueueSettings): RequiredNetwork = when { - entry.state == EntryState.AWAITING_AUTH -> RequiredNetwork.NONE - entry.nextAttemptAt != null -> RequiredNetwork.CONNECTED - settings.wifiOnlyFor(entry) -> RequiredNetwork.WIFI - else -> RequiredNetwork.CONNECTED +internal fun requiredNetwork(entry: QueueEntry, settings: QueueSettings, now: Long): RequiredNetwork { + if (entry.legacy || entry.state != EntryState.QUEUED) return RequiredNetwork.NONE + val at = entry.nextAttemptAt + if (at != null && at - now > RetryClassifier.IN_WORKER_BACKOFF_MAX_MS) return RequiredNetwork.NONE + return if (settings.wifiOnlyFor(entry)) RequiredNetwork.WIFI else RequiredNetwork.CONNECTED } /** What [RowScheduler.schedule] does to an entry's main chain. */ @@ -141,22 +135,26 @@ interface WorkScheduler { fun schedule(entry: QueueEntry) /** - * A run at [at] under a second unique name. Used for long backoffs, for - * the expiry of a parked entry, and for an entry that waits for the - * network ([network], see [requiredNetwork]). [replace] false keeps a wake - * that already exists (the boot sweep). + * A run at [at] under a second unique name. Used for long backoffs and for + * the expiry of a parked entry. [replace] false keeps a wake that already + * exists (the boot sweep). */ - fun scheduleWake(entry: QueueEntry, at: Long, replace: Boolean, network: RequiredNetwork) + fun scheduleWake(entry: QueueEntry, at: Long, replace: Boolean) /** Cancels the main chain and the wake. */ fun cancel(id: String) + /** Cancels the wake only. A wake that is no longer needed still holds a WorkManager slot. */ + fun cancelWake(id: String) + /** - * Cancels the wake only. A wake that is no longer needed still holds a - * WorkManager slot. [keepNetworkWake] keeps a wake with no delay that - * waits for a network: the wake of an entry that waits for the network. + * Makes sure the doorbell row for [network] (CONNECTED or WIFI) exists: one + * row for the whole queue, with that WorkManager network rule. When the + * network is up, WorkManager runs it, and its worker ([DoorbellWorker]) + * starts the entries that wait for a network. It runs no upload. Returns + * once WorkManager has stored the row; throws when it could not. */ - fun cancelWake(id: String, keepNetworkWake: Boolean = false) + fun ensureDoorbell(network: RequiredNetwork) /** Every unfinished WorkManager run of the library, main chains and wakes. For the boot snapshot. */ fun libraryRuns(): List @@ -166,24 +164,17 @@ interface WorkScheduler { } /** - * [startable]: the ids with a run in progress, or a row with no network rule - * that WorkManager may start now. [networkWait]: the ids with a due wake - * that waits for a network ([isNetworkWake]); WorkManager starts it once - * its network is up, which the summary cannot see. [withWake]: the ids with - * an unfinished delayed row or network wake. Only a wake has an initial - * delay or a network constraint; a main-chain row has neither. + * [startable]: the ids with a run in progress, or a row WorkManager may + * start now. [withWake]: the ids with an unfinished delayed row. Only a wake + * has an initial delay; a main-chain row never does. */ -data class RowSummary( - val startable: Set, - val withWake: Set, - val networkWait: Set = emptySet(), -) +data class RowSummary(val startable: Set, val withWake: Set) /** * One unfinished WorkManager run. [entryId] comes from the run's id tag. * [delayed] is a run enqueued with an initial delay, which only a wake has. - * [nextScheduleAt] is null when WorkManager reports none. [network] is its - * network constraint; only a wake has one. + * [nextScheduleAt] is null when WorkManager reports none. [doorbell] is the + * network of a doorbell row, which has no entry id; null for any other run. */ data class PendingRun( val entryId: String?, @@ -191,16 +182,16 @@ data class PendingRun( val runAttemptCount: Int, val nextScheduleAt: Long?, val delayed: Boolean, - val network: RequiredNetwork = RequiredNetwork.NONE, + val doorbell: RequiredNetwork? = null, ) /** * One unfinished WorkManager row. [nextScheduleAt] is the earliest time * WorkManager starts it (`WorkInfo.nextScheduleTimeMillis`): Long.MAX_VALUE - * for a row that is not ENQUEUED. WorkManager 2.12.0 leaves the network - * constraint out of that time (WorkSpec.WorkInfoPojo.calculateNextRunTimeMillis): - * a constrained row reports its enqueue time plus its delay, and starts - * once that time has come and its [network] is up. + * for a row that is not ENQUEUED. WorkManager 2.12.0 leaves a network rule + * out of that time (WorkSpec.WorkInfoPojo.calculateNextRunTimeMillis): a + * doorbell reports its enqueue time, and starts once its network is up. + * [doorbell]: the network of a doorbell row; null for an entry's row. */ data class WorkRow( val entryId: String?, @@ -208,12 +199,9 @@ data class WorkRow( val runAttemptCount: Int, val nextScheduleAt: Long, val delayed: Boolean, - val network: RequiredNetwork = RequiredNetwork.NONE, + val doorbell: RequiredNetwork? = null, ) -/** A wake with no delay that waits for a network: an entry that waits for the network sleeps on it. */ -internal val WorkRow.isNetworkWake get() = !delayed && network != RequiredNetwork.NONE - /** The WorkManager calls [RowScheduler] makes. Tests run the same rules against a fake. */ interface WorkRows { /** The unfinished rows of one unique name. */ @@ -222,7 +210,13 @@ interface WorkRows { /** Every row of the library, main chains and wakes, in one of [states]. One query. */ fun library(states: List): List - fun enqueue(name: String, policy: ExistingWorkPolicy, entry: QueueEntry, delayMs: Long, network: RequiredNetwork) + fun enqueue(name: String, policy: ExistingWorkPolicy, entry: QueueEntry, delayMs: Long) + + /** + * Enqueues the doorbell row [name] with [network] as its rule, and waits + * until WorkManager has stored it. Throws when it did not. + */ + fun enqueueDoorbell(name: String, policy: ExistingWorkPolicy, network: RequiredNetwork) fun cancel(name: String) } @@ -241,22 +235,24 @@ interface WorkRows { * back every later "run now" appended behind it. They use the wake name * (`#wake`) with an initial delay instead. * - * Main-chain rows carry no WorkManager constraint. A constrained row does - * not start while the app is visible and offline, so it could not take - * foreground mode then; its constraint is fixed when it is made, so a - * setWifiOnly() change would leave stale rows; and on API 29 WorkManager - * registers one network callback per constrained row. Only a wake row - * carries a network rule ([RequiredNetwork]): a backoff wake, and the wake - * of an entry whose run found no network ([EntryRun.waitForNetwork]). The - * worker checks the network itself before every attempt, also on a row a - * constraint started. + * No row that runs an upload carries a WorkManager constraint. WorkManager + * enforces a constraint while the worker runs: it stops the worker when the + * network loses the rule, also in the middle of a request, and the stopped + * row keeps its run count, so WorkManager then holds it back (see + * [WorkManagerBackoff]). A constrained row also does not start while the app + * is visible and offline, so it could not take foreground mode then. The + * worker checks the network itself before every attempt. * - * Earlier versions used no constraint at all, because the constraint path - * was unreliable. The WorkManager release notes for 2.10.1 to 2.12.0 list - * fixes for network constraints that were not met while the network was - * up, a stale capability set, an identical second constraint, and a - * blocked network (Android 15) that passed the constraint. The library is - * on 2.12.0. + * The network rule lives on two doorbell rows for the whole queue + * ([ensureDoorbell]). A doorbell runs no upload: it only starts the entries + * that wait for a network. Earlier versions used no constraint at all, + * because the constraint path was unreliable. The WorkManager release notes + * for 2.10.1 to 2.12.0 list fixes for network constraints that were not met + * while the network was up, re-added capabilities, an identical second + * constraint, and a blocked network (Android 15) that passed the + * constraint. The library is on 2.12.0. Even so, no start depends on the + * doorbell alone: the app's start, resume() and return to the foreground + * start an entry whose network fits. */ open class RowScheduler( private val rows: WorkRows, @@ -264,6 +260,9 @@ open class RowScheduler( ) : WorkScheduler { private companion object { val UNFINISHED = listOf(WorkInfo.State.ENQUEUED, WorkInfo.State.RUNNING, WorkInfo.State.BLOCKED) + + /** The module's scheduler and a doorbell worker's are separate objects in one process. */ + val DOORBELL_LOCK = Any() } /** @@ -273,19 +272,18 @@ open class RowScheduler( override fun schedule(entry: QueueEntry) { when (mainChainStep(rows.unfinished(entry.id), clock())) { MainChainStep.NONE -> Unit - MainChainStep.APPEND -> rows.enqueue(entry.id, ExistingWorkPolicy.APPEND_OR_REPLACE, entry, 0L, RequiredNetwork.NONE) - MainChainStep.REPLACE -> rows.enqueue(entry.id, ExistingWorkPolicy.REPLACE, entry, 0L, RequiredNetwork.NONE) + MainChainStep.APPEND -> rows.enqueue(entry.id, ExistingWorkPolicy.APPEND_OR_REPLACE, entry, 0L) + MainChainStep.REPLACE -> rows.enqueue(entry.id, ExistingWorkPolicy.REPLACE, entry, 0L) } } - override fun scheduleWake(entry: QueueEntry, at: Long, replace: Boolean, network: RequiredNetwork) { + override fun scheduleWake(entry: QueueEntry, at: Long, replace: Boolean) { val delay = WorkManagerScheduler.initialDelayMs(at, clock()) rows.enqueue( WorkManagerScheduler.wakeName(entry.id), if (replace) ExistingWorkPolicy.REPLACE else ExistingWorkPolicy.KEEP, entry, delay, - network, ) } @@ -294,14 +292,32 @@ open class RowScheduler( rows.cancel(WorkManagerScheduler.wakeName(id)) } + /** + * The same rule as an entry's main chain ([mainChainStep]): a doorbell + * that waits for its network, or a running one with one queued behind + * it, is enough. A running doorbell with none behind it gets one appended, + * so an entry that hands over while the doorbell answers is not missed. + * A doorbell that WorkManager holds back after stops is replaced. One lock + * for the process, and the enqueue is awaited, so two callers do not both + * append. + */ + override fun ensureDoorbell(network: RequiredNetwork) { + require(network != RequiredNetwork.NONE) { "a doorbell needs a network" } + val name = WorkManagerScheduler.doorbellName(network) + synchronized(DOORBELL_LOCK) { + when (mainChainStep(rows.unfinished(name), clock())) { + MainChainStep.NONE -> Unit + MainChainStep.APPEND -> rows.enqueueDoorbell(name, ExistingWorkPolicy.APPEND_OR_REPLACE, network) + MainChainStep.REPLACE -> rows.enqueueDoorbell(name, ExistingWorkPolicy.REPLACE, network) + } + } + } + // A cancel reschedules all pending work even when the name has none, and // the boot sweep calls this for every queued entry. - override fun cancelWake(id: String, keepNetworkWake: Boolean) { + override fun cancelWake(id: String) { val name = WorkManagerScheduler.wakeName(id) - val open = rows.unfinished(name) - if (open.isEmpty()) return - if (keepNetworkWake && open.any { it.isNetworkWake }) return - rows.cancel(name) + if (rows.unfinished(name).isNotEmpty()) rows.cancel(name) } // One query for every row, not one per entry: the boot sweep runs this on @@ -314,7 +330,7 @@ open class RowScheduler( runAttemptCount = row.runAttemptCount, nextScheduleAt = row.nextScheduleAt.takeIf { it != Long.MAX_VALUE }, delayed = row.delayed, - network = row.network, + doorbell = row.doorbell, ) } @@ -322,10 +338,8 @@ open class RowScheduler( val now = clock() val open = rows.library(UNFINISHED) return RowSummary( - startable = open.filter { it.state == WorkInfo.State.RUNNING || (it.startsBy(now) && !it.isNetworkWake) } - .mapNotNull { it.entryId }.toSet(), - withWake = open.filter { it.delayed || it.network != RequiredNetwork.NONE }.mapNotNull { it.entryId }.toSet(), - networkWait = open.filter { it.isNetworkWake && it.startsBy(now) }.mapNotNull { it.entryId }.toSet(), + startable = open.filter { it.state == WorkInfo.State.RUNNING || it.startsBy(now) }.mapNotNull { it.entryId }.toSet(), + withWake = open.filter { it.delayed }.mapNotNull { it.entryId }.toSet(), ) } } @@ -346,6 +360,16 @@ class WorkManagerScheduler private constructor(private val workManager: WorkMana fun wakeName(id: String) = "$id#wake" + /** The unique name of the doorbell for [network]. */ + fun doorbellName(network: RequiredNetwork) = "RNFileUploader.doorbell:${network.name}" + + const val DOORBELL_TAG_PREFIX = "RNFileUploaderDoorbell:" + /** A string literal, because WorkManager persists it across builds. */ + const val DOORBELL_NETWORK_KEY = "network" + + /** How long a doorbell enqueue may take before it counts as failed. */ + const val DOORBELL_ENQUEUE_TIMEOUT_MS = 10_000L + /** Milliseconds from [now] until [at]; never negative. */ fun initialDelayMs(at: Long?, now: Long): Long = if (at == null) 0L else (at - now).coerceAtLeast(0L) @@ -371,14 +395,53 @@ private class WorkManagerRows(private val workManager: WorkManager) : WorkRows { WorkQuery.Builder.fromTags(listOf(WorkManagerScheduler.WORK_TAG)).addStates(states).build(), ).get().map(::row) - override fun enqueue(name: String, policy: ExistingWorkPolicy, entry: QueueEntry, delayMs: Long, network: RequiredNetwork) { - workManager.beginUniqueWork(name, policy, request(entry, delayMs, network)).enqueue() + override fun enqueue(name: String, policy: ExistingWorkPolicy, entry: QueueEntry, delayMs: Long) { + workManager.beginUniqueWork(name, policy, request(entry, delayMs)).enqueue() } override fun cancel(name: String) { workManager.cancelUniqueWork(name) } + // Waits for the insert: an enqueue that fails later is otherwise not seen. + // The callers run on the queue executor or a worker thread, never on the + // main thread. + override fun enqueueDoorbell(name: String, policy: ExistingWorkPolicy, network: RequiredNetwork) { + val request = OneTimeWorkRequestBuilder() + .addTag(WorkManagerScheduler.WORK_TAG) + .addTag(WorkManagerScheduler.DOORBELL_TAG_PREFIX + network.name) + .setInputData(workDataOf(WorkManagerScheduler.DOORBELL_NETWORK_KEY to network.name)) + .setConstraints(constraints(network)) + .build() + workManager.beginUniqueWork(name, policy, request).enqueue().result + .get(WorkManagerScheduler.DOORBELL_ENQUEUE_TIMEOUT_MS, TimeUnit.MILLISECONDS) + } + + /** + * CONNECTED: NetworkType.CONNECTED, which WorkManager 2.12.0 turns into a + * request for INTERNET and VALIDATED (NetworkTypeCompatKt.toNetworkRequest). + * WIFI: the in-worker check's rule, a validated Wi-Fi network. WorkManager + * uses the request on API 28 and later; UNMETERED is its fallback below. + * A new NetworkRequest.Builder also asks for NOT_VPN and NOT_RESTRICTED; + * WorkManager removes both from its own CONNECTED request, and so does + * this one, so a VPN over Wi-Fi counts. + */ + private fun constraints(network: RequiredNetwork): Constraints = when (network) { + RequiredNetwork.WIFI -> Constraints.Builder() + .setRequiredNetworkRequest( + NetworkRequest.Builder() + .addTransportType(NetworkCapabilities.TRANSPORT_WIFI) + .addCapability(NetworkCapabilities.NET_CAPABILITY_INTERNET) + .addCapability(NetworkCapabilities.NET_CAPABILITY_VALIDATED) + .removeCapability(NetworkCapabilities.NET_CAPABILITY_NOT_VPN) + .removeCapability(NetworkCapabilities.NET_CAPABILITY_NOT_RESTRICTED) + .build(), + NetworkType.UNMETERED, + ) + .build() + else -> Constraints.Builder().setRequiredNetworkType(NetworkType.CONNECTED).build() + } + private fun row(info: WorkInfo) = WorkRow( entryId = info.tags.firstOrNull { it.startsWith(WorkManagerScheduler.ID_TAG_PREFIX) } ?.removePrefix(WorkManagerScheduler.ID_TAG_PREFIX), @@ -386,21 +449,12 @@ private class WorkManagerRows(private val workManager: WorkManager) : WorkRows { runAttemptCount = info.runAttemptCount, nextScheduleAt = info.nextScheduleTimeMillis, delayed = info.initialDelayMillis > 0, - network = networkOf(info.constraints), + doorbell = info.tags.firstOrNull { it.startsWith(WorkManagerScheduler.DOORBELL_TAG_PREFIX) } + ?.removePrefix(WorkManagerScheduler.DOORBELL_TAG_PREFIX) + ?.let { name -> RequiredNetwork.values().firstOrNull { it.name == name } }, ) - // On API 28 and later setRequiredNetworkRequest stores the request and - // sets the type to NOT_REQUIRED (Constraints.Builder in 2.12.0). - private fun networkOf(c: Constraints): RequiredNetwork { - val request = c.requiredNetworkRequest - return when { - request != null && request.hasTransport(NetworkCapabilities.TRANSPORT_WIFI) -> RequiredNetwork.WIFI - request != null || c.requiredNetworkType != NetworkType.NOT_REQUIRED -> RequiredNetwork.CONNECTED - else -> RequiredNetwork.NONE - } - } - - private fun request(entry: QueueEntry, delayMs: Long, network: RequiredNetwork): OneTimeWorkRequest { + private fun request(entry: QueueEntry, delayMs: Long): OneTimeWorkRequest { val builder = if (entry.body?.kind == StagedBody.CHUNKED) { OneTimeWorkRequestBuilder() } else { @@ -414,42 +468,6 @@ private class WorkManagerRows(private val workManager: WorkManager) : WorkRows { .addTag(WorkManagerScheduler.ID_TAG_PREFIX + entry.id) .setInputData(workDataOf(WorkManagerScheduler.ENTRY_ID_KEY to entry.id)) .setInitialDelay(delayMs, TimeUnit.MILLISECONDS) - .setConstraints(constraints(network)) .build() } - - /** - * The Wi-Fi request is the in-worker check's rule: a validated Wi-Fi - * network. WorkManager uses the request on API 28 and later (minSdk is - * 29); UNMETERED is only its fallback for older API levels. A new - * NetworkRequest.Builder also asks for NOT_VPN and NOT_RESTRICTED; - * WorkManager 2.12.0 removes both from the request it builds for - * CONNECTED (NetworkTypeCompatKt.toNetworkRequest), and so does this one. - * - * API 29 gets no constraint, as before: there WorkManager tracks each - * constrained row with its own network callback (IndividualNetworkCallback), - * and the app's callbacks share a limit of 100. Its runs keep the - * in-worker poll ([TransferHost.networkWakes]). - */ - private fun constraints(network: RequiredNetwork): Constraints { - if (Build.VERSION.SDK_INT < Build.VERSION_CODES.R) return Constraints.NONE - return when (network) { - RequiredNetwork.NONE -> Constraints.NONE - RequiredNetwork.CONNECTED -> Constraints.Builder().setRequiredNetworkType(NetworkType.CONNECTED).build() - RequiredNetwork.WIFI -> Constraints.Builder() - .setRequiredNetworkRequest( - NetworkRequest.Builder() - .addTransportType(NetworkCapabilities.TRANSPORT_WIFI) - .addCapability(NetworkCapabilities.NET_CAPABILITY_INTERNET) - .addCapability(NetworkCapabilities.NET_CAPABILITY_VALIDATED) - // As WorkManager's own CONNECTED request: a VPN over Wi-Fi counts. - .removeCapability(NetworkCapabilities.NET_CAPABILITY_NOT_VPN) - .removeCapability(NetworkCapabilities.NET_CAPABILITY_NOT_RESTRICTED) - .build(), - NetworkType.UNMETERED, - ) - .build() - } - } - } diff --git a/android/src/main/java/ai/openspace/backgroundupload/TransferHost.kt b/android/src/main/java/ai/openspace/backgroundupload/TransferHost.kt index 8db21dfa..ade70880 100644 --- a/android/src/main/java/ai/openspace/backgroundupload/TransferHost.kt +++ b/android/src/main/java/ai/openspace/backgroundupload/TransferHost.kt @@ -32,11 +32,11 @@ internal interface TransferHost { suspend fun foreground(entry: QueueEntry): Boolean /** - * Whether a run that finds no network may hand the wait to a wake row - * with a network constraint: API 30 and later. On API 29 WorkManager - * tracks each constrained row with its own network callback. + * The device's API level. It decides whether a run that finds no network + * hands over to a doorbell (API 30 and later), and whether a run with + * foreground mode does so too (API 35 and later). */ - fun networkWakes(): Boolean + fun sdkInt(): Int fun progressStarted(id: String, total: Long, sent: Long) diff --git a/android/src/main/java/ai/openspace/backgroundupload/UploadNotification.kt b/android/src/main/java/ai/openspace/backgroundupload/UploadNotification.kt index 4a31a3e8..efeaa48c 100644 --- a/android/src/main/java/ai/openspace/backgroundupload/UploadNotification.kt +++ b/android/src/main/java/ai/openspace/backgroundupload/UploadNotification.kt @@ -20,11 +20,29 @@ import androidx.work.ForegroundInfo // identified by the NotificationConfig that configure() saved. Its progress // bar is the total across the uploads. -internal enum class Connectivity { NoWifi, NoInternet, Ok } +/** + * The network check's answer. [BoundElsewhere]: the default network fits a + * Wi-Fi-only upload, but the app binds the process to a network that is not + * Wi-Fi, so the upload's sockets would not use Wi-Fi. That does not last + * (the app binds for one call), so a run waits for it and does not hand over. + */ +internal enum class Connectivity { NoWifi, NoInternet, Ok, BoundElsewhere } + +/** + * Runs a network check that must not throw: an error reads as no network, + * so the caller waits. WorkManager catches a SecurityException around the + * same ConnectivityManager calls. + */ +internal inline fun safeConnectivity(wifiOnly: Boolean, check: () -> Connectivity): Connectivity = + try { + check() + } catch (error: Exception) { + if (wifiOnly) Connectivity.NoWifi else Connectivity.NoInternet + } // This is synchronized to ensure consistent status across workers @Synchronized -internal fun validateConnectivity(context: Context, wifiOnly: Boolean): Connectivity { +internal fun validateConnectivity(context: Context, wifiOnly: Boolean): Connectivity = safeConnectivity(wifiOnly) { val manager = context.getSystemService(Context.CONNECTIVITY_SERVICE) as ConnectivityManager // activeNetwork is the system default network, not a network the process // is bound to (ConnectivityManager docs). @@ -32,7 +50,7 @@ internal fun validateConnectivity(context: Context, wifiOnly: Boolean): Connecti // While the app binds the process to a network (bindProcessToNetwork), new // sockets and DNS use that network, also OkHttp's. val bound = manager.boundNetworkForProcess - return connectivityOf( + connectivityOf( wifiOnly = wifiOnly, validated = capabilities?.hasCapability(NET_CAPABILITY_VALIDATED) == true, defaultWifi = capabilities?.hasTransport(TRANSPORT_WIFI) == true, @@ -43,16 +61,26 @@ internal fun validateConnectivity(context: Context, wifiOnly: Boolean): Connecti /** * The network check. The default network must be validated. Wi-Fi only * also needs it to be Wi-Fi, and a process bound to a network that is not - * Wi-Fi fails it: its sockets would send over that network. [boundWifi] is - * null when the process is not bound. Wi-Fi only answers NoWifi, not - * NoInternet: asking the user to join Wi-Fi is the more direct hint. + * Wi-Fi does not pass: its sockets would send over that network + * ([Connectivity.BoundElsewhere]). [boundWifi] is null when the process is + * not bound. Wi-Fi only answers NoWifi, not NoInternet: asking the user to + * join Wi-Fi is the more direct hint. */ internal fun connectivityOf(wifiOnly: Boolean, validated: Boolean, defaultWifi: Boolean, boundWifi: Boolean?): Connectivity = when { !wifiOnly -> if (validated) Connectivity.Ok else Connectivity.NoInternet - validated && defaultWifi && boundWifi != false -> Connectivity.Ok - else -> Connectivity.NoWifi + !validated || !defaultWifi -> Connectivity.NoWifi + boundWifi == false -> Connectivity.BoundElsewhere + else -> Connectivity.Ok } +/** + * Whether the network fits an upload with this Wi-Fi rule now. A process + * bound elsewhere counts: the default network fits, and a run waits out the + * bind. Never throws. + */ +internal fun networkFitsNow(context: Context, wifiOnly: Boolean): Boolean = + validateConnectivity(context, wifiOnly).let { it == Connectivity.Ok || it == Connectivity.BoundElsewhere } + // Makes sure that the channel for the foreground notification exists. It makes // the channel only when the channel is absent. Thus a channel that the consumer // registered, with their own name and importance, always wins. When configure() @@ -79,7 +107,7 @@ internal fun buildUploadNotification( val progress = UploadProgress.total() val progress2Decimals = "%.2f".format(progress) val title = when (connectivity) { - Connectivity.NoWifi -> config.notificationTitleNoWifi + Connectivity.NoWifi, Connectivity.BoundElsewhere -> config.notificationTitleNoWifi Connectivity.NoInternet -> config.notificationTitleNoInternet Connectivity.Ok -> config.notificationTitle } diff --git a/android/src/main/java/ai/openspace/backgroundupload/UploaderModule.kt b/android/src/main/java/ai/openspace/backgroundupload/UploaderModule.kt index 38f990e6..a49e3c03 100644 --- a/android/src/main/java/ai/openspace/backgroundupload/UploaderModule.kt +++ b/android/src/main/java/ai/openspace/backgroundupload/UploaderModule.kt @@ -1,5 +1,6 @@ package ai.openspace.backgroundupload +import android.os.Build import com.facebook.react.bridge.Arguments import com.facebook.react.bridge.LifecycleEventListener import com.facebook.react.bridge.Promise @@ -55,7 +56,8 @@ class UploaderModule(context: ReactApplicationContext) : private val scheduler = WorkManagerScheduler(context) private val controller = QueueController( store, journal, settings, EventReporter, scheduler, - networkFits = { validateConnectivity(context, settings.load().wifiOnlyFor(it)) == Connectivity.Ok }, + networkFits = { wifiOnly -> networkFitsNow(context, wifiOnly) }, + doorbells = Build.VERSION.SDK_INT >= EntryRun.DOORBELL_MIN_SDK, ) // Before init: the lifecycle listener it registers can fire at once. diff --git a/android/src/main/java/ai/openspace/backgroundupload/WorkerOps.kt b/android/src/main/java/ai/openspace/backgroundupload/WorkerOps.kt index 7c4cba81..3da2e3af 100644 --- a/android/src/main/java/ai/openspace/backgroundupload/WorkerOps.kt +++ b/android/src/main/java/ai/openspace/backgroundupload/WorkerOps.kt @@ -329,7 +329,7 @@ class WorkerOps( if (result == ParkResult.PARKED && next != null) { events.state(next.toRow()) // A parked entry still expires on time. A far expiry waits for a later boot sweep. - if (expiryWakeIsNear(next.expiresAt, now)) scheduler.scheduleWake(next, next.expiresAt, replace = true, requiredNetwork(next, settings())) + if (expiryWakeIsNear(next.expiresAt, now)) scheduler.scheduleWake(next, next.expiresAt, replace = true) } return result } @@ -361,7 +361,7 @@ class WorkerOps( events.state(next.toRow()) if (!wake) return true try { - scheduler.scheduleWake(next, nextAttemptAt, replace = true, requiredNetwork(next, settings())) + scheduler.scheduleWake(next, nextAttemptAt, replace = true) } catch (error: Exception) { throw WakeNotScheduledException(id, error) } @@ -379,7 +379,7 @@ class WorkerOps( * entry save and WorkManager's commit. Never throws. */ fun ensureWake(entry: QueueEntry, at: Long) { - runCatching { scheduler.scheduleWake(entry, at, replace = false, requiredNetwork(entry, settings())) } + runCatching { scheduler.scheduleWake(entry, at, replace = false) } .onFailure { Diag.error("could not schedule a wake for '${entry.id}'", it) } } @@ -391,9 +391,7 @@ class WorkerOps( * nextAttemptAt or now. A time-limit stop needs it when WorkManager holds * the stopped row back: it runs that row again only after its own * backoff, which grows with every start of that row (up to 5 hours). A - * fresh row starts its count at 0. The wake waits for the network the - * entry needs ([requiredNetwork]), so a run stopped while it waited for - * the network does not start again offline. + * fresh row starts its count at 0. */ fun stopped(id: String, generation: Int, wake: Boolean = false) { val now = clock() @@ -411,26 +409,29 @@ class WorkerOps( if (wake) { // A stop must not throw past the run. Without the wake, the stopped // row still runs again, after WorkManager's own backoff. - runCatching { scheduler.scheduleWake(next, next.nextAttemptAt ?: now, replace = true, requiredNetwork(next, settings())) } + runCatching { scheduler.scheduleWake(next, next.nextAttemptAt ?: now, replace = true) } .onFailure { Diag.error("could not schedule a wake for '$id' after a stop", it) } } } /** - * The run found no usable network and hands the wait to WorkManager: - * back to queued, and a wake now that waits for [network]. Not a failure: - * no attempt, no backoff step, no wait cause, no nextAttemptAt - * ([EntryTransitions.toStopped] keeps what the last failure saved). The - * wake replaces any other wake of the entry. False when the run no - * longer owns the entry (a pause, cancel or replace landed first); then - * nothing is scheduled. + * The run found no network that fits the entry, and hands the wait to a + * doorbell: back to queued, with no row of its own, and the doorbell for + * the entry's network made sure ([WorkScheduler.ensureDoorbell]). The rule + * is read now, from the stored entry and the current settings, so a + * setWifiOnly() change during the wait counts. * - * Throws [WakeNotScheduledException] when the wake could not be - * enqueued. The entry is saved queued. The caller's worker then returns - * retry, so WorkManager runs its row again; the boot sweep and a return - * to the foreground also start a queued entry with no row that can start. + * Not a failure: no attempt, no backoff step, no wait cause and no new + * nextAttemptAt ([EntryTransitions.toStopped] keeps what the last failure + * saved). False when the run no longer owns the entry (a pause, cancel or + * replace landed first); then nothing is scheduled. + * + * Throws [WakeNotScheduledException] when the doorbell could not be + * stored. The entry is saved queued. The worker then returns retry, so + * WorkManager runs the run's own row again. The boot sweep, resume() and + * a return to the foreground also start or ring for it. */ - fun awaitNetwork(id: String, generation: Int, network: RequiredNetwork): Boolean { + fun awaitNetwork(id: String, generation: Int): Boolean { val now = clock() var applied = false val next = store.compute(id) { e -> @@ -441,8 +442,10 @@ class WorkerOps( } if (!applied || next == null) return false events.state(next.toRow()) + val network = requiredNetwork(next, settings(), now) + if (network == RequiredNetwork.NONE) return true try { - scheduler.scheduleWake(next, now, replace = true, network) + scheduler.ensureDoorbell(network) } catch (error: Exception) { throw WakeNotScheduledException(id, error) } diff --git a/android/src/test/java/ai/openspace/backgroundupload/EntryRunTest.kt b/android/src/test/java/ai/openspace/backgroundupload/EntryRunTest.kt index 0749d6f2..09884d2c 100644 --- a/android/src/test/java/ai/openspace/backgroundupload/EntryRunTest.kt +++ b/android/src/test/java/ai/openspace/backgroundupload/EntryRunTest.kt @@ -41,8 +41,8 @@ internal class FakeHost(var clock: Long = 10_000L) : TransferHost { var foregroundError: Throwable? = null /** What foreground() answers: whether the run holds foreground mode. */ var foregroundGranted = true - /** What networkWakes() answers: false is API 29. */ - var networkWake = true + /** What sdkInt() answers. */ + var sdk = 35 /** sleep() also yields, so a sibling part coroutine can run. */ var yieldOnSleep = false /** The wifiOnly value of every connectivity check, in order. */ @@ -72,6 +72,10 @@ internal class FakeHost(var clock: Long = 10_000L) : TransferHost { override fun connectivity(wifiOnly: Boolean): Connectivity { wifiChecks += wifiOnly + if (connectivityErrors > 0) { + connectivityErrors-- + throw SecurityException("Package android does not belong to 10123") + } return network.removeFirstOrNull() ?: Connectivity.Ok } @@ -80,7 +84,7 @@ internal class FakeHost(var clock: Long = 10_000L) : TransferHost { return foregroundGranted } - override fun networkWakes() = networkWake + override fun sdkInt() = sdk override fun progressStarted(id: String, total: Long, sent: Long) { progress += "start:$total:$sent" @@ -92,8 +96,14 @@ internal class FakeHost(var clock: Long = 10_000L) : TransferHost { override fun progressEnded(id: String, completed: Boolean) { progress += "end:$completed" + onProgressEnded() } + var onProgressEnded: () -> Unit = {} + + /** How many of the next connectivity() calls throw, as a ConnectivityManager call can. */ + var connectivityErrors = 0 + override fun attempt(event: AttemptEvent) { attempts += event } @@ -397,8 +407,6 @@ class EntryRunTest { run() assertEquals(EntryState.AWAITING_AUTH, store.load("e1")!!.state) assertEquals(listOf("e1" to 90_000L), scheduler.wakes) - // The expiry wake settles the entry; that needs no network. - assertEquals(listOf(RequiredNetwork.NONE), scheduler.wakeNetworks) assertEquals(listOf("error"), host.attempts.map { it.outcome }) assertEquals(emptyList(), journal.unacknowledged()) } @@ -560,8 +568,6 @@ class EntryRunTest { // A fresh wake row, even for a wait a worker could sleep out in place. assertEquals(listOf("e1" to host.clock + 4_000), scheduler.wakes) assertEquals(listOf(true), scheduler.wakeReplaces) - // A backoff wake waits for a network too, so it does not start a run offline. - assertEquals(listOf(RequiredNetwork.CONNECTED), scheduler.wakeNetworks) } @Test @@ -592,20 +598,6 @@ class EntryRunTest { assertEquals(emptyList(), host.requests) assertEquals(listOf("e1" to host.clock), scheduler.wakes) assertEquals(listOf(true), scheduler.wakeReplaces) - // The fresh row waits for the network, so an offline entry does not start, poll and stop again. - assertEquals(listOf(RequiredNetwork.CONNECTED), scheduler.wakeNetworks) - } - - @Test - fun `a timeout stop while a Wi-Fi-only entry waits for Wi-Fi wakes it on Wi-Fi`() { - controller.setWifiOnly(true) - controller.enqueue(parsed()) - host.stop = StopKind.TIME_LIMIT - host.network = ArrayDeque(listOf(Connectivity.NoWifi)) - host.onSleep = { throw CancellationException("timeout") } - assertThrows(CancellationException::class.java) { run() } - assertEquals(listOf("e1" to host.clock), scheduler.wakes) - assertEquals(listOf(RequiredNetwork.WIFI), scheduler.wakeNetworks) } @Test @@ -903,10 +895,10 @@ class EntryRunTest { assertEquals(EntryState.COMPLETED, store.load("e1")!!.state) } - // MARK: - waiting for the network on a constrained wake + // MARK: - waiting for the network on a doorbell @Test - fun `a run with no foreground mode and no network for 60 s goes back to queued and waits on a network wake`() { + fun `a run with no foreground mode and no network for 60 s goes back to queued and waits on the network doorbell`() { controller.enqueue(parsed()) host.foregroundGranted = false host.network = ArrayDeque(List(7) { Connectivity.NoInternet }) @@ -924,9 +916,9 @@ class EntryRunTest { // Seven checks, 10 s apart: 60 s in all. assertEquals(7, host.wifiChecks.size) assertEquals(EntryRun.HEADLESS_NETWORK_WAIT_MS, host.sleeps.sum()) - assertEquals(listOf("e1" to host.clock), scheduler.wakes) - assertEquals(listOf(true), scheduler.wakeReplaces) - assertEquals(listOf(RequiredNetwork.CONNECTED), scheduler.wakeNetworks) + // No row of its own: the shared doorbell starts it. + assertEquals(listOf(RequiredNetwork.CONNECTED), scheduler.doorbells) + assertEquals(emptyList>(), scheduler.wakes) assertEquals("state:e1:queued", events.log.last()) assertEquals("end:false", host.progress.last()) } @@ -939,7 +931,7 @@ class EntryRunTest { respond(200) run() assertEquals(EntryState.COMPLETED, store.load("e1")!!.state) - assertEquals(emptyList>(), scheduler.wakes) + assertEquals(emptyList(), scheduler.doorbells) } @Test @@ -959,7 +951,7 @@ class EntryRunTest { } @Test - fun `a Wi-Fi-only entry hands over with the Wi-Fi constraint`() { + fun `a Wi-Fi-only entry hands over to the Wi-Fi doorbell`() { controller.setWifiOnly(true) controller.enqueue(parsed(id = "follow")) controller.enqueue(parsed(id = "own", descriptor = desc(dataJson = """{"a":1}""", wifiOnly = true))) @@ -969,12 +961,34 @@ class EntryRunTest { host.network = ArrayDeque(List(7) { Connectivity.NoWifi }) controller.setWifiOnly(false) run("own") - assertEquals(listOf(RequiredNetwork.WIFI, RequiredNetwork.WIFI), scheduler.wakeNetworks) - assertEquals(EntryState.QUEUED, store.load("own")!!.state) + assertEquals(listOf(RequiredNetwork.WIFI, RequiredNetwork.WIFI), scheduler.doorbells) } @Test - fun `a run that holds foreground mode keeps the poll for 30 minutes`() { + fun `a hand-over uses the Wi-Fi setting at the hand-over, not the one the run read first`() { + controller.setWifiOnly(true) + controller.enqueue(parsed()) + host.foregroundGranted = false + host.network = ArrayDeque(List(7) { Connectivity.NoWifi }) + host.onSleep = { if (host.sleeps.size == 6) controller.setWifiOnly(false) } + run() + assertEquals(listOf(RequiredNetwork.CONNECTED), scheduler.doorbells) + } + + @Test + fun `a run that holds foreground mode on API 35 polls for 30 minutes, then hands over`() { + controller.enqueue(parsed()) + host.foregroundGranted = true + host.sdk = 35 + host.network = ArrayDeque(List(181) { Connectivity.NoInternet }) + run() + assertEquals(EntryState.QUEUED, store.load("e1")!!.state) + assertEquals(EntryRun.FOREGROUND_NETWORK_WAIT_MS, host.sleeps.sum()) + assertEquals(listOf(RequiredNetwork.CONNECTED), scheduler.doorbells) + } + + @Test + fun `a run that holds foreground mode on API 35 that finds the network within 30 minutes sends`() { controller.enqueue(parsed()) host.foregroundGranted = true host.network = ArrayDeque(List(180) { Connectivity.NoInternet }) @@ -982,70 +996,112 @@ class EntryRunTest { run() assertEquals(EntryState.COMPLETED, store.load("e1")!!.state) assertEquals(EntryRun.FOREGROUND_NETWORK_WAIT_MS, host.sleeps.sum()) - assertEquals(emptyList>(), scheduler.wakes) } @Test - fun `a run that holds foreground mode hands over after 30 minutes with no network`() { + fun `below API 35 a run that holds foreground mode keeps the poll`() { controller.enqueue(parsed()) host.foregroundGranted = true - host.network = ArrayDeque(List(181) { Connectivity.NoInternet }) + host.sdk = 34 + host.network = ArrayDeque(List(300) { Connectivity.NoInternet }) respond(200) run() - assertEquals(EntryState.QUEUED, store.load("e1")!!.state) - assertEquals(EntryRun.FOREGROUND_NETWORK_WAIT_MS, host.sleeps.sum()) - assertEquals(listOf(RequiredNetwork.CONNECTED), scheduler.wakeNetworks) - assertEquals(emptyList(), host.requests) + assertEquals(EntryState.COMPLETED, store.load("e1")!!.state) + assertEquals(3_000_000L, host.sleeps.sum()) + assertEquals(emptyList(), scheduler.doorbells) } @Test fun `on API 29 a run with no foreground mode keeps the poll`() { controller.enqueue(parsed()) host.foregroundGranted = false - host.networkWake = false + host.sdk = 29 host.network = ArrayDeque(List(20) { Connectivity.NoInternet }) respond(200) run() assertEquals(EntryState.COMPLETED, store.load("e1")!!.state) assertEquals(200_000L, host.sleeps.sum()) - assertEquals(emptyList>(), scheduler.wakes) + assertEquals(emptyList(), scheduler.doorbells) } @Test - fun `a hand-over whose wake can not be scheduled leaves the entry queued and fails the run, so the worker retries`() { - val file = File(tmp.newFolder(), "diagnostics.json") - Diag.log = DiagnosticLog(file) + fun `a Wi-Fi-only run waits out a process bound to cellular and does not hand over`() { + controller.setWifiOnly(true) + controller.enqueue(parsed()) + host.foregroundGranted = false + host.network = ArrayDeque(List(20) { Connectivity.BoundElsewhere }) + respond(200) + run() + assertEquals(EntryState.COMPLETED, store.load("e1")!!.state) + assertEquals(200_000L, host.sleeps.sum()) + assertEquals(emptyList(), scheduler.doorbells) + } + + @Test + fun `a network check that throws reads as no network, so the run waits and sends`() { + controller.enqueue(parsed()) + host.connectivityErrors = 3 + respond(200) + run() + val e = store.load("e1")!! + assertEquals(EntryState.COMPLETED, e.state) + assertEquals(1, e.attempts) + assertEquals(3 * EntryRun.CONNECTIVITY_POLL_MS, host.sleeps.sum()) + } + + @Test + fun `a hand-over whose doorbell can not be stored leaves the entry queued and fails the run, so the worker retries`() { controller.enqueue(parsed()) host.foregroundGranted = false host.network = ArrayDeque(List(7) { Connectivity.NoInternet }) - scheduler.wakeFailures = 1 + scheduler.doorbellFailures = 1 assertThrows(WorkerOps.WakeNotScheduledException::class.java) { run() } val e = store.load("e1")!! assertEquals(EntryState.QUEUED, e.state) assertEquals(0, e.attempts) - assertEquals(emptyList>(), scheduler.wakes) } @Test - fun `a pause while it waits to hand over stops the run with no wake`() { + fun `a pause while it waits to hand over stops the run with no doorbell`() { controller.enqueue(parsed()) host.foregroundGranted = false host.network = ArrayDeque(List(7) { Connectivity.NoInternet }) host.onSleep = { if (host.sleeps.size == 5) controller.pause() } run() assertEquals(EntryState.PAUSED, store.load("e1")!!.state) - assertEquals(emptyList>(), scheduler.wakes) + assertEquals(emptyList(), scheduler.doorbells) } @Test fun `the send gate refuses a hand-over while a request is sending, and a send once a hand-over began`() { val gate = EntryRun.SendGate() gate.beginSend() - assertFalse(gate.tryHandOver(RequiredNetwork.WIFI)) + assertFalse(gate.tryHandOver()) gate.endSend() - assertTrue(gate.tryHandOver(RequiredNetwork.WIFI)) - val refused = assertThrows(EntryRun.AwaitNetworkException::class.java) { gate.beginSend() } - assertEquals(RequiredNetwork.WIFI, refused.network) + assertTrue(gate.tryHandOver()) + assertThrows(EntryRun.AwaitNetworkException::class.java) { gate.beginSend() } + } + + @Test + fun `a pass after a re-issue starts with a fresh send gate`() { + // Pass 1 ends with a 401 that parks; updateHeaders() lands before the + // park, so the run goes on (REISSUE). A sibling's hand-over had begun + // in pass 1. Pass 2 must still send. + controller.enqueue(parsed()) + val r = EntryRun("e1", store, ops, host) + respond(401, 200) + var once = false + host.onProgressEnded = { + if (!once) { + once = true + assertTrue(r.sends.tryHandOver()) + controller.updateHeaders(mapOf("Authorization" to "Bearer new")) + } + } + runBlocking { r.run() } + assertEquals(EntryState.COMPLETED, store.load("e1")!!.state) + assertEquals(2, host.requests.size) + assertEquals(emptyList(), scheduler.doorbells) } // MARK: - transient rules @@ -1603,6 +1659,6 @@ class EntryRunTest { assertEquals(listOf("completed"), host.attempts.map { it.outcome }) // Part 1 waited past 60 s while part 0 was sending, then handed over. assertTrue(host.sleeps.sum() >= 120_000) - assertEquals(listOf(RequiredNetwork.CONNECTED), scheduler.wakeNetworks) + assertEquals(listOf(RequiredNetwork.CONNECTED), scheduler.doorbells) } } diff --git a/android/src/test/java/ai/openspace/backgroundupload/MainChainTest.kt b/android/src/test/java/ai/openspace/backgroundupload/MainChainTest.kt index eb08ff73..a88230f8 100644 --- a/android/src/test/java/ai/openspace/backgroundupload/MainChainTest.kt +++ b/android/src/test/java/ai/openspace/backgroundupload/MainChainTest.kt @@ -7,7 +7,6 @@ import androidx.work.WorkInfo.State.ENQUEUED import androidx.work.WorkInfo.State.RUNNING import org.junit.Assert.assertEquals import org.junit.Assert.assertNull -import org.junit.Assert.assertThrows import org.junit.Assert.assertTrue import org.junit.Before import org.junit.Rule @@ -40,10 +39,10 @@ class MainChainTest { val log = mutableListOf() var enqueueFails = false - override fun enqueue(name: String, policy: ExistingWorkPolicy, entry: QueueEntry, delayMs: Long, network: RequiredNetwork) { + override fun enqueue(name: String, policy: ExistingWorkPolicy, entry: QueueEntry, delayMs: Long) { if (enqueueFails) throw IllegalStateException("the process died before the enqueue") - log += "enqueue:$name:$policy" + if (network == RequiredNetwork.NONE) "" else ":$network" - work.enqueue(name, policy, entry, delayMs, network) + log += "enqueue:$name:$policy" + work.enqueue(name, policy, entry, delayMs) } override fun cancel(name: String) { @@ -61,17 +60,13 @@ class MainChainTest { journal = EventJournal(tmp.newFolder("journal"), retryLater = { _, _ -> }) settings = QueueSettingsStore(File(root, "settings.json")) scheduler = RowScheduler(rows) { work.now } - controller = QueueController(store, journal, settings, events, scheduler, { it in running }, { work.now }, ::networkFits) + controller = QueueController(store, journal, settings, events, scheduler, { it in running }, { work.now }) } /** A queued entry that waits after a transport failure 10 minutes ago. */ private fun transportWait(id: String = "e1", streak: Int = 12) = entry(id = id, state = EntryState.QUEUED, nextAttemptAt = work.now + 60 * minute) .copy(backoffStreak = streak, waitCause = WaitCause.NO_RESPONSE, failedAt = work.now - 10 * minute) - /** The in-worker check, on the fake's network. */ - private fun networkFits(e: QueueEntry) = - work.network.ordinal >= (if (settings.load().wifiOnlyFor(e)) RequiredNetwork.WIFI else RequiredNetwork.CONNECTED).ordinal - /** The rows of [id] WorkManager may start now, main chain and wake. */ private fun startable(id: String = "e1") = work.startableNow().filter { it.entryId == id } @@ -223,7 +218,7 @@ class MainChainTest { work.add("e1#wake", state = RUNNING, runAttemptCount = 1) running += "e1" controller.retryNow() - scheduler.scheduleWake(store.load("e1")!!, work.now + 60 * minute, replace = true, RequiredNetwork.CONNECTED) + scheduler.scheduleWake(store.load("e1")!!, work.now + 60 * minute, replace = true) running -= "e1" assertEquals(CANCELLED, held.state) assertNull(store.load("e1")!!.nextAttemptAt) @@ -259,158 +254,6 @@ class MainChainTest { assertTrue(WorkManagerBackoff.restartBy(startedAt, 4) < startedAt + 10 * minute) } - // MARK: - an offline entry sleeps on a network wake - - /** - * A run of e1 under [rowName] waited for [network] and handed over: the - * entry is queued, the wake waits for the network, and the run's own row - * ends (a REPLACE of the wake name cancels a wake row that runs). - */ - private fun handedOver(rowName: String = "e1", network: RequiredNetwork = RequiredNetwork.CONNECTED) { - store.save(entry(state = EntryState.RUNNING)) - val own = work.add(rowName, state = RUNNING, runAttemptCount = 1) - running += "e1" - val ops = WorkerOps(store, journal, settings, events, scheduler) { work.now } - assertTrue(ops.awaitNetwork("e1", 1, network)) - running -= "e1" - if (own.state == RUNNING) own.state = androidx.work.WorkInfo.State.SUCCEEDED - rows.log.clear() - } - - private fun openRows() = work.rows.filter { !it.state.isFinished } - - @Test - fun `a constrained row reports its enqueue time as the next schedule time, and starts only when its network is up`() { - val row = work.add("e1#wake", network = RequiredNetwork.WIFI) - assertEquals(work.now, work.nextScheduleAt(row)) - work.network = RequiredNetwork.NONE - assertEquals(emptyList(), work.startableNow()) - work.network = RequiredNetwork.CONNECTED - assertEquals(emptyList(), work.startableNow()) - work.network = RequiredNetwork.WIFI - assertEquals(listOf(row), work.startableNow()) - } - - @Test - fun `a hand-over leaves one row, a wake that waits for the network`() { - work.network = RequiredNetwork.NONE - handedOver() - assertEquals(EntryState.QUEUED, store.load("e1")!!.state) - val wake = openRows().single() - assertEquals("e1#wake", wake.name) - assertEquals(RequiredNetwork.CONNECTED, wake.network) - assertEquals(0L, wake.initialDelay) - assertEquals(emptyList(), startable()) - // The network comes back: WorkManager starts the wake. - work.network = RequiredNetwork.CONNECTED - assertEquals(listOf(wake), startable()) - } - - @Test - fun `a hand-over from a run under the wake name replaces that wake`() { - work.network = RequiredNetwork.NONE - handedOver(rowName = "e1#wake", network = RequiredNetwork.WIFI) - assertEquals(listOf(CANCELLED, ENQUEUED), work.rows.map { it.state }) - assertEquals(RequiredNetwork.WIFI, openRows().single().network) - } - - @Test - fun `a return to the foreground while offline adds no row next to a network wake`() { - work.network = RequiredNetwork.NONE - handedOver() - controller.retryNow() - controller.retryNow() - assertEquals(emptyList(), rows.log) - assertEquals(listOf("e1#wake"), openRows().map { it.name }) - } - - @Test - fun `opening the app with a network starts an entry on a network wake, also when WorkManager has not started the wake`() { - work.network = RequiredNetwork.NONE - handedOver() - work.network = RequiredNetwork.CONNECTED - controller.retryNow() - // A main row starts now; the wake stays. The per-id gate runs them one after the other. - assertEquals(listOf("enqueue:e1:APPEND_OR_REPLACE"), rows.log) - assertEquals(listOf("e1#wake", "e1"), startable().map { it.name }) - // The next foreground finds a row that starts now and adds nothing. - rows.log.clear() - controller.retryNow() - assertEquals(emptyList(), rows.log) - } - - @Test - fun `a return to the foreground on cellular adds no row next to a Wi-Fi wake`() { - controller.setWifiOnly(true) - work.network = RequiredNetwork.NONE - handedOver(network = RequiredNetwork.WIFI) - work.network = RequiredNetwork.CONNECTED - controller.retryNow() - assertEquals(emptyList(), rows.log) - assertEquals(emptyList(), startable()) - } - - @Test - fun `rowSummary lists a network wake as the entry's wake and as a network wait, not as a row that starts now`() { - work.network = RequiredNetwork.NONE - work.add("n#wake", network = RequiredNetwork.CONNECTED) - work.add("w#wake", network = RequiredNetwork.WIFI) - work.add("b#wake", initialDelay = minute, network = RequiredNetwork.CONNECTED) - val summary = scheduler.rowSummary() - assertEquals(emptySet(), summary.startable) - assertEquals(setOf("n", "w", "b"), summary.withWake) - assertEquals(setOf("n", "w"), summary.networkWait) - } - - @Test - fun `a waiting entry with a short wait and a network wake gets no second wake`() { - work.network = RequiredNetwork.NONE - handedOver() - // A chunked sibling's short backoff was still pending at the hand-over. - store.save(store.load("e1")!!.copy(nextAttemptAt = work.now + 20_000, waitCause = WaitCause.RESPONSE, failedAt = work.now)) - controller.retryNow() - assertEquals(listOf("e1#wake"), openRows().map { it.name }) - } - - @Test - fun `the boot sweep adds a main row for an entry on a network wake and keeps the wake`() { - // The sweep is the repair path: a main row runs whatever WorkManager's constraint tracking does. - work.network = RequiredNetwork.NONE - handedOver() - controller.sweep() - assertEquals(listOf("enqueue:e1:APPEND_OR_REPLACE"), rows.log) - assertEquals(listOf("e1#wake", "e1"), openRows().map { it.name }) - } - - @Test - fun `setWifiOnly(false) starts an entry on a Wi-Fi wake with a main row and cancels the Wi-Fi wake`() { - controller.setWifiOnly(true) - work.network = RequiredNetwork.CONNECTED - handedOver(network = RequiredNetwork.WIFI) - assertEquals(emptyList(), startable()) - controller.setWifiOnly(false) - assertEquals(listOf("enqueue:e1:APPEND_OR_REPLACE", "cancel:e1#wake"), rows.log) - assertEquals(listOf("e1"), startable().map { it.name }) - } - - @Test - fun `a process death between the hand-over save and the wake enqueue leaves the run's own row to start again`() { - work.network = RequiredNetwork.NONE - store.save(entry(state = EntryState.RUNNING)) - val own = work.add("e1", state = RUNNING, runAttemptCount = 1, lastEnqueueTime = work.now - 10 * minute) - val ops = WorkerOps(store, journal, settings, events, scheduler) { work.now } - rows.enqueueFails = true - assertThrows(WorkerOps.WakeNotScheduledException::class.java) { ops.awaitNetwork("e1", 1, RequiredNetwork.CONNECTED) } - assertEquals(EntryState.QUEUED, store.load("e1")!!.state) - // The process dies. At the next start WorkManager sets the RUNNING row back to ENQUEUED (ForceStopRunnable). - own.state = ENQUEUED - rows.enqueueFails = false - controller.retryNow() - assertEquals(listOf(own), work.rows.filter { !it.state.isFinished }) - // It has no constraint, so it starts now, waits for the network and hands over again. - assertEquals(listOf(own), startable()) - } - // MARK: - order, and cost @Test @@ -459,7 +302,7 @@ class MainChainTest { fun `a return to the foreground makes the wake again for a wait that stays and has no row`() { store.save(transportWait().copy(waitCause = WaitCause.RESPONSE)) controller.retryNow() - assertEquals(listOf("enqueue:e1#wake:KEEP:CONNECTED"), rows.log) + assertEquals(listOf("enqueue:e1#wake:KEEP"), rows.log) assertEquals(listOf("e1#wake"), work.rows.filter { !it.state.isFinished }.map { it.name }) // With a wake there, KEEP adds nothing. controller.retryNow() @@ -474,7 +317,7 @@ class MainChainTest { store.save(transportWait(streak = 9).copy(waitCause = WaitCause.RESPONSE, nextAttemptAt = work.now + 10 * minute)) work.add("e1", runAttemptCount = 10, lastEnqueueTime = work.now - minute) controller.retryNow() - assertEquals(listOf("enqueue:e1#wake:KEEP:CONNECTED"), rows.log) + assertEquals(listOf("enqueue:e1#wake:KEEP"), rows.log) val wake = work.rows.single { it.name == "e1#wake" } assertEquals(work.now + 10 * minute, work.nextScheduleAt(wake)) } @@ -498,8 +341,8 @@ class MainChainTest { return work.library(states) } - override fun enqueue(name: String, policy: ExistingWorkPolicy, entry: QueueEntry, delayMs: Long, network: RequiredNetwork) { - rows.enqueue(name, policy, entry, delayMs, network) + override fun enqueue(name: String, policy: ExistingWorkPolicy, entry: QueueEntry, delayMs: Long) { + rows.enqueue(name, policy, entry, delayMs) if (name == "e1") { val row = work.rows.last() row.state = RUNNING diff --git a/android/src/test/java/ai/openspace/backgroundupload/NetworkDoorbellTest.kt b/android/src/test/java/ai/openspace/backgroundupload/NetworkDoorbellTest.kt new file mode 100644 index 00000000..4e909c87 --- /dev/null +++ b/android/src/test/java/ai/openspace/backgroundupload/NetworkDoorbellTest.kt @@ -0,0 +1,515 @@ +package ai.openspace.backgroundupload + +import androidx.work.WorkInfo.State.BLOCKED +import androidx.work.WorkInfo.State.CANCELLED +import androidx.work.WorkInfo.State.ENQUEUED +import androidx.work.WorkInfo.State.RUNNING +import org.junit.After +import org.junit.Assert.assertEquals +import org.junit.Assert.assertFalse +import org.junit.Assert.assertThrows +import org.junit.Assert.assertTrue +import org.junit.Before +import org.junit.Rule +import org.junit.Test +import org.junit.rules.TemporaryFolder +import java.io.File + +/** + * An entry with no network sleeps on a shared doorbell row, against a fake + * that follows WorkManager 2.12.0 ([FakeWorkManager]). The questions: what + * starts each entry, does any upload run on a constrained row, and how many + * rows sleep. + */ +class NetworkDoorbellTest { + @get:Rule + val tmp = TemporaryFolder() + + private val minute = 60_000L + private val work = FakeWorkManager(now = 100 * 60_000L) + private val running = mutableSetOf() + private lateinit var store: QueueStore + private lateinit var settings: QueueSettingsStore + private lateinit var journal: EventJournal + private lateinit var scheduler: RowScheduler + private lateinit var controller: QueueController + private val events = RecordingEvents() + private var fitsThrows = false + private var fitsCalls = 0 + + @Before + fun setUp() { + val root = tmp.newFolder("queue") + store = QueueStore(root, RequestIndex()) + journal = EventJournal(tmp.newFolder("journal"), retryLater = { _, _ -> }) + settings = QueueSettingsStore(File(root, "settings.json")) + scheduler = RowScheduler(work) { work.now } + controller = newController() + } + + @After + fun noUploadOnAConstrainedRow() { + // Every row that runs an upload has no network rule; only a doorbell has one. + for (r in work.rows) { + if (r.doorbell == null) assertEquals("row ${r.name}", RequiredNetwork.NONE, r.network) + else assertEquals("row ${r.name}", r.doorbell, r.network) + } + } + + /** A controller as the module or a doorbell worker builds it, with the in-worker check on the fake's network. */ + private fun newController() = QueueController( + store, journal, settings, events, scheduler, { it in running }, { work.now }, + networkFits = { wifiOnly -> + fitsCalls++ + if (fitsThrows) throw SecurityException("Package android does not belong to 10123") + work.network.ordinal >= (if (wifiOnly) RequiredNetwork.WIFI else RequiredNetwork.CONNECTED).ordinal + }, + ) + + private fun ops() = WorkerOps(store, journal, settings, events, scheduler) { work.now } + + private fun network(n: RequiredNetwork) = work.setNetwork(n) + + /** The unfinished rows that run an upload of [id]. */ + private fun uploadRows(id: String = "e1") = work.rows.filter { it.entryId == id && it.doorbell == null && !it.state.isFinished } + + private fun doorbellRows(n: RequiredNetwork? = null) = + work.rows.filter { it.doorbell != null && (n == null || it.doorbell == n) && !it.state.isFinished } + + /** The rows WorkManager may start now that run an upload of [id]. */ + private fun startable(id: String = "e1") = work.startableNow().filter { it.entryId == id && it.doorbell == null } + + /** The rows that wait and are not running: what holds WorkManager's slots. */ + private fun sleeping() = work.rows.filter { it.state == ENQUEUED && it !in work.startableNow() } + + /** + * A run of [id] on its main row found no network and handed over. Its row + * then ends. + */ + private fun handedOver(id: String = "e1", stored: QueueEntry = entry(id = id, state = EntryState.RUNNING)) { + store.save(stored) + val own = work.add(id, state = RUNNING, runAttemptCount = 1) + running += id + assertTrue(ops().awaitNetwork(id, stored.generation)) + running -= id + work.finish(own) + } + + /** WorkManager starts each doorbell that may start now; its worker answers with its own controller. */ + private fun ringDoorbells(): List { + val rung = work.startableNow().filter { it.doorbell != null } + for (row in rung) { + work.start(row) + if (newController().answerDoorbell()) work.finish(row) else work.stop(row) + } + return rung + } + + // MARK: - the fake follows WorkManager 2.12.0 + + @Test + fun `the fake, a doorbell reports its enqueue time, starts only on its network, and is stopped when the network goes`() { + scheduler.ensureDoorbell(RequiredNetwork.WIFI) + val bell = doorbellRows().single() + assertEquals(work.now, work.nextScheduleAt(bell)) + network(RequiredNetwork.CONNECTED) + assertEquals(emptyList(), work.startableNow()) + network(RequiredNetwork.WIFI) + work.start(bell) + assertEquals(listOf(bell), network(RequiredNetwork.CONNECTED)) + assertEquals(ENQUEUED, bell.state) + assertEquals(1, bell.runAttemptCount) + } + + @Test + fun `the fake, with the app process dead only the 20 oldest ENQUEUED rows can start`() { + for (n in 1..25) work.add("x$n") + assertEquals(20, work.startableWhileDead().size) + assertEquals((1..20).map { "x$it" }, work.startableWhileDead().map { it.name }) + } + + // MARK: - a hand-over, and what starts the entry + + @Test + fun `after a hand-over the entry has no row of its own, and the network doorbell starts it when the network returns`() { + network(RequiredNetwork.NONE) + handedOver() + assertEquals(emptyList(), uploadRows()) + assertEquals(1, doorbellRows(RequiredNetwork.CONNECTED).size) + assertEquals(emptyList(), ringDoorbells()) // offline: it does not ring + network(RequiredNetwork.CONNECTED) + assertEquals(1, ringDoorbells().size) + assertEquals(1, startable().size) + // The entry has a row now, so the doorbell ends with none behind it. + assertEquals(emptyList(), doorbellRows()) + } + + @Test + fun `a Wi-Fi entry waits on the Wi-Fi doorbell through cellular, and starts on Wi-Fi`() { + controller.setWifiOnly(true) + network(RequiredNetwork.NONE) + handedOver() + assertEquals(1, doorbellRows(RequiredNetwork.WIFI).size) + network(RequiredNetwork.CONNECTED) + assertEquals(emptyList(), ringDoorbells()) + controller.retryNow() + assertEquals(emptyList(), uploadRows()) + network(RequiredNetwork.WIFI) + ringDoorbells() + assertEquals(1, startable().size) + } + + @Test + fun `the network doorbell on cellular starts the entries that may use cellular and leaves the Wi-Fi ones on the Wi-Fi doorbell`() { + network(RequiredNetwork.NONE) + handedOver("cell") + store.save(entry(id = "wifi", descriptor = desc(dataJson = """{"a":1}""", wifiOnly = true))) + controller.sweep() + assertEquals(1, doorbellRows(RequiredNetwork.CONNECTED).size) + assertEquals(1, doorbellRows(RequiredNetwork.WIFI).size) + network(RequiredNetwork.CONNECTED) + ringDoorbells() + assertEquals(1, startable("cell").size) + assertEquals(emptyList(), uploadRows("wifi")) + assertEquals(1, doorbellRows(RequiredNetwork.WIFI).size) + assertEquals(emptyList(), doorbellRows(RequiredNetwork.CONNECTED)) + } + + @Test + fun `a doorbell whose check finds no network after all gets one queued behind it, which rings later`() { + network(RequiredNetwork.NONE) + handedOver() + network(RequiredNetwork.CONNECTED) + val bell = doorbellRows().single() + work.start(bell) + // WorkManager saw the network; the in-worker check does not (not validated any more). + fitsThrows = true + assertTrue(newController().answerDoorbell()) + fitsThrows = false + work.finish(bell) + assertEquals(emptyList(), uploadRows()) + val next = doorbellRows().single() + assertTrue(next !== bell) + ringDoorbells() + assertEquals(1, startable().size) + } + + @Test + fun `a doorbell stopped by its own network rule runs again when the network returns`() { + network(RequiredNetwork.NONE) + handedOver() + network(RequiredNetwork.CONNECTED) + val bell = doorbellRows().single() + work.start(bell) + assertEquals(listOf(bell), network(RequiredNetwork.NONE)) // stopped before it answered + network(RequiredNetwork.CONNECTED) + work.now += 30_000 // WorkManager's backoff for a row started once + assertEquals(listOf(bell), ringDoorbells()) + assertEquals(1, startable().size) + } + + @Test + fun `a doorbell that WorkManager holds back after many stops is replaced by the next hand-over or repair`() { + network(RequiredNetwork.NONE) + handedOver() + val bell = doorbellRows().single() + bell.runAttemptCount = 9 // stopped nine times: held for over two hours + controller.sweep() + assertEquals(CANCELLED, bell.state) + val fresh = doorbellRows().single() + assertEquals(0, fresh.runAttemptCount) + network(RequiredNetwork.CONNECTED) + assertEquals(listOf(fresh), ringDoorbells()) + } + + @Test + fun `a hand-over while the doorbell runs gets one doorbell queued behind it, however many hand over`() { + network(RequiredNetwork.NONE) + handedOver("a") + network(RequiredNetwork.CONNECTED) + val bell = doorbellRows().single() + work.start(bell) + handedOver("b") + handedOver("c") + assertEquals(listOf(RUNNING, BLOCKED), doorbellRows().map { it.state }) + } + + @Test + fun `the doorbell rings with the app process dead and its worker starts the entries`() { + network(RequiredNetwork.NONE) + for (n in 1..3) handedOver("e$n") + network(RequiredNetwork.CONNECTED) + // Only JobScheduler starts rows now; the doorbell is among the rows it holds. + val bell = doorbellRows().single() + assertTrue(bell in work.startableWhileDead()) + work.start(bell) + assertTrue(newController().answerDoorbell()) // the worker's own controller, no module + work.finish(bell) + for (n in 1..3) assertTrue("e$n", startable("e$n").single() in work.startableWhileDead()) + } + + @Test + fun `250 entries on the Wi-Fi doorbell do not starve an entry that may use cellular`() { + controller.setWifiOnly(true) + network(RequiredNetwork.CONNECTED) + for (n in 1..250) store.save(entry(id = "w$n")) + controller.sweep() + // One sleeping row for all 250. + assertEquals(1, sleeping().size) + assertEquals(0, work.rows.count { it.doorbell == null }) + controller.enqueue(parsed(id = "cell", descriptor = desc(dataJson = """{"a":1}""", wifiOnly = false))) + // The app process dies: only the 20 oldest ENQUEUED rows can start. + assertEquals(1, startable("cell").size) + assertTrue(startable("cell").single() in work.startableWhileDead()) + assertEquals(1, sleeping().size) + } + + // MARK: - the doorbell is lost + + @Test + fun `a doorbell insert that fails makes the run fail, so its own row runs again, and a repair makes the doorbell`() { + network(RequiredNetwork.NONE) + store.save(entry(state = EntryState.RUNNING)) + val own = work.add("e1", state = RUNNING, runAttemptCount = 1) + work.doorbellInsertFailures = 1 + assertThrows(WorkerOps.WakeNotScheduledException::class.java) { ops().awaitNetwork("e1", 1) } + assertEquals(EntryState.QUEUED, store.load("e1")!!.state) + assertEquals(emptyList(), doorbellRows()) + work.stop(own) // the worker returns retry + // An app start offline makes the doorbell and adds no row. + controller.sweep() + assertEquals(1, doorbellRows(RequiredNetwork.CONNECTED).size) + assertEquals(listOf(own), uploadRows()) + } + + @Test + fun `a process death between the hand-over save and the doorbell leaves an entry that every repair finds`() { + // The entry is queued with no row: the run's own row ended, the doorbell never landed. + network(RequiredNetwork.NONE) + store.save(entry(state = EntryState.QUEUED)) + controller.retryNow() + assertEquals(1, doorbellRows(RequiredNetwork.CONNECTED).size) + assertEquals(emptyList(), uploadRows()) + // The same with the network up: a foreground starts it. + work.rows.clear() + network(RequiredNetwork.CONNECTED) + controller.retryNow() + assertEquals(1, startable().size) + assertEquals(emptyList(), doorbellRows()) + } + + @Test + fun `a doorbell insert that fails at a repair gives the entry its main row`() { + network(RequiredNetwork.NONE) + store.save(entry(state = EntryState.QUEUED)) + work.doorbellInsertFailures = 1 + controller.retryNow() + assertEquals(1, uploadRows().size) + assertEquals(emptyList(), doorbellRows()) + } + + // MARK: - one row per entry after each repair + + /** e1 was enqueued, ran offline and handed over. */ + private fun waitingOffline() { + controller.enqueue(parsed()) + work.rows.clear() + network(RequiredNetwork.NONE) + handedOver(stored = store.load("e1")!!.copy(state = EntryState.RUNNING)) + } + + @Test + fun `offline, the boot sweep, resume and a foreground add no row and keep one doorbell`() { + waitingOffline() + for (repair in listOf<() -> Unit>({ controller.sweep() }, { controller.resume() }, { controller.retryNow() }, { controller.enqueue(parsed()) })) { + repair() + assertEquals(emptyList(), uploadRows()) + assertEquals(1, doorbellRows().size) + } + } + + @Test + fun `online, the boot sweep adds one row for an entry that waited`() { + waitingOffline() + network(RequiredNetwork.CONNECTED) + controller.sweep() + controller.sweep() + assertEquals(1, uploadRows().size) + assertEquals(1, startable().size) + } + + @Test + fun `online, resume adds one row for an entry that waited`() { + waitingOffline() + network(RequiredNetwork.CONNECTED) + controller.resume() + controller.resume() + assertEquals(1, uploadRows().size) + assertEquals(1, startable().size) + } + + @Test + fun `online, a foreground adds one row for an entry that waited`() { + waitingOffline() + network(RequiredNetwork.CONNECTED) + controller.retryNow() + controller.retryNow() + assertEquals(1, uploadRows().size) + assertEquals(1, startable().size) + } + + @Test + fun `an offline app start starts no run`() { + network(RequiredNetwork.NONE) + for (n in 1..5) store.save(entry(id = "q$n")) + store.save(entry(id = "r", state = EntryState.RUNNING)) // its process died mid-run + controller.sweep() + assertEquals(0, work.rows.count { it.doorbell == null }) + assertEquals(1, doorbellRows().size) + } + + @Test + fun `a foreground checks the network at most twice, whatever the number of entries`() { + network(RequiredNetwork.NONE) + for (n in 1..10) store.save(entry(id = "q$n", descriptor = desc(dataJson = """{"a":1}""", wifiOnly = n % 2 == 0))) + fitsCalls = 0 + controller.retryNow() + assertTrue("calls: $fitsCalls", fitsCalls <= 2) + } + + @Test + fun `a network check that throws reads as offline at a foreground and does not throw`() { + waitingOffline() + network(RequiredNetwork.CONNECTED) + fitsThrows = true + controller.retryNow() + assertEquals(emptyList(), uploadRows()) + assertEquals(1, doorbellRows().size) + } + + // MARK: - an entry whose rule changes while it waits + + @Test + fun `a paused entry is not started by the doorbell, and resume starts it`() { + waitingOffline() + controller.pause() + network(RequiredNetwork.CONNECTED) + ringDoorbells() + assertEquals(emptyList(), uploadRows()) + assertEquals(emptyList(), doorbellRows()) + controller.resume() + assertEquals(1, startable().size) + } + + @Test + fun `a parked entry is not started by the doorbell, and new headers start it`() { + waitingOffline() + store.save(store.load("e1")!!.copy(state = EntryState.AWAITING_AUTH, parkedGeneration = 0)) + network(RequiredNetwork.CONNECTED) + ringDoorbells() + assertEquals(emptyList(), uploadRows()) + controller.updateHeaders(mapOf("Authorization" to "Bearer new")) + assertEquals(1, startable().size) + } + + @Test + fun `an entry that gets a long backoff wait is started by its wake, which has no network rule`() { + waitingOffline() + val at = work.now + 60 * minute + store.save(store.load("e1")!!.copy(nextAttemptAt = at, waitCause = WaitCause.RESPONSE, failedAt = work.now)) + controller.sweep() + val wake = uploadRows().single() + assertEquals("e1#wake", wake.name) + network(RequiredNetwork.CONNECTED) + ringDoorbells() + assertEquals(listOf(wake), uploadRows()) + // The wake starts at its time whatever the network; offline its run hands over again. + work.now = at + network(RequiredNetwork.NONE) + assertEquals(listOf(wake), startable()) + } + + @Test + fun `setWifiOnly off and on moves a waiting entry between the doorbells`() { + controller.setWifiOnly(true) + network(RequiredNetwork.NONE) + handedOver() + assertEquals(1, doorbellRows(RequiredNetwork.WIFI).size) + // Off while offline: it now waits for any network. + controller.setWifiOnly(false) + assertEquals(1, doorbellRows(RequiredNetwork.CONNECTED).size) + assertEquals(emptyList(), uploadRows()) + // On again: the CONNECTED doorbell rings on cellular and hands it back to the Wi-Fi doorbell. + controller.setWifiOnly(true) + network(RequiredNetwork.CONNECTED) + ringDoorbells() + assertEquals(emptyList(), uploadRows()) + assertEquals(emptyList(), doorbellRows(RequiredNetwork.CONNECTED)) + assertEquals(1, doorbellRows(RequiredNetwork.WIFI).size) + // Off on cellular: it starts now; the Wi-Fi doorbell finds nothing and ends. + controller.setWifiOnly(false) + assertEquals(1, startable().size) + network(RequiredNetwork.WIFI) + ringDoorbells() + assertEquals(emptyList(), doorbellRows()) + assertEquals(1, uploadRows().size) + } + + @Test + fun `setWifiOnly(false) starts an entry whose short wait from a hand-over is long over`() { + // Probe P1: a chunked hand-over kept a sibling part's short wait. + controller.setWifiOnly(true) + network(RequiredNetwork.CONNECTED) + handedOver( + stored = entry(state = EntryState.RUNNING, nextAttemptAt = work.now + 20_000) + .copy(waitCause = WaitCause.RESPONSE, failedAt = work.now, backoffStreak = 1), + ) + work.now += 10 * minute + controller.setWifiOnly(false) + assertEquals(1, startable().size) + } + + // MARK: - what the old constrained rows did, and no longer do + + @Test + fun `a network loss stops no upload row, and a due backoff wake starts with no network rule`() { + // Probes P3 and P4: an upload row with a rule was stopped by WorkManager and held back. + store.save(entry(state = EntryState.RUNNING)) + val main = work.add("e1", state = RUNNING, runAttemptCount = 1) + assertEquals(emptyList(), network(RequiredNetwork.NONE)) + assertEquals(RUNNING, main.state) + work.rows.clear() + store.save(entry(state = EntryState.QUEUED, nextAttemptAt = work.now + 60 * minute).copy(waitCause = WaitCause.RESPONSE, failedAt = work.now)) + controller.sweep() + work.now += 60 * minute + network(RequiredNetwork.CONNECTED) + assertEquals(listOf("e1#wake"), startable().map { it.name }) + } + + @Test + fun `the source sets a WorkManager constraint only on the doorbell`() { + val source = File("src/main/java/ai/openspace/backgroundupload/Scheduler.kt").readText() + assertEquals(1, Regex("""\.setConstraints\(""").findAll(source).count()) + val doorbell = source.substringAfter("override fun enqueueDoorbell(").substringBefore("\n }\n") + assertTrue(doorbell.contains(".setConstraints(")) + assertFalse(source.substringAfter("private fun request(").contains(".setConstraints(")) + } + + @Test + fun `a doorbell that waits is enough, and a doorbell needs a network`() { + scheduler.ensureDoorbell(RequiredNetwork.CONNECTED) + scheduler.ensureDoorbell(RequiredNetwork.CONNECTED) + assertEquals(1, doorbellRows().size) + assertThrows(IllegalArgumentException::class.java) { scheduler.ensureDoorbell(RequiredNetwork.NONE) } + } + + @Test + fun `a doorbell row has no entry id, so the summary and the snapshot do not count it as an entry's row`() { + scheduler.ensureDoorbell(RequiredNetwork.WIFI) + val summary = scheduler.rowSummary() + assertEquals(emptySet(), summary.startable) + assertEquals(emptySet(), summary.withWake) + assertEquals(listOf(RequiredNetwork.WIFI), scheduler.libraryRuns().map { it.doorbell }) + assertEquals(listOf(null), scheduler.libraryRuns().map { it.entryId }) + } +} diff --git a/android/src/test/java/ai/openspace/backgroundupload/QueueControllerTest.kt b/android/src/test/java/ai/openspace/backgroundupload/QueueControllerTest.kt index 7747c641..4d43b07b 100644 --- a/android/src/test/java/ai/openspace/backgroundupload/QueueControllerTest.kt +++ b/android/src/test/java/ai/openspace/backgroundupload/QueueControllerTest.kt @@ -824,10 +824,17 @@ class QueueControllerTest { assertTrue(QueueSettingsStore(file).load().wifiOnly) } + /** The controller with a network check that answers [fits], on an API level with doorbells or not. */ + private fun withNetwork(fits: Boolean, doorbells: Boolean = true) = + QueueController(store, journal, settings, events, scheduler, { it in running }, { now }, { fits }, doorbells) + @Test - fun `setWifiOnly(false) runs the queued entries that follow the setting`() { - controller.setWifiOnly(true) + fun `setWifiOnly(false) starts the queued entries that follow the setting when the network fits`() { + val c = withNetwork(fits = true) + c.setWifiOnly(true) store.save(entry(id = "follow", state = EntryState.QUEUED)) + // A short wait kept at a hand-over, long over now (probe P1). + store.save(entry(id = "stale", state = EntryState.QUEUED, nextAttemptAt = now - 60_000)) store.save(entry(id = "own", state = EntryState.QUEUED, descriptor = desc(dataJson = """{"a":1}""", wifiOnly = true))) store.save(entry(id = "cell", state = EntryState.QUEUED, descriptor = desc(dataJson = """{"a":1}""", wifiOnly = false))) store.save(entry(id = "waits", state = EntryState.QUEUED, nextAttemptAt = now + 3_600_000).copy(waitCause = WaitCause.RESPONSE, failedAt = now)) @@ -835,23 +842,79 @@ class QueueControllerTest { store.save(entry(id = "paused", state = EntryState.PAUSED)) store.save(entry(id = "running", state = EntryState.RUNNING)) running += "busy" - controller.setWifiOnly(false) - assertEquals(listOf("follow"), scheduler.scheduled) - // Its Wi-Fi wake no longer fits the setting. - assertEquals(listOf("follow"), scheduler.cancelledWakes) - assertEquals(listOf(false), scheduler.cancelWakeKeeps) + c.setWifiOnly(false) + assertEquals(setOf("follow", "stale"), scheduler.scheduled.toSet()) + assertEquals(emptyList(), scheduler.doorbells) assertEquals(emptyList>(), scheduler.wakes) } + @Test + fun `setWifiOnly(false) offline moves the entries that follow it to the network doorbell`() { + val c = withNetwork(fits = false) + c.setWifiOnly(true) + store.save(entry(id = "follow", state = EntryState.QUEUED)) + c.setWifiOnly(false) + assertEquals(emptyList(), scheduler.scheduled) + assertEquals(listOf(RequiredNetwork.CONNECTED), scheduler.doorbells) + } + @Test fun `setWifiOnly with the same value schedules nothing, and turning it on schedules nothing`() { + val c = withNetwork(fits = false) store.save(entry(id = "follow", state = EntryState.QUEUED)) - controller.setWifiOnly(false) - // A run that starts on cellular fails the check and hands over with the Wi-Fi constraint. - controller.setWifiOnly(true) - controller.setWifiOnly(true) + c.setWifiOnly(false) + c.setWifiOnly(true) + c.setWifiOnly(true) assertEquals(emptyList(), scheduler.scheduled) - assertEquals(emptyList(), scheduler.cancelledWakes) + assertEquals(emptyList(), scheduler.doorbells) + } + + @Test + fun `offline, a same-id enqueue of a queued entry rings for it, and a new entry starts a run`() { + controller.enqueue(parsed()) + scheduler.scheduled.clear() + val c = withNetwork(fits = false) + c.enqueue(parsed()) + assertEquals(emptyList(), scheduler.scheduled) + assertEquals(listOf(RequiredNetwork.CONNECTED), scheduler.doorbells) + // New work starts a run: it shows the notification and may take foreground mode. + c.enqueue(parsed(id = "new")) + assertEquals(listOf("new"), scheduler.scheduled) + } + + @Test + fun `on API 29 the repairs start a run offline and make no doorbell`() { + val c = withNetwork(fits = false, doorbells = false) + store.save(entry(id = "q", state = EntryState.QUEUED)) + c.sweep() + c.resume() + c.retryNow() + assertEquals(listOf("q", "q", "q"), scheduler.scheduled) + assertEquals(emptyList(), scheduler.doorbells) + } + + @Test + fun `a doorbell answer that can not read WorkManager or schedule an entry asks to ring again`() { + store.save(entry(id = "q", state = EntryState.QUEUED)) + val broken = object : WorkScheduler by scheduler { + override fun rowSummary(): RowSummary = throw IllegalStateException("no WorkManager") + } + assertFalse(QueueController(store, journal, settings, events, broken, { false }, { now }).answerDoorbell()) + val failing = object : WorkScheduler by scheduler { + override fun schedule(entry: QueueEntry) = throw IllegalStateException("enqueue failed") + } + assertFalse(QueueController(store, journal, settings, events, failing, { false }, { now }).answerDoorbell()) + assertTrue(controller.answerDoorbell()) + assertEquals(listOf("q"), scheduler.scheduled) + } + + @Test + fun `a doorbell answer with no entry waiting does nothing`() { + store.save(entry(id = "p", state = EntryState.PAUSED)) + store.save(entry(id = "w", state = EntryState.QUEUED, nextAttemptAt = now + 3_600_000)) + assertTrue(withNetwork(fits = false).answerDoorbell()) + assertEquals(0, scheduler.startableQueries) + assertEquals(emptyList(), scheduler.doorbells) } @Test @@ -1556,24 +1619,25 @@ class QueueControllerTest { } @Test - fun `sweep counts the entries by the wake they sleep on, and reports the wifiOnly setting`() { + fun `sweep counts the entries by what starts them, and reports the doorbells and the wifiOnly setting`() { val file = File(tmp.newFolder(), "diagnostics.json") Diag.log = DiagnosticLog(file) controller.setWifiOnly(true) store.save(entry(id = "b", state = EntryState.QUEUED, nextAttemptAt = now + 3_600_000).copy(waitCause = WaitCause.RESPONSE, failedAt = now)) - store.save(entry(id = "n", state = EntryState.QUEUED)) - store.save(entry(id = "w", state = EntryState.QUEUED)) + store.save(entry(id = "n", state = EntryState.QUEUED, descriptor = desc(dataJson = """{"a":1}""", wifiOnly = false))) + store.save(entry(id = "w1", state = EntryState.QUEUED)) + store.save(entry(id = "w2", state = EntryState.QUEUED)) store.save(entry(id = "m", state = EntryState.QUEUED)) store.save(entry(id = "p", state = EntryState.AWAITING_AUTH, expiresAt = now + 1_000)) - scheduler.runs += PendingRun("b", "ENQUEUED", 0, now + 3_600_000, delayed = true, network = RequiredNetwork.CONNECTED) - scheduler.runs += PendingRun("n", "ENQUEUED", 0, now - 5_000, delayed = false, network = RequiredNetwork.CONNECTED) - scheduler.runs += PendingRun("w", "ENQUEUED", 0, now - 5_000, delayed = false, network = RequiredNetwork.WIFI) - // A main-chain row and a parked entry's expiry wake are not waits for a network or a backoff. + scheduler.runs += PendingRun("b", "ENQUEUED", 0, now + 3_600_000, delayed = true) + // "m" has a main-chain row: it is not waiting for a doorbell. scheduler.runs += PendingRun("m", "ENQUEUED", 0, now, delayed = false) scheduler.runs += PendingRun("p", "ENQUEUED", 0, now + 1_000, delayed = true) + scheduler.runs += PendingRun(null, "ENQUEUED", 0, now, delayed = false, doorbell = RequiredNetwork.WIFI) controller.sweep() val data = DiagnosticLog(file).drain().single().data!! - assertEquals(mapOf("backoff" to 1.0, "network" to 1.0, "wifi" to 1.0), data["entriesByWake"]) + assertEquals(mapOf("backoff" to 1.0, "network" to 1.0, "wifi" to 2.0), data["entriesByWake"]) + assertEquals(mapOf("network" to false, "wifi" to true), data["doorbells"]) assertEquals(true, data["wifiOnly"]) } @@ -1589,36 +1653,7 @@ class QueueControllerTest { assertEquals(false, data["wifiOnly"]) assertEquals(true, data["runsUnavailable"]) assertFalse(data.containsKey("entriesByWake")) - } - - @Test - fun `the wakes the controller makes wait for the network a run needs`() { - controller.setWifiOnly(true) - // A backoff wait after a server response stays at the boot sweep: its wake waits for any network. - store.save(entry(id = "b", state = EntryState.QUEUED, nextAttemptAt = now + 3_600_000).copy(waitCause = WaitCause.RESPONSE, failedAt = now)) - // A parked entry's expiry wake settles it, and needs no network. - store.save(entry(id = "p", state = EntryState.AWAITING_AUTH, expiresAt = now + 1_000)) - controller.sweep() - assertEquals( - mapOf("b" to RequiredNetwork.CONNECTED, "p" to RequiredNetwork.NONE), - scheduler.wakes.map { it.first }.zip(scheduler.wakeNetworks).toMap(), - ) - // A return to the foreground makes a missing backoff wake again, with the same rule. - scheduler.wakes.clear() - scheduler.wakeNetworks.clear() - controller.retryNow() - assertEquals(listOf("b" to now + 3_600_000), scheduler.wakes) - assertEquals(listOf(RequiredNetwork.CONNECTED), scheduler.wakeNetworks) - } - - @Test - fun `the sweep, resume and a same-id enqueue keep the network wake of a queued entry with no wait`() { - controller.enqueue(parsed()) - scheduler.cancelWakeKeeps.clear() - controller.sweep() - controller.resume() - controller.enqueue(parsed()) - assertEquals(listOf(true, true, true), scheduler.cancelWakeKeeps) + assertFalse(data.containsKey("doorbells")) } @Test diff --git a/android/src/test/java/ai/openspace/backgroundupload/RequiredNetworkTest.kt b/android/src/test/java/ai/openspace/backgroundupload/RequiredNetworkTest.kt index 5fee84f6..ff9289f3 100644 --- a/android/src/test/java/ai/openspace/backgroundupload/RequiredNetworkTest.kt +++ b/android/src/test/java/ai/openspace/backgroundupload/RequiredNetworkTest.kt @@ -3,30 +3,33 @@ package ai.openspace.backgroundupload import org.junit.Assert.assertEquals import org.junit.Test -/** The network rule of a wake row, and what the in-worker network check accepts. */ +/** The doorbell an entry waits for, and what the in-worker network check accepts. */ class RequiredNetworkTest { private val now = 10_000L @Test - fun `the network a wake waits for, for each entry and setting`() { - // state, nextAttemptAt, the entry's own wifiOnly, the queue's wifiOnly → the rule. + fun `the doorbell an entry waits for, for each entry and setting`() { + // state, nextAttemptAt, the entry's own wifiOnly, the queue's wifiOnly → the doorbell. val table = listOf( Row(EntryState.QUEUED, null, null, false, RequiredNetwork.CONNECTED), Row(EntryState.QUEUED, null, null, true, RequiredNetwork.WIFI), Row(EntryState.QUEUED, null, true, false, RequiredNetwork.WIFI), Row(EntryState.QUEUED, null, false, true, RequiredNetwork.CONNECTED), - // A backoff wake waits for any network: the run checks Wi-Fi and hands over if it must. - Row(EntryState.QUEUED, now + 60_000, null, true, RequiredNetwork.CONNECTED), - Row(EntryState.QUEUED, now + 60_000, true, false, RequiredNetwork.CONNECTED), - // A parked entry's wake only settles its expiry. + // A short wait, or one already over, keeps the rule (a chunked sibling's backoff at a hand-over). + Row(EntryState.QUEUED, now + 20_000, null, true, RequiredNetwork.WIFI), + Row(EntryState.QUEUED, now - 60_000, null, false, RequiredNetwork.CONNECTED), + // A long wait: its wake starts it, not a doorbell. + Row(EntryState.QUEUED, now + 60_000, null, true, RequiredNetwork.NONE), + Row(EntryState.RUNNING, null, null, true, RequiredNetwork.NONE), + Row(EntryState.PAUSED, null, null, true, RequiredNetwork.NONE), Row(EntryState.AWAITING_AUTH, null, null, true, RequiredNetwork.NONE), - Row(EntryState.AWAITING_AUTH, null, true, false, RequiredNetwork.NONE), - Row(EntryState.RUNNING, null, null, true, RequiredNetwork.WIFI), + Row(EntryState.ERROR, null, null, false, RequiredNetwork.NONE), ) for (r in table) { val e = entry(state = r.state, nextAttemptAt = r.nextAttemptAt, descriptor = desc(dataJson = """{"a":1}""", wifiOnly = r.own)) - assertEquals(r.toString(), r.expected, requiredNetwork(e, QueueSettings(wifiOnly = r.queue))) + assertEquals(r.toString(), r.expected, requiredNetwork(e, QueueSettings(wifiOnly = r.queue), now)) } + assertEquals(RequiredNetwork.NONE, requiredNetwork(entry(legacy = true), QueueSettings(), now)) } private data class Row( @@ -46,8 +49,9 @@ class RequiredNetworkTest { Check(true, true, true, null, Connectivity.Ok), Check(true, true, false, null, Connectivity.NoWifi), Check(true, false, true, null, Connectivity.NoWifi), - // The process is bound to cellular: its sockets use cellular, so a Wi-Fi-only upload must not start. - Check(true, true, true, false, Connectivity.NoWifi), + // Bound to cellular: its sockets use cellular, so a Wi-Fi-only upload waits; the default network fits. + Check(true, true, true, false, Connectivity.BoundElsewhere), + Check(true, true, false, false, Connectivity.NoWifi), Check(true, true, true, true, Connectivity.Ok), // Not Wi-Fi only: the bind does not change the answer. Check(false, true, true, false, Connectivity.Ok), @@ -58,6 +62,13 @@ class RequiredNetworkTest { } } + @Test + fun `a network check that throws reads as no network`() { + assertEquals(Connectivity.NoInternet, safeConnectivity(false) { throw SecurityException("no permission") }) + assertEquals(Connectivity.NoWifi, safeConnectivity(true) { throw IllegalStateException("binder died") }) + assertEquals(Connectivity.Ok, safeConnectivity(true) { Connectivity.Ok }) + } + private data class Check( val wifiOnly: Boolean, val validated: Boolean, diff --git a/android/src/test/java/ai/openspace/backgroundupload/TestSupport.kt b/android/src/test/java/ai/openspace/backgroundupload/TestSupport.kt index ebc2a9b8..1f23f2ae 100644 --- a/android/src/test/java/ai/openspace/backgroundupload/TestSupport.kt +++ b/android/src/test/java/ai/openspace/backgroundupload/TestSupport.kt @@ -116,22 +116,20 @@ internal class FakeScheduler : WorkScheduler { val wakes = mutableListOf>() /** The replace argument of every scheduleWake, in order. REPLACE makes a fresh row. */ val wakeReplaces = mutableListOf() - /** The network argument of every scheduleWake, in order. */ - val wakeNetworks = mutableListOf() /** How many of the next scheduleWake calls throw, as a failed WorkManager enqueue. */ var wakeFailures = 0 val cancelled = mutableListOf() val cancelledWakes = mutableListOf() - /** The keepNetworkWake argument of every cancelWake, in order. */ - val cancelWakeKeeps = mutableListOf() + /** Every ensureDoorbell call, in order. */ + val doorbells = mutableListOf() + /** How many of the next ensureDoorbell calls throw, as a failed insert. */ + var doorbellFailures = 0 /** What libraryRuns() returns. */ val runs = mutableListOf() /** What rowSummary() returns as startable. */ val startable = mutableSetOf() /** What rowSummary() returns as withWake. */ val withWake = mutableSetOf() - /** What rowSummary() returns as networkWait. */ - val networkWait = mutableSetOf() /** How many rowSummary() calls were made. */ var startableQueries = 0 @@ -139,30 +137,36 @@ internal class FakeScheduler : WorkScheduler { scheduled += entry.id } - override fun scheduleWake(entry: QueueEntry, at: Long, replace: Boolean, network: RequiredNetwork) { + override fun scheduleWake(entry: QueueEntry, at: Long, replace: Boolean) { if (wakeFailures > 0) { wakeFailures-- throw IllegalStateException("enqueue failed") } wakes += entry.id to at wakeReplaces += replace - wakeNetworks += network } override fun cancel(id: String) { cancelled += id } - override fun cancelWake(id: String, keepNetworkWake: Boolean) { + override fun cancelWake(id: String) { cancelledWakes += id - cancelWakeKeeps += keepNetworkWake + } + + override fun ensureDoorbell(network: RequiredNetwork) { + if (doorbellFailures > 0) { + doorbellFailures-- + throw IllegalStateException("insert failed") + } + doorbells += network } override fun libraryRuns(): List = runs.toList() override fun rowSummary(): RowSummary { startableQueries++ - return RowSummary(startable.toSet(), withWake.toSet(), networkWait.toSet()) + return RowSummary(startable.toSet(), withWake.toSet()) } } @@ -181,31 +185,59 @@ internal class FakeScheduler : WorkScheduler { * - When a row finishes, the rows blocked behind it become ENQUEUED with a * new lastEnqueueTime. * - A row with a network constraint is ENQUEUED like any other, and - * nextScheduleTimeMillis does not look at the constraint: it is - * lastEnqueueTime + initialDelay (WorkSpec.WorkInfoPojo.calculateNextRunTimeMillis - * passes no constraint to calculateNextRunTime). WorkManager starts the - * row only once that time has come and its network is up. + * nextScheduleTimeMillis does not look at the constraint + * (WorkSpec.WorkInfoPojo.calculateNextRunTimeMillis). WorkManager starts + * it only once that time has come and its network is up, and stops it + * while it runs when the network loses the rule ([setNetwork]; a stop as + * above). + * - With the app process dead, only JobScheduler starts rows. WorkManager + * hands it at most [JOB_SCHEDULER_LIMIT] rows, the oldest ENQUEUED first + * (Configuration.maxSchedulerLimit, default 20), whatever their constraint + * ([jobSchedulerRows]). + * + * Not modelled: JobScheduler's quotas, standby buckets and Doze; the 200-row + * limit of the in-process scheduler; WorkManager's 1 s "not met" timeout and + * its network callbacks; rows that stay with JobScheduler once handed over; + * and a process death inside one WorkManager call. */ internal class FakeWorkManager(var now: Long = 10_000L) : WorkRows { + companion object { + const val JOB_SCHEDULER_LIMIT = 20 + } + class Row( val serial: Int, val name: String, - val entryId: String, + val entryId: String?, var state: WorkInfo.State, var runAttemptCount: Int, var lastEnqueueTime: Long, val initialDelay: Long, + /** The row's network rule. Only a doorbell has one. */ val network: RequiredNetwork = RequiredNetwork.NONE, + val doorbell: RequiredNetwork? = null, ) val rows = mutableListOf() + private var serial = 0 /** The best network up now: NONE offline, CONNECTED a validated network that is not Wi-Fi, WIFI a validated Wi-Fi network. */ var network = RequiredNetwork.WIFI + private set - /** Whether [r]'s network constraint holds now. */ + /** Whether [r]'s network rule holds now. */ fun constraintMet(r: Row) = r.network.ordinal <= network.ordinal - private var serial = 0 + + /** The network changes. A running row whose rule no longer holds is stopped, as WorkManager does. */ + fun setNetwork(n: RequiredNetwork): List { + network = n + val stopped = rows.filter { it.state == WorkInfo.State.RUNNING && !constraintMet(it) } + stopped.forEach { stop(it) } + return stopped + } + + /** How many of the next enqueueDoorbell calls fail, as an insert that fails after the call. */ + var doorbellInsertFailures = 0 fun nextScheduleAt(r: Row): Long = when { r.state != WorkInfo.State.ENQUEUED -> Long.MAX_VALUE @@ -223,8 +255,7 @@ internal class FakeWorkManager(var now: Long = 10_000L) : WorkRows { runAttemptCount: Int = 0, lastEnqueueTime: Long = now, initialDelay: Long = 0, - network: RequiredNetwork = RequiredNetwork.NONE, - ): Row = Row(serial++, name, name.substringBefore("#"), state, runAttemptCount, lastEnqueueTime, initialDelay, network) + ): Row = Row(serial++, name, name.substringBefore("#"), state, runAttemptCount, lastEnqueueTime, initialDelay) .also { rows += it } fun open(name: String) = rows.filter { it.name == name && !it.state.isFinished } @@ -233,6 +264,13 @@ internal class FakeWorkManager(var now: Long = 10_000L) : WorkRows { fun startableNow(): List = rows.filter { it.state == WorkInfo.State.ENQUEUED && nextScheduleAt(it) <= now && constraintMet(it) } + /** The rows JobScheduler holds while the app process is dead: the oldest ENQUEUED ones, at most [JOB_SCHEDULER_LIMIT]. */ + fun jobSchedulerRows(): List = + rows.filter { it.state == WorkInfo.State.ENQUEUED }.sortedBy { it.serial }.take(JOB_SCHEDULER_LIMIT) + + /** The rows that start now with the app process dead. */ + fun startableWhileDead(): List = jobSchedulerRows().filter { it in startableNow() } + override fun unfinished(name: String): List { nameQueries++ return open(name).map(::view) @@ -249,7 +287,19 @@ internal class FakeWorkManager(var now: Long = 10_000L) : WorkRows { return rows.filter { it.state in states }.map(::view) } - override fun enqueue(name: String, policy: ExistingWorkPolicy, entry: QueueEntry, delayMs: Long, network: RequiredNetwork) { + override fun enqueue(name: String, policy: ExistingWorkPolicy, entry: QueueEntry, delayMs: Long) { + add(name, policy, entry.id, delayMs, RequiredNetwork.NONE, null) + } + + override fun enqueueDoorbell(name: String, policy: ExistingWorkPolicy, network: RequiredNetwork) { + if (doorbellInsertFailures > 0) { + doorbellInsertFailures-- + throw java.util.concurrent.ExecutionException(IllegalStateException("insert failed")) + } + add(name, policy, null, 0L, network, network) + } + + private fun add(name: String, policy: ExistingWorkPolicy, entryId: String?, delayMs: Long, network: RequiredNetwork, doorbell: RequiredNetwork?) { val open = open(name) val state = when (policy) { ExistingWorkPolicy.KEEP -> if (open.isNotEmpty()) return else WorkInfo.State.ENQUEUED @@ -262,7 +312,7 @@ internal class FakeWorkManager(var now: Long = 10_000L) : WorkRows { if (open.isNotEmpty()) WorkInfo.State.BLOCKED else WorkInfo.State.ENQUEUED else -> error("unused policy $policy") } - rows += Row(serial++, name, entry.id, state, 0, now, delayMs, network) + rows += Row(serial++, name, entryId, state, 0, now, delayMs, network, doorbell) } override fun cancel(name: String) { @@ -280,5 +330,14 @@ internal class FakeWorkManager(var now: Long = 10_000L) : WorkRows { row.state = WorkInfo.State.ENQUEUED } - private fun view(r: Row) = WorkRow(r.entryId, r.state, r.runAttemptCount, nextScheduleAt(r), r.initialDelay > 0, r.network) + /** A run ends: the row succeeds, and a row blocked behind it becomes ENQUEUED. */ + fun finish(row: Row) { + row.state = WorkInfo.State.SUCCEEDED + rows.firstOrNull { it.name == row.name && it.state == WorkInfo.State.BLOCKED }?.let { + it.state = WorkInfo.State.ENQUEUED + it.lastEnqueueTime = now + } + } + + private fun view(r: Row) = WorkRow(r.entryId, r.state, r.runAttemptCount, nextScheduleAt(r), r.initialDelay > 0, r.doorbell) } From 3660a972a5d4bb9a1fbc55ed87f1cabea1091d92 Mon Sep 17 00:00:00 2001 From: Dylan Murphy Date: Tue, 17 Nov 2026 20:26:08 -0500 Subject: [PATCH 04/12] Docs: Android uploads with no network wait on a doorbell CHANGELOG (Unreleased) and README describe the doorbell design, the repair rules, the 30-minute foreground cap on API 35 and later with its known limit, and the boot record's doorbells field. BootDoorbells type and type tests. Co-Authored-By: Claude Fable 5.2 --- CHANGELOG.md | 99 ++++++++++++++++++++----------------- README.md | 40 ++++++++------- src/__typetests__/define.ts | 21 +++++++- src/types.ts | 18 +++++-- 4 files changed, 111 insertions(+), 67 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 49ddcd1b..648d0be1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -110,54 +110,65 @@ for the next, longer wait. **Behavior change:** on Android 11 and later (API 30), an upload that finds no usable network no longer polls inside a running job until the network -returns. A run with no foreground mode checks every 10 s for at most -60 s. Then the entry goes back to `queued` and sleeps on a WorkManager -wake row that waits for a validated network, or a validated Wi-Fi network -when Wi-Fi only applies to it. Android starts that row when the network is -up. This is not a failure: `attempts`, the backoff step and the last -failure's cause stay as they were, and the row shows no `nextAttemptAt`. A -run that holds foreground mode polls for at most 30 minutes, then hands -over the same way, because on Android 15 and later each minute of it uses -the `dataSync` budget of 6 hours in 24. A run hands over only while none -of its requests is on its way: a chunked part that waits for the network -waits for a sibling part to finish sending, and once it hands over, no -part starts a request. Before, the job used the app's job quota while it -waited, Android stopped it at its time limit, and in the `rare` standby -bucket it could wait without a network for the whole run. Android 10 (API -29) keeps the poll, and its wake rows have no network rule, as before: -there WorkManager registers one network callback per constrained row. - -Android: backoff wakes (API 30 and later) now also wait for a validated -network, so a wake does not start a run offline. A wake after a time-limit -stop waits for the network the entry needs. A parked entry's expiry wake -has no network rule. Main-chain rows still have none, and every run still -checks the network itself before each attempt. One effect: an entry whose -`expiresAt` passes while it sleeps on such a wake settles `expired` when -the network returns or the app starts, not at that time. - -Android: `setWifiOnly(false)` now runs at once each queued entry that -follows the setting and has no wait and no worker, and cancels its -Wi-Fi wake. The same value schedules nothing, and `setWifiOnly(true)` -schedules nothing: a run that starts on another network hands over with -the Wi-Fi rule. - -Android: an entry that sleeps on a network wake keeps it through the boot -sweep, `resume()` and a same-id `mutate()`. The boot sweep and `resume()` -still add a main row too, so a start never depends on WorkManager's -constraint tracking alone; offline, that row waits up to 60 s and hands -over again. A return to the foreground adds a main row only when the -network fits the entry now, so it does not start a run offline. +returns. A run with no foreground mode checks every 10 s for at most 60 s. +Then the entry goes back to `queued` with no WorkManager row of its own, +and waits on a doorbell: one row for the whole queue that waits for a +validated network, and a second one that waits for a validated Wi-Fi +network, for the entries that Wi-Fi only applies to. When Android starts a +doorbell, it sends nothing: it starts each waiting entry whose network +fits on an ordinary row, and keeps a doorbell for the others. No row that +runs an upload has a network rule, so WorkManager never stops an upload +because the network rule is lost. However many entries wait, at most two +rows sleep for them. The hand-over is not a failure: `attempts`, the +backoff step and the last failure's cause stay as they were, and it sets +no `nextAttemptAt`. A run hands over only while none of its requests is on +its way: a chunked part that waits for the network waits for a sibling +part to finish sending, and once it hands over, no part starts a request. +Before, the job used the app's job quota while it waited (in the `rare` +standby bucket, 10 minutes in 24 hours) and Android stopped it at its time +limit. Android 10 (API 29) keeps the poll and has no doorbell. + +On Android 15 and later (API 35), a run that holds foreground mode polls +for at most 30 minutes, then hands over the same way: each minute of it +uses the `dataSync` budget of 6 hours in 24. Below API 35 it polls as +before. Known limit: on API 31 and later a run that a doorbell starts in +the background cannot take foreground mode, so a single body (`data`, +`form`, `file`) that needs more than the 10-minute job limit cannot finish +until the app is opened. Use `parts` for large bodies. + +**Behavior change:** on Android 11 and later, the boot sweep, `resume()`, +a return to the foreground, a same-id `mutate()` of a queued entry and +`setWifiOnly()` start a queued entry with no backoff wait only when the +network fits it now. Otherwise they add no row and make sure its doorbell +exists. So an app start with no network starts no run. A new entry, and an +entry that `updateHeaders()` takes out of `awaiting-auth`, start a run as +before. Backoff wakes and a parked entry's expiry wake have no network +rule, as before. An entry whose `expiresAt` passes while it waits for a +doorbell settles `expired` when its next run starts: when the network +returns, or at the next start, resume or foreground with a network. + +Android: `setWifiOnly(false)` now moves the queued entries that follow the +setting from the Wi-Fi rule to any network: each one with no long backoff +wait and no worker starts at once when a network fits, else it waits on +the network doorbell. The same value, or `setWifiOnly(true)`, schedules +nothing: a run that starts on another network hands over to the Wi-Fi +doorbell. A run that hands over uses the setting at that moment. Android: a Wi-Fi-only upload now also waits while the app binds its -process to a network that is not Wi-Fi (`bindProcessToNetwork`). Its -sockets would use that network. The network check reads the system -default network, which a bind does not change. +process to a network that is not Wi-Fi (`bindProcessToNetwork`), because +its sockets would use that network. The network check reads the system +default network, which a bind does not change. Such a run keeps waiting +and does not hand over, because the Wi-Fi doorbell would start it again at +once. A network check that throws now reads as no network, so the upload +waits. Before, the error settled the upload as failed. Android: the boot record (`takeDiagnostics()`, message `boot sweep`) now -has `wifiOnly`, the queue's setting, and `entriesByWake` -(`{ backoff, network, wifi }`): the queued entries by the wake row they -sleep on. `entriesByWake` is left out when WorkManager could not be read. -The `Diagnostic` type has both fields. +has `wifiOnly`, the queue's setting; `entriesByWake` +(`{ backoff, network, wifi }`), the queued entries a backoff wake, the +network doorbell or the Wi-Fi doorbell starts; and `doorbells` +(`{ network, wifi }`), whether each doorbell row exists. `entriesByWake` +and `doorbells` are left out when WorkManager could not be read. The +`Diagnostic` type has the three fields. ## 11.3.0 diff --git a/README.md b/README.md index e780bcf5..eb43a977 100644 --- a/README.md +++ b/README.md @@ -221,16 +221,20 @@ at the next run, after a growing backoff. A body that needs more than accepted parts are kept across runs. iOS has no equal limit. **No network.** On API 30 and later a run that finds no usable network -checks every 10 s for at most 60 s, or 30 minutes when it holds foreground -mode. Then the entry goes back to `queued` and sleeps on a WorkManager row -that waits for a validated network (a validated Wi-Fi network when Wi-Fi -only applies). Android starts it when the network is up, within the -limits of the app's standby bucket and Doze. This is not a failed attempt and adds no backoff. Backoff wakes wait for a -network too. The boot sweep, `resume()`, and a return to the foreground -when the network fits, also start the entry from a plain row, so a start -does not depend on that rule alone. A Wi-Fi-only upload also waits while -the app binds its process to a network that is not Wi-Fi. API 29 keeps the -10 s check for as long as the run lasts. +checks every 10 s for at most 60 s; with foreground mode, on API 35 and +later, for at most 30 minutes. Then the entry goes back to `queued` with no +row of its own and waits on a doorbell: one WorkManager row for the whole +queue that waits for a validated network, and one that waits for a +validated Wi-Fi network. When Android starts a doorbell, it starts the +waiting entries whose network fits. The doorbell sends nothing, and no row +that sends has a network rule. The app's start, `resume()` and return to +the foreground also start a waiting entry when its network fits, and +otherwise make sure its doorbell exists. This is not a failed attempt and +adds no backoff. A Wi-Fi-only upload also waits while the app binds its +process to a network that is not Wi-Fi. API 29, and a foreground run below +API 35, keep the 10 s check for as long as the run lasts. A run that a +doorbell starts in the background cannot take foreground mode on API 31 +and later, so the headless time limit above applies to it. **Backoff and the app.** When the app starts, calls `resume()`, or comes to the foreground (every Activity resume, at most once every 30 s), the @@ -532,8 +536,9 @@ The queue's Wi-Fi setting. Persisted natively. Applies to queued and future entries whose descriptor does not set `wifiOnly`. An entry that sets it keeps its own value. Both are checked before each attempt, so a toggle moves the queued entries that follow the setting. On Android, turning it off also -starts at once the queued entries that follow it and have no backoff wait. -A call with the current value resolves without a write. +starts at once the queued entries that follow it and have no backoff wait, +when a network fits. A call with the current value resolves without a +write. It does not reject. The new value applies at once, also when the disk is full. See [Settings on a full disk](#settings-on-a-full-disk). @@ -788,11 +793,12 @@ describes every unfinished WorkManager run the library had before the start changed them: the count by state, how many are delayed wakes, how many of those are due more than 24 hours out, the highest run attempt count, and how many queued or running entries had no run at all. `entriesByWake` -(`{ backoff, network, wifi }`) counts the queued entries by the wake they -sleep on: the end of a backoff wait, any validated network, or Wi-Fi. It -is left out when WorkManager could not be read. `wifiOnly` is the queue's -Wi-Fi setting. Together they tell "waits for the network or for Wi-Fi by -choice" from "waits for a backoff". +(`{ backoff, network, wifi }`) counts the queued entries by what starts +them: a backoff wake, the network doorbell, or the Wi-Fi doorbell. +`doorbells` (`{ network, wifi }`) says whether each doorbell row exists. +Both are left out when WorkManager could not be read. `wifiOnly` is the +queue's Wi-Fi setting. Together they tell "waits for the network or for +Wi-Fi" from "waits for a backoff". On iOS the boot record has only `v9Leftovers` and `orphans`. The other iOS records are the warnings of the count and of the orphan cleanup, and a diff --git a/src/__typetests__/define.ts b/src/__typetests__/define.ts index 103ae858..cd79e1ab 100644 --- a/src/__typetests__/define.ts +++ b/src/__typetests__/define.ts @@ -2,6 +2,7 @@ // nothing here runs. Each @ts-expect-error line fails the build when the // error it expects goes away. import type { + BootDoorbells, BootWakeCounts, Diagnostic, Meta, @@ -356,6 +357,12 @@ if (wakes) { expectType(wakes.network); expectType(wakes.wifi); } +const bells = record.data?.doorbells; +assertEqual(true); +if (bells) { + expectType(bells.network); + expectType(bells.wifi); +} const wifiOnlySetting = record.data?.wifiOnly; assertEqual(true); expectType(wifiOnlySetting); @@ -363,7 +370,11 @@ const androidBoot: Diagnostic = { at: 1, level: 'info', message: 'boot sweep', - data: { wifiOnly: true, entriesByWake: { backoff: 1, network: 2, wifi: 0 } }, + data: { + wifiOnly: true, + entriesByWake: { backoff: 1, network: 2, wifi: 0 }, + doorbells: { network: true, wifi: false }, + }, }; expectType(androidBoot); const wakesMissingWifi: Diagnostic = { @@ -374,6 +385,14 @@ const wakesMissingWifi: Diagnostic = { data: { entriesByWake: { backoff: 1, network: 2 } }, }; expectType(wakesMissingWifi); +const doorbellsMissingWifi: Diagnostic = { + at: 1, + level: 'info', + message: 'boot sweep', + // @ts-expect-error both doorbells are reported + data: { doorbells: { network: true } }, +}; +expectType(doorbellsMissingWifi); const wifiOnlyText: Diagnostic = { at: 1, level: 'info', diff --git a/src/types.ts b/src/types.ts index a35c074f..9ce16866 100644 --- a/src/types.ts +++ b/src/types.ts @@ -481,10 +481,11 @@ export type QueueOrphans = { }; /** - * Android boot snapshot: the queued entries by the wake row they sleep on. - * Each entry counts once. `backoff`: a wake at the end of a backoff wait. - * `wifi`: a wake that waits for Wi-Fi. `network`: a wake that waits for any - * validated network. An entry with none of these is not counted. + * Android boot snapshot: the queued entries by what starts them. Each entry + * counts once. `backoff`: its wake at the end of a backoff wait. `network`: + * the doorbell that waits for any validated network. `wifi`: the doorbell + * that waits for Wi-Fi. An entry with a row of its own that is not a + * backoff wake is not counted. */ export type BootWakeCounts = { backoff: number; @@ -492,17 +493,24 @@ export type BootWakeCounts = { wifi: number; }; +/** Android boot snapshot: whether each doorbell row exists. */ +export type BootDoorbells = { + network: boolean; + wifi: boolean; +}; + /** * A diagnostic record's structured fields. The boot snapshot (message * `'boot sweep'`) always has `v9Leftovers` and `orphans`. On Android it also * has `wifiOnly` (the queue's setting) and, when WorkManager could be read, - * `entriesByWake`. Its other fields are untyped. + * `entriesByWake` and `doorbells`. Its other fields are untyped. */ export type DiagnosticData = Record & { v9Leftovers?: V9Leftovers; orphans?: QueueOrphans; wifiOnly?: boolean; entriesByWake?: BootWakeCounts; + doorbells?: BootDoorbells; }; /** From 828723183c7f9733e0a9e10d5b495cb9905cf11c Mon Sep 17 00:00:00 2001 From: Dylan Murphy Date: Tue, 17 Nov 2026 20:26:47 -0500 Subject: [PATCH 05/12] Test that the doorbell insert is awaited Co-Authored-By: Claude Fable 5.2 --- .../ai/openspace/backgroundupload/NetworkDoorbellTest.kt | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/android/src/test/java/ai/openspace/backgroundupload/NetworkDoorbellTest.kt b/android/src/test/java/ai/openspace/backgroundupload/NetworkDoorbellTest.kt index 4e909c87..a2e5207a 100644 --- a/android/src/test/java/ai/openspace/backgroundupload/NetworkDoorbellTest.kt +++ b/android/src/test/java/ai/openspace/backgroundupload/NetworkDoorbellTest.kt @@ -487,11 +487,14 @@ class NetworkDoorbellTest { } @Test - fun `the source sets a WorkManager constraint only on the doorbell`() { + fun `the source sets a WorkManager constraint only on the doorbell, and awaits its insert`() { val source = File("src/main/java/ai/openspace/backgroundupload/Scheduler.kt").readText() assertEquals(1, Regex("""\.setConstraints\(""").findAll(source).count()) val doorbell = source.substringAfter("override fun enqueueDoorbell(").substringBefore("\n }\n") assertTrue(doorbell.contains(".setConstraints(")) + // The insert is awaited, so one that fails later is seen. + assertTrue(doorbell.contains(".enqueue().result")) + assertTrue(doorbell.contains(".get(WorkManagerScheduler.DOORBELL_ENQUEUE_TIMEOUT_MS")) assertFalse(source.substringAfter("private fun request(").contains(".setConstraints(")) } From 53c0773b1655d5107f4bde6c68a25d650d6cb373 Mon Sep 17 00:00:00 2001 From: Dylan Murphy Date: Wed, 18 Nov 2026 11:03:32 -0500 Subject: [PATCH 06/12] Android doorbell: start a leaving entry, never fail, space the rings From the second review of the doorbell design: - A doorbell no longer skips an entry because a worker or a RUNNING row of it exists: that can be a worker that handed over and is leaving. It skips only an entry with an ENQUEUED row that starts now; for the others schedule() queues one row behind the running one. setWifiOnly() too. - The doorbell worker returns retry on any error, never failure, so a doorbell queued behind it is not failed with it. - A doorbell insert that fails is tried once per pass; the other entries of that rule get their main row, with one diagnostics record. WorkManager queries before an insert have a timeout. - retryNow treats a short or due wait as no wait, so an entry with a kept sibling wait follows the network rule and starts nothing offline. - awaitNetwork schedules the entry's wake when its kept wait reads long. - A doorbell whose network the library's check refuses queues the next one with a 30 s delay. - The doorbell waits until each main row is stored, and retries if not. - Each attempt checks the network again inside the transmit slot; on a failed check it leaves the slot with no attempt and waits again. - The 30-minute foreground cap needs a target of API 35 too. Co-Authored-By: Claude Fable 5.2 --- .../backgroundupload/BootSnapshot.kt | 7 +- .../backgroundupload/DoorbellWorker.kt | 49 ++- .../ai/openspace/backgroundupload/EntryRun.kt | 83 +++-- .../openspace/backgroundupload/EntryWorker.kt | 2 + .../backgroundupload/QueueController.kt | 54 +++- .../openspace/backgroundupload/Scheduler.kt | 67 +++- .../backgroundupload/TransferHost.kt | 3 + .../openspace/backgroundupload/WorkerOps.kt | 9 +- .../backgroundupload/EntryRunTest.kt | 61 +++- .../backgroundupload/NetworkDoorbellTest.kt | 291 ++++++++++++++++-- .../backgroundupload/QueueControllerTest.kt | 6 +- .../openspace/backgroundupload/TestSupport.kt | 54 +++- 12 files changed, 583 insertions(+), 103 deletions(-) diff --git a/android/src/main/java/ai/openspace/backgroundupload/BootSnapshot.kt b/android/src/main/java/ai/openspace/backgroundupload/BootSnapshot.kt index ada2a7d7..339bb894 100644 --- a/android/src/main/java/ai/openspace/backgroundupload/BootSnapshot.kt +++ b/android/src/main/java/ai/openspace/backgroundupload/BootSnapshot.kt @@ -4,13 +4,16 @@ package ai.openspace.backgroundupload * What the boot sweep found, as one diagnostic record: the entries it * schedules, and every unfinished WorkManager run of the library before the * sweep changes them. Stale wakes show as delayed runs far in the future. - * A queued or running entry with no run at all shows in entriesWithoutRun. + * A queued or running entry with no run at all shows in entriesWithoutRun; + * that includes an entry that waits for a doorbell, so by itself it does + * not mean a stuck entry. * v9Leftovers counts the v9 work still on disk, and orphans the * directories with no entry file that are not v9: what the boot cleanup * left, and what it deleted. Zeros included. wifiOnly is the queue's * setting. entriesByWake counts the queued entries by what starts them: * their backoff wake, or the network or Wi-Fi doorbell; doorbells says - * which doorbell rows exist. + * which doorbell rows exist. Both describe the rows and entries before the + * sweep changed them. */ internal object BootSnapshot { const val MESSAGE = "boot sweep" diff --git a/android/src/main/java/ai/openspace/backgroundupload/DoorbellWorker.kt b/android/src/main/java/ai/openspace/backgroundupload/DoorbellWorker.kt index 5d538615..dcad4a44 100644 --- a/android/src/main/java/ai/openspace/backgroundupload/DoorbellWorker.kt +++ b/android/src/main/java/ai/openspace/backgroundupload/DoorbellWorker.kt @@ -14,24 +14,45 @@ import kotlinx.coroutines.withContext * the entries whose network does not fit yet. It also runs with no JS and * no module, so it builds its own controller over the shared stores. * - * Returns retry when a WorkManager call failed, so the row runs again. + * Returns retry when a WorkManager call failed or anything threw, so the + * row runs again ([DoorbellRing]); it never fails. */ class DoorbellWorker(private val context: Context, params: WorkerParameters) : CoroutineWorker(context, params) { override suspend fun doWork(): Result = withContext(Dispatchers.IO) { - // A headless run has no module, so it opens the log that its Diag calls write to. - DiagnosticLog.get(context) - val settings = QueueSettingsStore.get(context) - val controller = QueueController( - QueueStore.get(context), - EventJournal.get(context), - settings, - EventReporter, - WorkManagerScheduler(context), - networkFits = { wifiOnly -> networkFitsNow(context, wifiOnly) }, - ) - // Every queue change runs on this one thread; a blocking wait here is off the main thread. - val answered = UploaderModule.queueExecutor.submit { controller.answerDoorbell() }.get() + val rung = inputData.getString(WorkManagerScheduler.DOORBELL_NETWORK_KEY) + ?.let { name -> RequiredNetwork.values().firstOrNull { it.name == name } } + val answered = DoorbellRing.answered { + // A headless run has no module, so it opens the log that its Diag calls write to. + DiagnosticLog.get(context) + val controller = QueueController( + QueueStore.get(context), + EventJournal.get(context), + QueueSettingsStore.get(context), + EventReporter, + WorkManagerScheduler(context), + networkFits = { wifiOnly -> networkFitsNow(context, wifiOnly) }, + ) + // Every queue change runs on this one thread; a blocking wait here is off the main thread. + UploaderModule.queueExecutor.submit { controller.answerDoorbell(rung) }.get() + } if (answered) Result.success() else Result.retry() } } + +/** + * The doorbell's result, from its answer. Any error is a retry, never a + * failure: WorkManager fails a FAILED row's successors without a run, so a + * doorbell queued behind it would be lost. A cancellation (a stop) goes on. + */ +internal object DoorbellRing { + fun answered(answer: () -> Boolean): Boolean = + try { + answer() + } catch (stop: java.util.concurrent.CancellationException) { + throw stop + } catch (error: Throwable) { + Diag.error("doorbell: the answer failed; it rings again", error) + false + } +} diff --git a/android/src/main/java/ai/openspace/backgroundupload/EntryRun.kt b/android/src/main/java/ai/openspace/backgroundupload/EntryRun.kt index 812453d0..65abd594 100644 --- a/android/src/main/java/ai/openspace/backgroundupload/EntryRun.kt +++ b/android/src/main/java/ai/openspace/backgroundupload/EntryRun.kt @@ -123,16 +123,18 @@ internal class EntryRun( const val HEADLESS_NETWORK_WAIT_MS = 60_000L /** - * The same for a run that holds foreground mode, on Android 15 (API 35) - * and later only: there each minute of it uses the dataSync budget of 6 - * hours in 24. Below API 35 such a run polls for as long as it runs. + * The same for a run that holds foreground mode, in an app that targets + * API 35 and runs on Android 15 (API 35) or later: only there does each + * minute of it use the dataSync budget of 6 hours in 24 (Android's + * foreground-service timeout docs; the limit is gated on the target). + * Otherwise such a run polls for as long as it runs. */ const val FOREGROUND_NETWORK_WAIT_MS = 30 * 60_000L /** The first API level with a network doorbell. API 29 keeps the poll. */ const val DOORBELL_MIN_SDK = 30 - /** The first API level whose dataSync foreground service has a daily budget (Android 15). */ + /** The first API level, of the device and of the app's target, with the dataSync daily budget (Android 15). */ const val FOREGROUND_BUDGET_MIN_SDK = 35 /** The poll while a short backoff remainder runs out before the run starts. */ @@ -340,6 +342,16 @@ internal class EntryRun( } } + /** The network no longer fits the entry when its request got a transmit slot. */ + private class NetworkGoneException : Exception("the network no longer fits") + + /** The network check, once more, inside the transmit slot. */ + private fun networkStillFits(): Boolean { + val entry = ops.latest(entryId, generation) + val wifiOnly = ops.settings().wifiOnlyFor(entry) + return safeConnectivity(wifiOnly) { host.connectivity(wifiOnly) } == Connectivity.Ok + } + /** * Sleeps out a short backoff remainder. False when the wait is long (the * wake run comes back for it) or the entry is no longer queued. For a long @@ -371,10 +383,11 @@ internal class EntryRun( * * On API 30 and later the run does not poll without end. With no * foreground mode it waits [HEADLESS_NETWORK_WAIT_MS]; with foreground - * mode on API 35 and later [FOREGROUND_NETWORK_WAIT_MS]. Then it throws + * mode on API 35 and later, in an app that targets API 35, + * [FOREGROUND_NETWORK_WAIT_MS]. Then it throws * [AwaitNetworkException]: the entry goes back to queued and a doorbell - * starts it once the network is up. Below API 35 a run with foreground - * mode polls as before. It hands over only while no request of the run is + * starts it once the network is up. Otherwise a run with foreground mode + * polls as before. It hands over only while no request of the run is * sending ([SendGate]), and only when the default network itself fails * the rule: while the app binds the process to another network * ([Connectivity.BoundElsewhere]) the run keeps waiting, because the @@ -396,7 +409,7 @@ internal class EntryRun( val limit = when { sdk < DOORBELL_MIN_SDK || network == Connectivity.BoundElsewhere -> null !holdsForeground -> HEADLESS_NETWORK_WAIT_MS - sdk >= FOREGROUND_BUDGET_MIN_SDK -> FOREGROUND_NETWORK_WAIT_MS + sdk >= FOREGROUND_BUDGET_MIN_SDK && host.targetSdk() >= FOREGROUND_BUDGET_MIN_SDK -> FOREGROUND_NETWORK_WAIT_MS else -> null } if (limit != null && host.now() - since >= limit && sends.tryHandOver()) throw AwaitNetworkException() @@ -429,29 +442,41 @@ internal class EntryRun( val highPriority = ops.latest(entryId, generation).descriptor!!.highPriority var written: QueueEntry? = null var failure: Throwable? = null - val response = try { - host.withSlot(highPriority) { - // A sibling part that handed over meanwhile: no request starts. - sends.beginSend() - try { - val e = ops.recordAttempt(entryId, generation, requestId) - written = e - val d = e.descriptor!! - val part = partIndex?.let { d.parts!![it] } - host.send(TransferRequest(part?.url ?: d.url!!, d.method, headersFor(d, part, requestId), body(d, part)), onProgress) - } finally { - sends.endSend() + var response: UploadResponse? = null + var done = false + while (!done) { + try { + response = host.withSlot(highPriority) { + // The wait for a slot can be long: check the network again, so a + // Wi-Fi-only request does not go out over another network. + if (!networkStillFits()) throw NetworkGoneException() + // A sibling part that handed over meanwhile: no request starts. + sends.beginSend() + try { + val e = ops.recordAttempt(entryId, generation, requestId) + written = e + val d = e.descriptor!! + val part = partIndex?.let { d.parts!![it] } + host.send(TransferRequest(part?.url ?: d.url!!, d.method, headersFor(d, part, requestId), body(d, part)), onProgress) + } finally { + sends.endSend() + } } + done = true + } catch (gone: NetworkGoneException) { + // The slot is left with no attempt counted. Wait for the network + // again, with its own limits and hand-over. + waitForNetwork() + } catch (error: CancellationException) { + // A stop after the write-ahead cut a request that was on its way. + if (written != null) stopCutRequest = true + throw error + } catch (error: Throwable) { + // With no write-ahead, the store write failed: not a network attempt. + if (written == null) throw error + failure = error + done = true } - } catch (error: CancellationException) { - // A stop after the write-ahead cut a request that was on its way. - if (written != null) stopCutRequest = true - throw error - } catch (error: Throwable) { - // With no write-ahead, the store write failed: not a network attempt. - if (written == null) throw error - failure = error - null } val entry = written!! diff --git a/android/src/main/java/ai/openspace/backgroundupload/EntryWorker.kt b/android/src/main/java/ai/openspace/backgroundupload/EntryWorker.kt index e0e3cc95..a4a214f1 100644 --- a/android/src/main/java/ai/openspace/backgroundupload/EntryWorker.kt +++ b/android/src/main/java/ai/openspace/backgroundupload/EntryWorker.kt @@ -105,6 +105,8 @@ open class EntryWorker(protected val context: Context, params: WorkerParameters) override fun sdkInt() = Build.VERSION.SDK_INT + override fun targetSdk() = context.applicationInfo.targetSdkVersion + override fun progressStarted(id: String, total: Long, sent: Long) { UploadProgress.add(id, total) UploadProgress.set(id, sent) diff --git a/android/src/main/java/ai/openspace/backgroundupload/QueueController.kt b/android/src/main/java/ai/openspace/backgroundupload/QueueController.kt index e6b90d09..d90c6495 100644 --- a/android/src/main/java/ai/openspace/backgroundupload/QueueController.kt +++ b/android/src/main/java/ai/openspace/backgroundupload/QueueController.kt @@ -300,8 +300,12 @@ class QueueController( val probe = NetworkProbe() cleared.forEach { e -> runCatching { scheduleRun(e, probe = probe) }.onFailure(::failed) } val clearedIds = cleared.map { it.id }.toSet() - val idle = queued.filter { it.nextAttemptAt == null } - val waiting = queued.filter { it.nextAttemptAt != null && it.id !in clearedIds } + // A wait that is due, or shorter than a worker sleeps out, is no wait + // here: such an entry follows the network rule like any other (a + // chunked hand-over can keep a sibling part's short wait). + fun longWait(e: QueueEntry) = e.nextAttemptAt?.let { it - now > RetryClassifier.IN_WORKER_BACKOFF_MAX_MS } == true + val idle = queued.filter { !longWait(it) && it.id !in clearedIds } + val waiting = queued.filter { longWait(it) && it.id !in clearedIds } if (idle.isNotEmpty() || waiting.isNotEmpty()) { val rows = runCatching { scheduler.rowSummary() }.onFailure(::failed).getOrNull() idle.filter { rows == null || it.id !in rows.startable } @@ -327,17 +331,23 @@ class QueueController( * save, a WorkManager call or the query failed: the doorbell worker then * returns retry, so its row runs again. */ - fun answerDoorbell(): Boolean { + fun answerDoorbell(rung: RequiredNetwork? = null): Boolean { val now = clock() val s = settings.load() - val waiting = store.all().filter { requiredNetwork(it, s, now) != RequiredNetwork.NONE && !isWorkerRunning(it.id) } + val waiting = store.all().filter { requiredNetwork(it, s, now) != RequiredNetwork.NONE } if (waiting.isEmpty()) return true val rows = runCatching { scheduler.rowSummary() } .onFailure { Diag.error("doorbell: could not read WorkManager; it rings again", it) } .getOrNull() ?: return false - val probe = NetworkProbe() + // WorkManager saw [rung]'s network. When the library's check does not, + // the doorbell queued behind this one waits DOORBELL_REPEAT_MS. + val probe = NetworkProbe(repeatOf = rung, awaitRows = true) var answered = true - waiting.filter { it.id !in rows.startable }.forEach { e -> + // Only an ENQUEUED row that starts now covers an entry. A worker of the + // entry, or a RUNNING row, may be one that handed over and is on its way + // out; schedule() then queues one row behind it, and the per-id gate + // runs them one after the other. + waiting.filter { it.id !in rows.due }.forEach { e -> runCatching { scheduleRun(e, probe = probe) }.onFailure { answered = false Diag.error("doorbell: could not schedule '${e.id}'; it rings again", it) @@ -347,15 +357,22 @@ class QueueController( } /** - * The network check for one pass, at most once per Wi-Fi rule. A check - * that throws reads as no network. + * One pass: the network check at most once per Wi-Fi rule (a check that + * throws reads as no network), and each doorbell made sure at most once. + * A doorbell that could not be stored is not tried again in the pass: its + * entries get their main row. [repeatOf]: the doorbell that is answering; + * one made for its own rule waits DOORBELL_REPEAT_MS. [awaitRows]: wait + * until each main row is stored. */ - private inner class NetworkProbe { + private inner class NetworkProbe(val repeatOf: RequiredNetwork? = null, val awaitRows: Boolean = false) { private val answers = mutableMapOf() /** The doorbells already made sure in this pass. */ val ensured = mutableSetOf() + /** The doorbells that could not be stored in this pass. */ + val failed = mutableSetOf() + fun fits(wifiOnly: Boolean): Boolean = answers.getOrPut(wifiOnly) { runCatching { networkFits(wifiOnly) }.getOrDefault(false) } } @@ -471,15 +488,16 @@ class QueueController( val after = settings.update { it.copy(wifiOnly = enabled) }.wifiOnly if (!before || after) return // Off: the entries that follow the setting now wait for any network. - // Each one with no long wait and no worker starts when that fits, else - // it waits on the CONNECTED doorbell. A Wi-Fi doorbell left behind + // Each one with no long wait starts when that fits, else it waits on + // the CONNECTED doorbell. A worker of the entry may be one that handed + // over and is on its way out: schedule() queues one row behind it. A Wi-Fi doorbell left behind // finds nothing to start and ends. On needs no work: a run that starts // on another network fails the check and hands over to the Wi-Fi doorbell. val now = clock() val s = settings.load() val probe = NetworkProbe() store.all() - .filter { it.descriptor?.wifiOnly == null && requiredNetwork(it, s, now) != RequiredNetwork.NONE && !isWorkerRunning(it.id) } + .filter { it.descriptor?.wifiOnly == null && requiredNetwork(it, s, now) != RequiredNetwork.NONE } .forEach { e -> runCatching { scheduleRun(e, probe = probe) } .onFailure { Diag.error("setWifiOnly: could not schedule '${e.id}'", it) } @@ -731,7 +749,7 @@ class QueueController( // The main row first: these are two separate WorkManager calls, // and a process death between them must not leave the entry // with neither a wake nor a main row. - scheduler.schedule(entry) + if (probe?.awaitRows == true) scheduler.scheduleAndWait(entry) else scheduler.schedule(entry) cancelStaleWake(entry.id) } } @@ -754,9 +772,15 @@ class QueueController( val network = requiredNetwork(entry, settings.load(), now) if (network == RequiredNetwork.NONE || probe.fits(network == RequiredNetwork.WIFI)) return false if (network in probe.ensured) return true - return runCatching { scheduler.ensureDoorbell(network) } + if (network in probe.failed) return false + val delay = if (network == probe.repeatOf) WorkManagerScheduler.DOORBELL_REPEAT_MS else 0L + return runCatching { scheduler.ensureDoorbell(network, delay) } .onSuccess { probe.ensured += network } - .onFailure { Diag.error("could not store the ${network.name} doorbell for '${entry.id}'; it starts a run instead", it) } + .onFailure { + // One record per pass: the other entries of this rule skip the doorbell. + probe.failed += network + Diag.error("could not store the ${network.name} doorbell; its waiting entries start a run instead", it) + } .isSuccess } diff --git a/android/src/main/java/ai/openspace/backgroundupload/Scheduler.kt b/android/src/main/java/ai/openspace/backgroundupload/Scheduler.kt index a57e65e3..c2580cab 100644 --- a/android/src/main/java/ai/openspace/backgroundupload/Scheduler.kt +++ b/android/src/main/java/ai/openspace/backgroundupload/Scheduler.kt @@ -134,6 +134,12 @@ interface WorkScheduler { */ fun schedule(entry: QueueEntry) + /** + * [schedule], and waits until WorkManager has stored the row it adds. + * Throws when it did not. + */ + fun scheduleAndWait(entry: QueueEntry) = schedule(entry) + /** * A run at [at] under a second unique name. Used for long backoffs and for * the expiry of a parked entry. [replace] false keeps a wake that already @@ -152,9 +158,10 @@ interface WorkScheduler { * row for the whole queue, with that WorkManager network rule. When the * network is up, WorkManager runs it, and its worker ([DoorbellWorker]) * starts the entries that wait for a network. It runs no upload. Returns - * once WorkManager has stored the row; throws when it could not. + * once WorkManager has stored the row; throws when it could not. [delayMs] + * is the initial delay of a new row; a doorbell that is there is kept. */ - fun ensureDoorbell(network: RequiredNetwork) + fun ensureDoorbell(network: RequiredNetwork, delayMs: Long = 0L) /** Every unfinished WorkManager run of the library, main chains and wakes. For the boot snapshot. */ fun libraryRuns(): List @@ -166,9 +173,15 @@ interface WorkScheduler { /** * [startable]: the ids with a run in progress, or a row WorkManager may * start now. [withWake]: the ids with an unfinished delayed row. Only a wake - * has an initial delay; a main-chain row never does. + * has an initial delay; a main-chain row never does. [due]: the ids with an + * ENQUEUED row WorkManager may start now; a RUNNING row does not count, so + * a worker that is on its way out does not hide its entry. */ -data class RowSummary(val startable: Set, val withWake: Set) +data class RowSummary( + val startable: Set, + val withWake: Set, + val due: Set = emptySet(), +) /** * One unfinished WorkManager run. [entryId] comes from the run's id tag. @@ -212,11 +225,15 @@ interface WorkRows { fun enqueue(name: String, policy: ExistingWorkPolicy, entry: QueueEntry, delayMs: Long) + /** [enqueue], and waits until WorkManager has stored the row. Throws when it did not. */ + fun enqueueAndWait(name: String, policy: ExistingWorkPolicy, entry: QueueEntry, delayMs: Long) = + enqueue(name, policy, entry, delayMs) + /** * Enqueues the doorbell row [name] with [network] as its rule, and waits * until WorkManager has stored it. Throws when it did not. */ - fun enqueueDoorbell(name: String, policy: ExistingWorkPolicy, network: RequiredNetwork) + fun enqueueDoorbell(name: String, policy: ExistingWorkPolicy, network: RequiredNetwork, delayMs: Long = 0L) fun cancel(name: String) } @@ -277,6 +294,14 @@ open class RowScheduler( } } + override fun scheduleAndWait(entry: QueueEntry) { + when (mainChainStep(rows.unfinished(entry.id), clock())) { + MainChainStep.NONE -> Unit + MainChainStep.APPEND -> rows.enqueueAndWait(entry.id, ExistingWorkPolicy.APPEND_OR_REPLACE, entry, 0L) + MainChainStep.REPLACE -> rows.enqueueAndWait(entry.id, ExistingWorkPolicy.REPLACE, entry, 0L) + } + } + override fun scheduleWake(entry: QueueEntry, at: Long, replace: Boolean) { val delay = WorkManagerScheduler.initialDelayMs(at, clock()) rows.enqueue( @@ -301,14 +326,14 @@ open class RowScheduler( * for the process, and the enqueue is awaited, so two callers do not both * append. */ - override fun ensureDoorbell(network: RequiredNetwork) { + override fun ensureDoorbell(network: RequiredNetwork, delayMs: Long) { require(network != RequiredNetwork.NONE) { "a doorbell needs a network" } val name = WorkManagerScheduler.doorbellName(network) synchronized(DOORBELL_LOCK) { when (mainChainStep(rows.unfinished(name), clock())) { MainChainStep.NONE -> Unit - MainChainStep.APPEND -> rows.enqueueDoorbell(name, ExistingWorkPolicy.APPEND_OR_REPLACE, network) - MainChainStep.REPLACE -> rows.enqueueDoorbell(name, ExistingWorkPolicy.REPLACE, network) + MainChainStep.APPEND -> rows.enqueueDoorbell(name, ExistingWorkPolicy.APPEND_OR_REPLACE, network, delayMs) + MainChainStep.REPLACE -> rows.enqueueDoorbell(name, ExistingWorkPolicy.REPLACE, network, delayMs) } } } @@ -340,6 +365,7 @@ open class RowScheduler( return RowSummary( startable = open.filter { it.state == WorkInfo.State.RUNNING || it.startsBy(now) }.mapNotNull { it.entryId }.toSet(), withWake = open.filter { it.delayed }.mapNotNull { it.entryId }.toSet(), + due = open.filter { it.startsBy(now) }.mapNotNull { it.entryId }.toSet(), ) } } @@ -367,8 +393,15 @@ class WorkManagerScheduler private constructor(private val workManager: WorkMana /** A string literal, because WorkManager persists it across builds. */ const val DOORBELL_NETWORK_KEY = "network" - /** How long a doorbell enqueue may take before it counts as failed. */ - const val DOORBELL_ENQUEUE_TIMEOUT_MS = 10_000L + /** How long a WorkManager query or an awaited enqueue may take before it counts as failed. */ + const val WORK_CALL_TIMEOUT_MS = 10_000L + + /** + * The initial delay of the doorbell a doorbell queues behind itself when + * WorkManager saw its network but the library's check did not. So such + * a doorbell rings at most once in this time. + */ + const val DOORBELL_REPEAT_MS = 30_000L /** Milliseconds from [now] until [at]; never negative. */ fun initialDelayMs(at: Long?, now: Long): Long = if (at == null) 0L else (at - now).coerceAtLeast(0L) @@ -386,8 +419,10 @@ class WorkManagerScheduler private constructor(private val workManager: WorkMana /** [WorkRows] on WorkManager. */ private class WorkManagerRows(private val workManager: WorkManager) : WorkRows { + // A timeout, so a WorkManager that does not answer does not hold the queue executor. override fun unfinished(name: String): List = - workManager.getWorkInfosForUniqueWork(name).get().filter { !it.state.isFinished }.map(::row) + workManager.getWorkInfosForUniqueWork(name).get(WorkManagerScheduler.WORK_CALL_TIMEOUT_MS, TimeUnit.MILLISECONDS) + .filter { !it.state.isFinished }.map(::row) // The states are filtered in the query, so finished rows are never loaded. override fun library(states: List): List = @@ -399,6 +434,11 @@ private class WorkManagerRows(private val workManager: WorkManager) : WorkRows { workManager.beginUniqueWork(name, policy, request(entry, delayMs)).enqueue() } + override fun enqueueAndWait(name: String, policy: ExistingWorkPolicy, entry: QueueEntry, delayMs: Long) { + workManager.beginUniqueWork(name, policy, request(entry, delayMs)).enqueue().result + .get(WorkManagerScheduler.WORK_CALL_TIMEOUT_MS, TimeUnit.MILLISECONDS) + } + override fun cancel(name: String) { workManager.cancelUniqueWork(name) } @@ -406,15 +446,16 @@ private class WorkManagerRows(private val workManager: WorkManager) : WorkRows { // Waits for the insert: an enqueue that fails later is otherwise not seen. // The callers run on the queue executor or a worker thread, never on the // main thread. - override fun enqueueDoorbell(name: String, policy: ExistingWorkPolicy, network: RequiredNetwork) { + override fun enqueueDoorbell(name: String, policy: ExistingWorkPolicy, network: RequiredNetwork, delayMs: Long) { val request = OneTimeWorkRequestBuilder() .addTag(WorkManagerScheduler.WORK_TAG) .addTag(WorkManagerScheduler.DOORBELL_TAG_PREFIX + network.name) .setInputData(workDataOf(WorkManagerScheduler.DOORBELL_NETWORK_KEY to network.name)) + .setInitialDelay(delayMs, TimeUnit.MILLISECONDS) .setConstraints(constraints(network)) .build() workManager.beginUniqueWork(name, policy, request).enqueue().result - .get(WorkManagerScheduler.DOORBELL_ENQUEUE_TIMEOUT_MS, TimeUnit.MILLISECONDS) + .get(WorkManagerScheduler.WORK_CALL_TIMEOUT_MS, TimeUnit.MILLISECONDS) } /** diff --git a/android/src/main/java/ai/openspace/backgroundupload/TransferHost.kt b/android/src/main/java/ai/openspace/backgroundupload/TransferHost.kt index ade70880..569a8553 100644 --- a/android/src/main/java/ai/openspace/backgroundupload/TransferHost.kt +++ b/android/src/main/java/ai/openspace/backgroundupload/TransferHost.kt @@ -38,6 +38,9 @@ internal interface TransferHost { */ fun sdkInt(): Int + /** The app's targetSdkVersion. */ + fun targetSdk(): Int + fun progressStarted(id: String, total: Long, sent: Long) fun progress(id: String, sent: Long, total: Long) diff --git a/android/src/main/java/ai/openspace/backgroundupload/WorkerOps.kt b/android/src/main/java/ai/openspace/backgroundupload/WorkerOps.kt index 3da2e3af..1b7e85fb 100644 --- a/android/src/main/java/ai/openspace/backgroundupload/WorkerOps.kt +++ b/android/src/main/java/ai/openspace/backgroundupload/WorkerOps.kt @@ -443,9 +443,14 @@ class WorkerOps( if (!applied || next == null) return false events.state(next.toRow()) val network = requiredNetwork(next, settings(), now) - if (network == RequiredNetwork.NONE) return true try { - scheduler.ensureDoorbell(network) + if (network != RequiredNetwork.NONE) { + scheduler.ensureDoorbell(network) + } else { + // A kept wait that reads as long now (the clock moved): its wake + // starts the entry, as after a release. + scheduler.scheduleWake(next, next.nextAttemptAt ?: now, replace = true) + } } catch (error: Exception) { throw WakeNotScheduledException(id, error) } diff --git a/android/src/test/java/ai/openspace/backgroundupload/EntryRunTest.kt b/android/src/test/java/ai/openspace/backgroundupload/EntryRunTest.kt index 09884d2c..e462824e 100644 --- a/android/src/test/java/ai/openspace/backgroundupload/EntryRunTest.kt +++ b/android/src/test/java/ai/openspace/backgroundupload/EntryRunTest.kt @@ -86,6 +86,11 @@ internal class FakeHost(var clock: Long = 10_000L) : TransferHost { override fun sdkInt() = sdk + /** What targetSdk() answers. */ + var target = 35 + + override fun targetSdk() = target + override fun progressStarted(id: String, total: Long, sent: Long) { progress += "start:$total:$sent" } @@ -469,7 +474,8 @@ class EntryRunTest { controller.enqueue(parsed(id = "cell", descriptor = desc(dataJson = """{"a":1}""", wifiOnly = false))) respond(200) run("cell") - assertEquals(listOf(false), host.wifiChecks) // its own false beats the queue's true + // Before the transmit slot and once more in it: its own false beats the queue's true. + assertEquals(listOf(false, false), host.wifiChecks) host.wifiChecks.clear() controller.enqueue(parsed(id = "follow")) @@ -477,7 +483,7 @@ class EntryRunTest { host.onSleep = { controller.setWifiOnly(false) } // a toggle while it waits respond(200) run("follow") - assertEquals(listOf(true, false), host.wifiChecks) + assertEquals(listOf(true, false, false), host.wifiChecks) assertEquals(EntryState.COMPLETED, store.load("follow")!!.state) } @@ -998,6 +1004,53 @@ class EntryRunTest { assertEquals(EntryRun.FOREGROUND_NETWORK_WAIT_MS, host.sleeps.sum()) } + @Test + fun `a run that holds foreground mode on API 35 in an app that targets API 34 keeps the poll`() { + // The dataSync time limit applies to apps that target API 35. + controller.enqueue(parsed()) + host.foregroundGranted = true + host.sdk = 35 + host.target = 34 + host.network = ArrayDeque(List(200) { Connectivity.NoInternet }) + respond(200) + run() + assertEquals(EntryState.COMPLETED, store.load("e1")!!.state) + assertEquals(emptyList(), scheduler.doorbells) + } + + @Test + fun `the network is checked again inside the transmit slot, and a failed check sends nothing and counts nothing`() { + controller.setWifiOnly(true) + controller.enqueue(parsed()) + // Before the slot: Wi-Fi. In the slot: cellular only. Then Wi-Fi again. + host.network = ArrayDeque(listOf(Connectivity.Ok, Connectivity.NoWifi, Connectivity.NoWifi, Connectivity.Ok, Connectivity.Ok)) + var attemptsInSlot = mutableListOf() + host.beforeSlot = { attemptsInSlot += store.load("e1")!!.attempts } + respond(200) + run() + val e = store.load("e1")!! + assertEquals(EntryState.COMPLETED, e.state) + assertEquals(1, e.attempts) + assertEquals(1, host.requests.size) + assertEquals(5, host.wifiChecks.size) + assertEquals(listOf(0, 0), attemptsInSlot) // two slots taken, one attempt counted + assertEquals(listOf(EntryRun.CONNECTIVITY_POLL_MS), host.sleeps) + } + + @Test + fun `a chunked part checks the network again inside the transmit slot`() { + controller.setWifiOnly(true) + val src = File(tmp.newFolder(), "video.bin").apply { writeBytes(ByteArray(10) { it.toByte() }) } + controller.enqueue(parsed(descriptor = desc(url = null, method = "PUT", file = src.path, parts = listOf(part(0, 10))))) + host.network = ArrayDeque(listOf(Connectivity.Ok, Connectivity.NoWifi, Connectivity.Ok, Connectivity.Ok)) + run() + val e = store.load("e1")!! + assertEquals(EntryState.COMPLETED, e.state) + assertEquals(1, e.attempts) + assertEquals(1, host.requests.size) + assertEquals(4, host.wifiChecks.size) + } + @Test fun `below API 35 a run that holds foreground mode keeps the poll`() { controller.enqueue(parsed()) @@ -1645,8 +1698,8 @@ class EntryRunTest { parts = listOf(part(0, 10), part(10, 20))))) host.foregroundGranted = false host.yieldOnSleep = true - // Part 0 finds the network and sends; part 1 then finds none. - host.network = ArrayDeque(listOf(Connectivity.Ok) + List(30) { Connectivity.NoInternet }) + // Part 0 finds the network (before and in its slot) and sends; part 1 then finds none. + host.network = ArrayDeque(listOf(Connectivity.Ok, Connectivity.Ok) + List(30) { Connectivity.NoInternet }) val answer = CompletableDeferred() host.handler = { _, _ -> answer.await() } val start = host.clock diff --git a/android/src/test/java/ai/openspace/backgroundupload/NetworkDoorbellTest.kt b/android/src/test/java/ai/openspace/backgroundupload/NetworkDoorbellTest.kt index a2e5207a..0ac6f9af 100644 --- a/android/src/test/java/ai/openspace/backgroundupload/NetworkDoorbellTest.kt +++ b/android/src/test/java/ai/openspace/backgroundupload/NetworkDoorbellTest.kt @@ -87,24 +87,253 @@ class NetworkDoorbellTest { * then ends. */ private fun handedOver(id: String = "e1", stored: QueueEntry = entry(id = id, state = EntryState.RUNNING)) { + handingOver(id, stored).leave() + } + + /** + * A worker of [id] that has handed over and is still on its way out: the + * entry is saved queued and the doorbell is made sure, but the worker + * still holds the per-id gate and its row is RUNNING. + */ + private inner class Leaving(val id: String, val row: FakeWorkManager.Row) { + /** doWork's finally: the gate is released. The row stays RUNNING until WorkManager records the result. */ + fun releaseGate() { + running -= id + } + + fun leave() { + releaseGate() + work.finish(row) + } + } + + private fun handingOver(id: String = "e1", stored: QueueEntry = entry(id = id, state = EntryState.RUNNING)): Leaving { store.save(stored) val own = work.add(id, state = RUNNING, runAttemptCount = 1) running += id assertTrue(ops().awaitNetwork(id, stored.generation)) - running -= id - work.finish(own) + return Leaving(id, own) } - /** WorkManager starts each doorbell that may start now; its worker answers with its own controller. */ + /** + * WorkManager starts each doorbell that may start now; its worker answers + * with its own controller. Its result: success finishes the row, retry + * puts it back with WorkManager's backoff. + */ private fun ringDoorbells(): List { val rung = work.startableNow().filter { it.doorbell != null } for (row in rung) { work.start(row) - if (newController().answerDoorbell()) work.finish(row) else work.stop(row) + val c = newController() + if (DoorbellRing.answered { c.answerDoorbell(row.doorbell) }) work.finish(row) else work.retry(row) } return rung } + // MARK: - review fixes (second review) + + @Test + fun `a doorbell that answers while a hand-over still holds the gate starts the entry after the worker leaves`() { + // Review probe P1a. + network(RequiredNetwork.NONE) + handedOver("a") + val b = handingOver("b") + assertEquals(1, doorbellRows().size) + network(RequiredNetwork.CONNECTED) + assertEquals(1, ringDoorbells().size) + b.leave() + assertEquals(1, startable("a").size) + assertEquals(1, startable("b").size) + } + + @Test + fun `a doorbell that answers after the gate is released but before the row ends starts the entry`() { + // Review probe P1b. + network(RequiredNetwork.NONE) + handedOver("a") + val b = handingOver("b") + b.releaseGate() + network(RequiredNetwork.CONNECTED) + assertEquals(1, ringDoorbells().size) + work.finish(b.row) + assertEquals(1, startable("b").size) + } + + @Test + fun `the doorbell appended behind a running one starts an entry whose hand-over row has not ended`() { + // Review probe P1c. + network(RequiredNetwork.NONE) + handedOver("a") + network(RequiredNetwork.CONNECTED) + val first = doorbellRows().single() + work.start(first) + val b = handingOver("b") + assertEquals(2, doorbellRows().size) + assertTrue(newController().answerDoorbell(RequiredNetwork.CONNECTED)) + work.finish(first) + ringDoorbells() + b.leave() + assertEquals(1, startable("b").size) + assertEquals(1, uploadRows("b").size) + } + + @Test + fun `a doorbell that rings after the hand-over worker left starts the entry too`() { + // The other order of the three above. + network(RequiredNetwork.NONE) + handingOver("b").leave() + network(RequiredNetwork.CONNECTED) + ringDoorbells() + assertEquals(1, startable("b").size) + assertEquals(1, uploadRows("b").size) + } + + @Test + fun `setWifiOnly(false) starts an entry whose hand-over worker is still leaving`() { + controller.setWifiOnly(true) + network(RequiredNetwork.CONNECTED) + val b = handingOver("b") + controller.setWifiOnly(false) + b.leave() + assertEquals(1, startable("b").size) + } + + @Test + fun `a doorbell whose library check keeps refusing rings at most once every 30 s`() { + // Review probe P2: WorkManager says the rule is met; the library's check says no. + network(RequiredNetwork.NONE) + handedOver("a") + network(RequiredNetwork.CONNECTED) + fitsThrows = true + val rings = mutableListOf() + repeat(20) { + if (ringDoorbells().isNotEmpty()) rings += work.now + work.now += 1_000 + } + assertTrue("rings at $rings", rings.size >= 1) + assertTrue("rings at $rings", rings.zipWithNext().all { (x, y) -> y - x >= 30_000 }) + assertTrue("rows: ${work.rows.size}", work.rows.count { it.doorbell != null } <= rings.size + 1) + assertEquals(1, doorbellRows().size) + // The network check passes again: the next ring starts the entry. + fitsThrows = false + work.now += 30_000 + ringDoorbells() + assertEquals(1, startable("a").size) + } + + @Test + fun `a doorbell insert that fails is tried once per pass, and the other entries get their main row`() { + // Review probe P5. + val file = File(tmp.newFolder(), "diagnostics.json") + Diag.log = DiagnosticLog(file) + try { + network(RequiredNetwork.NONE) + for (n in 1..10) store.save(entry(id = "q$n")) + work.doorbellInsertFailures = 10 + controller.retryNow() + assertEquals(1, work.doorbellInserts) + assertEquals(10, work.rows.count { it.doorbell == null && !it.state.isFinished }) + assertEquals(1, DiagnosticLog(file).drain().count { it.level == DiagnosticLog.ERROR }) + } finally { + Diag.log = null + } + } + + @Test + fun `offline, a foreground starts no run for a waiting entry that kept a short wait after a response`() { + // Review probe P3. + network(RequiredNetwork.NONE) + handedOver( + "c", + entry(id = "c", state = EntryState.RUNNING, nextAttemptAt = work.now + 20_000) + .copy(waitCause = WaitCause.RESPONSE, failedAt = work.now, backoffStreak = 1), + ) + work.now += 10 * minute + controller.retryNow() + assertEquals(emptyList(), uploadRows("c")) + assertEquals(1, doorbellRows().size) + } + + @Test + fun `a hand-over whose kept wait reads as long after a clock change gets its wake`() { + network(RequiredNetwork.NONE) + handedOver("e1", entry(state = EntryState.RUNNING, nextAttemptAt = work.now + 60_000)) + assertEquals(listOf("e1#wake"), uploadRows().map { it.name }) + work.now += 60_000 + assertEquals(1, startable().size) + } + + @Test + fun `a doorbell does not end while an entry's main row insert fails`() { + network(RequiredNetwork.NONE) + handedOver() + network(RequiredNetwork.CONNECTED) + work.lateInsertFailures = 1 + val bell = doorbellRows().single() + ringDoorbells() + // The ring returned retry: the doorbell stays, and rings again after WorkManager's backoff. + assertEquals(ENQUEUED, bell.state) + assertEquals(emptyList(), uploadRows()) + work.now += 30_000 + ringDoorbells() + assertEquals(1, startable().size) + } + + @Test + fun `a doorbell answer that throws is a retry, not a failure`() { + assertFalse(DoorbellRing.answered { throw IllegalStateException("store unreadable") }) + assertTrue(DoorbellRing.answered { true }) + assertFalse(DoorbellRing.answered { false }) + assertThrows(java.util.concurrent.CancellationException::class.java) { + DoorbellRing.answered { throw java.util.concurrent.CancellationException("stopped") } + } + } + + @Test + fun `the fake, a worker that fails takes the doorbell queued behind it with it`() { + // Why the doorbell worker must not fail: WorkManager fails the rows blocked behind it. + scheduler.ensureDoorbell(RequiredNetwork.CONNECTED) + val first = doorbellRows().single() + work.start(first) + scheduler.ensureDoorbell(RequiredNetwork.CONNECTED) + work.fail(first) + assertEquals(emptyList(), doorbellRows()) + } + + @Test + fun `the fake, a retry counts its backoff from the retry`() { + scheduler.ensureDoorbell(RequiredNetwork.CONNECTED) + val bell = doorbellRows().single() + work.now += 10 * minute + work.start(bell) + work.retry(bell) + assertEquals(work.now + 30_000, work.nextScheduleAt(bell)) + } + + @Test + fun `with 20 older sleeping rows the doorbell is outside the JobScheduler window until some of them run`() { + // Review probe P4. The result: while the app process is dead, the + // doorbell cannot ring before one of the older rows has run. A backoff + // wake runs at the latest at its backoff maximum, an expiry wake within + // the 24-hour horizon. The app's start, resume() or foreground starts + // the entry sooner when the network fits. + network(RequiredNetwork.NONE) + val older = (1..20).map { work.add("x$it#wake", initialDelay = 6 * 60 * minute) } + handedOver("a") + network(RequiredNetwork.CONNECTED) + val bell = doorbellRows().single() + assertTrue(bell in work.startableNow()) + assertTrue(bell !in work.startableWhileDead()) + // One older row runs and ends: the doorbell enters the window. + work.now += 6 * 60 * minute + work.start(older.first()) + work.finish(older.first()) + assertTrue(bell in work.startableWhileDead()) + // The app opens with the network: a foreground starts the entry at once. + controller.retryNow() + assertEquals(1, startable("a").size) + } + // MARK: - the fake follows WorkManager 2.12.0 @Test @@ -184,13 +413,19 @@ class NetworkDoorbellTest { work.start(bell) // WorkManager saw the network; the in-worker check does not (not validated any more). fitsThrows = true - assertTrue(newController().answerDoorbell()) + assertTrue(newController().answerDoorbell(RequiredNetwork.CONNECTED)) fitsThrows = false + val rungAt = work.now work.finish(bell) assertEquals(emptyList(), uploadRows()) val next = doorbellRows().single() assertTrue(next !== bell) - ringDoorbells() + // It waits at least 30 s, so a refusing check does not ring without pause. + assertEquals(emptyList(), ringDoorbells()) + work.now = rungAt + 29_999 + assertEquals(emptyList(), ringDoorbells()) + work.now = rungAt + 30_000 + assertEquals(listOf(next), ringDoorbells()) assertEquals(1, startable().size) } @@ -232,6 +467,12 @@ class NetworkDoorbellTest { handedOver("b") handedOver("c") assertEquals(listOf(RUNNING, BLOCKED), doorbellRows().map { it.state }) + // The running doorbell answers, then the one behind it: every entry starts. + assertTrue(newController().answerDoorbell(RequiredNetwork.CONNECTED)) + work.finish(bell) + ringDoorbells() + for (id in listOf("a", "b", "c")) assertEquals(id, 1, startable(id).size) + assertEquals(emptyList(), doorbellRows()) } @Test @@ -285,17 +526,31 @@ class NetworkDoorbellTest { @Test fun `a process death between the hand-over save and the doorbell leaves an entry that every repair finds`() { // The entry is queued with no row: the run's own row ended, the doorbell never landed. - network(RequiredNetwork.NONE) - store.save(entry(state = EntryState.QUEUED)) - controller.retryNow() - assertEquals(1, doorbellRows(RequiredNetwork.CONNECTED).size) - assertEquals(emptyList(), uploadRows()) - // The same with the network up: a foreground starts it. - work.rows.clear() - network(RequiredNetwork.CONNECTED) - controller.retryNow() - assertEquals(1, startable().size) - assertEquals(emptyList(), doorbellRows()) + controller.enqueue(parsed()) + val repairs = listOf Unit>>( + "boot sweep" to { controller.sweep() }, + "resume" to { controller.resume() }, + "foreground" to { controller.retryNow() }, + "same-id enqueue" to { controller.enqueue(parsed()) }, + "setWifiOnly" to { + controller.setWifiOnly(true) + controller.setWifiOnly(false) + }, + ) + for ((name, repair) in repairs) { + for (online in listOf(false, true)) { + work.rows.clear() + network(if (online) RequiredNetwork.CONNECTED else RequiredNetwork.NONE) + repair() + if (online) { + assertEquals("$name online", 1, startable().size) + assertEquals("$name online", emptyList(), doorbellRows()) + } else { + assertEquals("$name offline", emptyList(), uploadRows()) + assertEquals("$name offline", 1, doorbellRows(RequiredNetwork.CONNECTED).size) + } + } + } } @Test @@ -494,7 +749,7 @@ class NetworkDoorbellTest { assertTrue(doorbell.contains(".setConstraints(")) // The insert is awaited, so one that fails later is seen. assertTrue(doorbell.contains(".enqueue().result")) - assertTrue(doorbell.contains(".get(WorkManagerScheduler.DOORBELL_ENQUEUE_TIMEOUT_MS")) + assertTrue(doorbell.contains(".get(WorkManagerScheduler.WORK_CALL_TIMEOUT_MS")) assertFalse(source.substringAfter("private fun request(").contains(".setConstraints(")) } diff --git a/android/src/test/java/ai/openspace/backgroundupload/QueueControllerTest.kt b/android/src/test/java/ai/openspace/backgroundupload/QueueControllerTest.kt index 4d43b07b..52dcdd0e 100644 --- a/android/src/test/java/ai/openspace/backgroundupload/QueueControllerTest.kt +++ b/android/src/test/java/ai/openspace/backgroundupload/QueueControllerTest.kt @@ -841,9 +841,10 @@ class QueueControllerTest { store.save(entry(id = "busy", state = EntryState.QUEUED)) store.save(entry(id = "paused", state = EntryState.PAUSED)) store.save(entry(id = "running", state = EntryState.RUNNING)) + // A worker of "busy" may be one that handed over and is leaving: schedule() queues a row behind it. running += "busy" c.setWifiOnly(false) - assertEquals(setOf("follow", "stale"), scheduler.scheduled.toSet()) + assertEquals(setOf("follow", "stale", "busy"), scheduler.scheduled.toSet()) assertEquals(emptyList(), scheduler.doorbells) assertEquals(emptyList>(), scheduler.wakes) } @@ -900,8 +901,9 @@ class QueueControllerTest { override fun rowSummary(): RowSummary = throw IllegalStateException("no WorkManager") } assertFalse(QueueController(store, journal, settings, events, broken, { false }, { now }).answerDoorbell()) + // The doorbell waits for each main row to be stored. val failing = object : WorkScheduler by scheduler { - override fun schedule(entry: QueueEntry) = throw IllegalStateException("enqueue failed") + override fun scheduleAndWait(entry: QueueEntry) = throw IllegalStateException("insert failed") } assertFalse(QueueController(store, journal, settings, events, failing, { false }, { now }).answerDoorbell()) assertTrue(controller.answerDoorbell()) diff --git a/android/src/test/java/ai/openspace/backgroundupload/TestSupport.kt b/android/src/test/java/ai/openspace/backgroundupload/TestSupport.kt index 1f23f2ae..83319aa0 100644 --- a/android/src/test/java/ai/openspace/backgroundupload/TestSupport.kt +++ b/android/src/test/java/ai/openspace/backgroundupload/TestSupport.kt @@ -130,6 +130,8 @@ internal class FakeScheduler : WorkScheduler { val startable = mutableSetOf() /** What rowSummary() returns as withWake. */ val withWake = mutableSetOf() + /** What rowSummary() returns as due. */ + val due = mutableSetOf() /** How many rowSummary() calls were made. */ var startableQueries = 0 @@ -154,7 +156,11 @@ internal class FakeScheduler : WorkScheduler { cancelledWakes += id } - override fun ensureDoorbell(network: RequiredNetwork) { + /** The delayMs of every ensureDoorbell call, in order. */ + val doorbellDelays = mutableListOf() + + override fun ensureDoorbell(network: RequiredNetwork, delayMs: Long) { + doorbellDelays += delayMs if (doorbellFailures > 0) { doorbellFailures-- throw IllegalStateException("insert failed") @@ -166,7 +172,7 @@ internal class FakeScheduler : WorkScheduler { override fun rowSummary(): RowSummary { startableQueries++ - return RowSummary(startable.toSet(), withWake.toSet()) + return RowSummary(startable.toSet(), withWake.toSet(), due.toSet()) } } @@ -287,16 +293,39 @@ internal class FakeWorkManager(var now: Long = 10_000L) : WorkRows { return rows.filter { it.state in states }.map(::view) } + /** + * How many of the next upload-row inserts fail after the call returned, + * as WorkManager's insert on its own thread can: the row never lands, and + * only a caller that waits for the result sees it. + */ + var lateInsertFailures = 0 + + /** How many doorbell inserts were tried. */ + var doorbellInserts = 0 + override fun enqueue(name: String, policy: ExistingWorkPolicy, entry: QueueEntry, delayMs: Long) { + if (lateInsertFailures > 0) { + lateInsertFailures-- + return + } add(name, policy, entry.id, delayMs, RequiredNetwork.NONE, null) } - override fun enqueueDoorbell(name: String, policy: ExistingWorkPolicy, network: RequiredNetwork) { + override fun enqueueAndWait(name: String, policy: ExistingWorkPolicy, entry: QueueEntry, delayMs: Long) { + if (lateInsertFailures > 0) { + lateInsertFailures-- + throw java.util.concurrent.ExecutionException(IllegalStateException("insert failed")) + } + add(name, policy, entry.id, delayMs, RequiredNetwork.NONE, null) + } + + override fun enqueueDoorbell(name: String, policy: ExistingWorkPolicy, network: RequiredNetwork, delayMs: Long) { + doorbellInserts++ if (doorbellInsertFailures > 0) { doorbellInsertFailures-- throw java.util.concurrent.ExecutionException(IllegalStateException("insert failed")) } - add(name, policy, null, 0L, network, network) + add(name, policy, null, delayMs, network, network) } private fun add(name: String, policy: ExistingWorkPolicy, entryId: String?, delayMs: Long, network: RequiredNetwork, doorbell: RequiredNetwork?) { @@ -330,6 +359,23 @@ internal class FakeWorkManager(var now: Long = 10_000L) : WorkRows { row.state = WorkInfo.State.ENQUEUED } + /** + * The worker returned retry: back to ENQUEUED. WorkManager keeps the count + * and sets the last enqueue time to now, so the backoff counts from now + * (WorkerWrapper.reschedule). + */ + fun retry(row: Row) { + check(row.state == WorkInfo.State.RUNNING) + row.state = WorkInfo.State.ENQUEUED + row.lastEnqueueTime = now + } + + /** The worker threw or returned failure: FAILED, and every row blocked behind it fails without a run. */ + fun fail(row: Row) { + row.state = WorkInfo.State.FAILED + rows.filter { it.name == row.name && it.state == WorkInfo.State.BLOCKED }.forEach { it.state = WorkInfo.State.FAILED } + } + /** A run ends: the row succeeds, and a row blocked behind it becomes ENQUEUED. */ fun finish(row: Row) { row.state = WorkInfo.State.SUCCEEDED From 649b4a7a6f78da4aecbe81c9e72d28d0230d5e51 Mon Sep 17 00:00:00 2001 From: Dylan Murphy Date: Wed, 18 Nov 2026 11:03:39 -0500 Subject: [PATCH 07/12] Docs: doorbell known limits, target-API rule, snapshot before the sweep Co-Authored-By: Claude Fable 5.2 --- CHANGELOG.md | 48 ++++++++++++++++++++++++++++++++++++------------ README.md | 14 +++++++++----- src/types.ts | 10 ++++++---- 3 files changed, 51 insertions(+), 21 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 648d0be1..b2826e03 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -128,13 +128,34 @@ Before, the job used the app's job quota while it waited (in the `rare` standby bucket, 10 minutes in 24 hours) and Android stopped it at its time limit. Android 10 (API 29) keeps the poll and has no doorbell. -On Android 15 and later (API 35), a run that holds foreground mode polls -for at most 30 minutes, then hands over the same way: each minute of it -uses the `dataSync` budget of 6 hours in 24. Below API 35 it polls as -before. Known limit: on API 31 and later a run that a doorbell starts in -the background cannot take foreground mode, so a single body (`data`, -`form`, `file`) that needs more than the 10-minute job limit cannot finish -until the app is opened. Use `parts` for large bodies. +In an app that targets API 35, on Android 15 and later, a run that holds +foreground mode polls for at most 30 minutes, then hands over the same +way: Android documents the `dataSync` limit of 6 hours in 24 for apps that +target Android 15, and each minute of the poll uses it. In other apps, and +below Android 15, such a run polls as before. + +A doorbell whose network WorkManager sees, but the library's check does +not, queues the next doorbell with a 30 s delay, so it rings at most once +every 30 s. Each attempt now checks the network again once it has a +transmit slot, just before it is written ahead. When the check fails, the +request leaves the slot with no attempt counted and waits for the network +again. Before, a Wi-Fi-only request that waited for a slot could go out +over another network. + +Known limits: + +- An app start with no network starts no run, so no "waiting for the + network" notification shows until a run starts. +- On API 31 and later a run that a doorbell starts in the background + cannot take foreground mode, so a single body (`data`, `form`, `file`) + that needs more than the 10-minute job limit cannot finish until the app + is opened. Use `parts` for large bodies. +- WorkManager hands JobScheduler at most 20 rows, the oldest first. With + 20 or more older sleeping rows (backoff or expiry wakes), the doorbell is + not handed to JobScheduler until some of them have run, while the app + process is not alive. That wait is bounded by the backoff maximum and + the 24-hour expiry-wake horizon. The app's start, `resume()` and return + to the foreground start the entry sooner when its network fits. **Behavior change:** on Android 11 and later, the boot sweep, `resume()`, a return to the foreground, a same-id `mutate()` of a queued entry and @@ -149,8 +170,8 @@ returns, or at the next start, resume or foreground with a network. Android: `setWifiOnly(false)` now moves the queued entries that follow the setting from the Wi-Fi rule to any network: each one with no long backoff -wait and no worker starts at once when a network fits, else it waits on -the network doorbell. The same value, or `setWifiOnly(true)`, schedules +wait starts at once when a network fits, else it waits on the network +doorbell. The same value, or `setWifiOnly(true)`, schedules nothing: a run that starts on another network hands over to the Wi-Fi doorbell. A run that hands over uses the setting at that moment. @@ -166,9 +187,12 @@ Android: the boot record (`takeDiagnostics()`, message `boot sweep`) now has `wifiOnly`, the queue's setting; `entriesByWake` (`{ backoff, network, wifi }`), the queued entries a backoff wake, the network doorbell or the Wi-Fi doorbell starts; and `doorbells` -(`{ network, wifi }`), whether each doorbell row exists. `entriesByWake` -and `doorbells` are left out when WorkManager could not be read. The -`Diagnostic` type has the three fields. +(`{ network, wifi }`), whether each doorbell row exists. Both describe the +state the start found, before the boot sweep changed it, and are left out +when WorkManager could not be read. `entriesWithoutRun` now also counts the +entries that wait for a doorbell, so by itself it no longer means an entry +is stuck: compare it with `entriesByWake`. The `Diagnostic` type has the +three new fields. ## 11.3.0 diff --git a/README.md b/README.md index eb43a977..9dd25019 100644 --- a/README.md +++ b/README.md @@ -221,8 +221,8 @@ at the next run, after a growing backoff. A body that needs more than accepted parts are kept across runs. iOS has no equal limit. **No network.** On API 30 and later a run that finds no usable network -checks every 10 s for at most 60 s; with foreground mode, on API 35 and -later, for at most 30 minutes. Then the entry goes back to `queued` with no +checks every 10 s for at most 60 s; with foreground mode, in an app that +targets API 35 on Android 15 and later, for at most 30 minutes. Then the entry goes back to `queued` with no row of its own and waits on a doorbell: one WorkManager row for the whole queue that waits for a validated network, and one that waits for a validated Wi-Fi network. When Android starts a doorbell, it starts the @@ -231,8 +231,9 @@ that sends has a network rule. The app's start, `resume()` and return to the foreground also start a waiting entry when its network fits, and otherwise make sure its doorbell exists. This is not a failed attempt and adds no backoff. A Wi-Fi-only upload also waits while the app binds its -process to a network that is not Wi-Fi. API 29, and a foreground run below -API 35, keep the 10 s check for as long as the run lasts. A run that a +process to a network that is not Wi-Fi, and every attempt checks the +network again once it has a transmit slot. API 29, and any other +foreground run, keep the 10 s check for as long as the run lasts. A run that a doorbell starts in the background cannot take foreground mode on API 31 and later, so the headless time limit above applies to it. @@ -796,7 +797,10 @@ many queued or running entries had no run at all. `entriesByWake` (`{ backoff, network, wifi }`) counts the queued entries by what starts them: a backoff wake, the network doorbell, or the Wi-Fi doorbell. `doorbells` (`{ network, wifi }`) says whether each doorbell row exists. -Both are left out when WorkManager could not be read. `wifiOnly` is the +Both describe the state before the boot sweep changed it, and are left out +when WorkManager could not be read. `entriesWithoutRun` also counts the +entries that wait for a doorbell, so by itself it does not mean an entry +is stuck. `wifiOnly` is the queue's Wi-Fi setting. Together they tell "waits for the network or for Wi-Fi" from "waits for a backoff". diff --git a/src/types.ts b/src/types.ts index 9ce16866..4e583cfb 100644 --- a/src/types.ts +++ b/src/types.ts @@ -481,8 +481,8 @@ export type QueueOrphans = { }; /** - * Android boot snapshot: the queued entries by what starts them. Each entry - * counts once. `backoff`: its wake at the end of a backoff wait. `network`: + * Android boot snapshot: the queued entries by what starts them, as the + * start found them, before the boot sweep. Each entry counts once. `backoff`: its wake at the end of a backoff wait. `network`: * the doorbell that waits for any validated network. `wifi`: the doorbell * that waits for Wi-Fi. An entry with a row of its own that is not a * backoff wake is not counted. @@ -493,7 +493,7 @@ export type BootWakeCounts = { wifi: number; }; -/** Android boot snapshot: whether each doorbell row exists. */ +/** Android boot snapshot: whether each doorbell row existed before the boot sweep. */ export type BootDoorbells = { network: boolean; wifi: boolean; @@ -503,7 +503,9 @@ export type BootDoorbells = { * A diagnostic record's structured fields. The boot snapshot (message * `'boot sweep'`) always has `v9Leftovers` and `orphans`. On Android it also * has `wifiOnly` (the queue's setting) and, when WorkManager could be read, - * `entriesByWake` and `doorbells`. Its other fields are untyped. + * `entriesByWake` and `doorbells`, both from the state before the boot + * sweep. Its `entriesWithoutRun` also counts entries that wait for a + * doorbell. Its other fields are untyped. */ export type DiagnosticData = Record & { v9Leftovers?: V9Leftovers; From 8d4f1753ee3a3be5bf836d6bb08aabced71da881 Mon Sep 17 00:00:00 2001 From: Dylan Murphy Date: Wed, 18 Nov 2026 12:50:23 -0500 Subject: [PATCH 08/12] Bound the chunked hand-over test so a missing in-slot check fails in seconds Co-Authored-By: Claude Fable 5.2 --- .../test/java/ai/openspace/backgroundupload/EntryRunTest.kt | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/android/src/test/java/ai/openspace/backgroundupload/EntryRunTest.kt b/android/src/test/java/ai/openspace/backgroundupload/EntryRunTest.kt index e462824e..84548031 100644 --- a/android/src/test/java/ai/openspace/backgroundupload/EntryRunTest.kt +++ b/android/src/test/java/ai/openspace/backgroundupload/EntryRunTest.kt @@ -9,6 +9,7 @@ import kotlinx.coroutines.currentCoroutineContext import kotlinx.coroutines.ensureActive import kotlinx.coroutines.launch import kotlinx.coroutines.runBlocking +import kotlinx.coroutines.withTimeout import kotlinx.coroutines.yield import org.junit.After import org.junit.Assert.assertEquals @@ -1701,7 +1702,9 @@ class EntryRunTest { // Part 0 finds the network (before and in its slot) and sends; part 1 then finds none. host.network = ArrayDeque(listOf(Connectivity.Ok, Connectivity.Ok) + List(30) { Connectivity.NoInternet }) val answer = CompletableDeferred() - host.handler = { _, _ -> answer.await() } + // Bounded: if a second part were let through to send (no network check + // in its slot), it would wait on this answer for ever. It fails in 5 s. + host.handler = { _, _ -> withTimeout(5_000) { answer.await() } } val start = host.clock host.onSleep = { if (host.clock - start >= 120_000 && !answer.isCompleted) answer.complete(UploadResponse(200, "", mapOf())) } run() From 1e87a0fa1ef70eabd1643db94b359daa98ff12a7 Mon Sep 17 00:00:00 2001 From: Dylan Murphy Date: Wed, 18 Nov 2026 13:05:41 -0500 Subject: [PATCH 09/12] Android: a WorkManager timeout does not reject a call or stop a pass From the delta review of the doorbell design: - enqueue, resume, updateHeaders, the boot sweep, retryNow and setWifiOnly catch a scheduling failure per entry. At the first WorkManager timeout a pass stops scheduling, writes one diagnostics record and asks for a repair; mutate() resolves once the entry is saved, and the sweep still writes its boot record. - repairRows() runs on the queue executor after 30 s, doubling to 5 minutes while it fails, and starts every queued entry that has no row that can start now (main row or doorbell) and makes missing wakes. - A doorbell ring stops at the first timeout with one record and returns retry. - The WorkManager library query also times out after 10 s. - The ring-spacing test covers ten minutes and asserts the gaps. Co-Authored-By: Claude Fable 5.2 --- .../backgroundupload/QueueController.kt | 184 ++++++++++++++---- .../openspace/backgroundupload/Scheduler.kt | 2 +- .../backgroundupload/UploaderModule.kt | 8 + .../backgroundupload/NetworkDoorbellTest.kt | 65 ++++++- .../backgroundupload/QueueControllerTest.kt | 74 +++++++ .../openspace/backgroundupload/TestSupport.kt | 18 ++ 6 files changed, 314 insertions(+), 37 deletions(-) diff --git a/android/src/main/java/ai/openspace/backgroundupload/QueueController.kt b/android/src/main/java/ai/openspace/backgroundupload/QueueController.kt index d90c6495..d6437a26 100644 --- a/android/src/main/java/ai/openspace/backgroundupload/QueueController.kt +++ b/android/src/main/java/ai/openspace/backgroundupload/QueueController.kt @@ -28,8 +28,25 @@ class QueueController( private val networkFits: (wifiOnly: Boolean) -> Boolean = { true }, /** Whether an entry with no network may wait on a doorbell: API 30 and later. */ private val doorbells: Boolean = true, + /** + * Runs [task] on the queue executor after [delayMs]. A pass whose + * WorkManager calls failed asks for [repairRows] this way, so an entry + * left with no row is started with no app call. + */ + private val repairLater: (delayMs: Long, task: () -> Unit) -> Unit = { _, _ -> }, ) { + companion object { + /** The first wait before [repairRows] runs after a failed pass; it doubles up to [REPAIR_MAX_MS]. */ + const val REPAIR_FIRST_MS = 30_000L + const val REPAIR_MAX_MS = 300_000L + } + + private val repairPending = java.util.concurrent.atomic.AtomicBoolean(false) + + @Volatile + private var repairDelayMs = REPAIR_FIRST_MS + // MARK: - enqueue /** [repair]: the entry was queued already (a same-id enqueue of the same body). */ @@ -66,8 +83,12 @@ class QueueController( result.entry?.let { entry -> // New work starts a run, which shows the notification and may take // foreground mode. A same-id enqueue of a queued entry is a repair. - scheduleRun(entry, probe = if (result.repair) NetworkProbe() else null) + // The entry is saved: a WorkManager call that fails or times out does + // not reject mutate(). The repair below starts the entry later. + val pass = SchedulePass("enqueue") + pass.schedule { scheduleRun(entry, probe = if (result.repair) NetworkProbe() else null) } events.state(entry.toRow()) + pass.finish() } return p.id } @@ -246,7 +267,9 @@ class QueueController( result.changed.forEach { events.state(it.toRow()) } // Every queued entry, not only the resumed ones: a run is idempotent. val probe = NetworkProbe() - store.all().forEach { scheduleRun(it, probe = probe) } + val pass = SchedulePass("resume") + store.all().forEach { e -> pass.schedule { scheduleRun(e, probe = probe) } } + pass.finish() result.throwIfFailed("resume") } @@ -277,12 +300,8 @@ class QueueController( val now = clock() val queued = store.all().filter { !it.legacy && it.state == EntryState.QUEUED && inScope(it, keys) } val cleared = mutableListOf() - var failures = 0 - var first: Throwable? = null - fun failed(error: Throwable) { - failures++ - if (first == null) first = error - } + val pass = SchedulePass("retryNow") + fun failed(error: Throwable) = pass.failed(error) store.locked { queued.filter { EntryTransitions.clearsWait(it, now) }.forEach { e -> try { @@ -298,7 +317,7 @@ class QueueController( } } val probe = NetworkProbe() - cleared.forEach { e -> runCatching { scheduleRun(e, probe = probe) }.onFailure(::failed) } + cleared.forEach { e -> pass.schedule { scheduleRun(e, probe = probe) } } val clearedIds = cleared.map { it.id }.toSet() // A wait that is due, or shorter than a worker sleeps out, is no wait // here: such an entry follows the network rule like any other (a @@ -309,14 +328,14 @@ class QueueController( if (idle.isNotEmpty() || waiting.isNotEmpty()) { val rows = runCatching { scheduler.rowSummary() }.onFailure(::failed).getOrNull() idle.filter { rows == null || it.id !in rows.startable } - .forEach { e -> runCatching { scheduleRun(e, probe = probe) }.onFailure(::failed) } + .forEach { e -> pass.schedule { scheduleRun(e, probe = probe) } } // A wait that stays needs its wake. KEEP leaves one that exists, so // this is at most one enqueue per entry that lacks a wake. waiting.filter { rows == null || it.id !in rows.withWake } - .forEach { e -> runCatching { scheduler.scheduleWake(e, e.nextAttemptAt!!, replace = false) }.onFailure(::failed) } + .forEach { e -> pass.schedule { scheduler.scheduleWake(e, e.nextAttemptAt!!, replace = false) } } } cleared.forEach { events.state(it.toRow()) } - first?.let { Diag.error("retryNow: $failures entry saves or WorkManager calls failed; the next app start or foreground tries again", it) } + pass.finish() } private fun inScope(e: QueueEntry, keys: List?) = keys == null || e.key in keys @@ -336,24 +355,25 @@ class QueueController( val s = settings.load() val waiting = store.all().filter { requiredNetwork(it, s, now) != RequiredNetwork.NONE } if (waiting.isEmpty()) return true - val rows = runCatching { scheduler.rowSummary() } - .onFailure { Diag.error("doorbell: could not read WorkManager; it rings again", it) } - .getOrNull() ?: return false + val pass = SchedulePass("doorbell") + val rows = try { + scheduler.rowSummary() + } catch (error: Exception) { + pass.failed(error) + pass.finish(repair = false) + return false + } // WorkManager saw [rung]'s network. When the library's check does not, // the doorbell queued behind this one waits DOORBELL_REPEAT_MS. val probe = NetworkProbe(repeatOf = rung, awaitRows = true) - var answered = true // Only an ENQUEUED row that starts now covers an entry. A worker of the // entry, or a RUNNING row, may be one that handed over and is on its way // out; schedule() then queues one row behind it, and the per-id gate - // runs them one after the other. - waiting.filter { it.id !in rows.due }.forEach { e -> - runCatching { scheduleRun(e, probe = probe) }.onFailure { - answered = false - Diag.error("doorbell: could not schedule '${e.id}'; it rings again", it) - } - } - return answered + // runs them one after the other. The pass stops at the first + // WorkManager timeout: the doorbell returns retry and rings again. + waiting.filter { it.id !in rows.due }.forEach { e -> pass.schedule { scheduleRun(e, probe = probe) } } + pass.finish(repair = false) + return !pass.anyFailed } /** @@ -490,18 +510,18 @@ class QueueController( // Off: the entries that follow the setting now wait for any network. // Each one with no long wait starts when that fits, else it waits on // the CONNECTED doorbell. A worker of the entry may be one that handed - // over and is on its way out: schedule() queues one row behind it. A Wi-Fi doorbell left behind - // finds nothing to start and ends. On needs no work: a run that starts - // on another network fails the check and hands over to the Wi-Fi doorbell. + // over and is on its way out: schedule() queues one row behind it. A + // Wi-Fi doorbell left behind finds nothing to start and ends. On needs + // no work: a run that starts on another network fails the check and + // hands over to the Wi-Fi doorbell. val now = clock() val s = settings.load() val probe = NetworkProbe() + val pass = SchedulePass("setWifiOnly") store.all() .filter { it.descriptor?.wifiOnly == null && requiredNetwork(it, s, now) != RequiredNetwork.NONE } - .forEach { e -> - runCatching { scheduleRun(e, probe = probe) } - .onFailure { Diag.error("setWifiOnly: could not schedule '${e.id}'", it) } - } + .forEach { e -> pass.schedule { scheduleRun(e, probe = probe) } } + pass.finish() } fun configureRetry(defaults: RetryDefaults) { @@ -533,10 +553,12 @@ class QueueController( unparkedIds += e.id EntryTransitions.toUnparked(patched, s.isPaused(patched.key), now) } + val pass = SchedulePass("updateHeaders") result.changed.filter { it.id in unparkedIds }.forEach { entry -> - scheduleRun(entry) + pass.schedule { scheduleRun(entry) } events.state(entry.toRow()) } + pass.finish() result.throwIfFailed("updateHeaders") } @@ -696,7 +718,8 @@ class QueueController( val runs = runCatching { scheduler.libraryRuns() }.getOrNull() toForget.forEach { scheduler.cancel(it) } val probe = NetworkProbe() - toSchedule.forEach { scheduleRun(it, keepWake = true, probe = probe) } + val pass = SchedulePass("boot sweep") + toSchedule.forEach { e -> pass.schedule { scheduleRun(e, keepWake = true, probe = probe) } } // Every start, even an idle one: a v9 leftover count of zero is a value // the app reports. It replaces a snapshot JS has not taken yet, so // launches with no takeDiagnostics() do not push out real warnings. @@ -710,6 +733,7 @@ class QueueController( replaceUntaken = true, ) changed.forEach { events.state(it.toRow()) } + pass.finish() } private fun trySave(entry: QueueEntry): Boolean = try { @@ -763,6 +787,98 @@ class QueueController( } } + /** + * The scheduling of one pass. A WorkManager call that fails is caught per + * entry, so the pass goes on and the caller does not reject. At the first + * timeout (a WorkManager that does not answer) the pass stops scheduling: + * each further call would hold the queue executor, which also serves + * every JS call, for 10 s more. [finish] writes one diagnostics record + * for the pass, and asks for [repairRows] later, which starts the entries + * the pass left with no row. + */ + private inner class SchedulePass(val what: String) { + private var failures = 0 + private var skipped = 0 + private var first: Throwable? = null + private var stopped = false + + val anyFailed get() = failures + skipped > 0 + + fun failed(error: Throwable) { + failures++ + if (first == null) first = error + if (isTimeout(error)) stopped = true + } + + fun schedule(block: () -> Unit) { + if (stopped) { + skipped++ + return + } + try { + block() + } catch (error: Exception) { + failed(error) + } + } + + fun finish(repair: Boolean = true) { + if (!anyFailed) return + val tail = if (stopped) "; stopped at a WorkManager timeout with $skipped left" else "" + Diag.error("$what: $failures entry saves or WorkManager calls failed$tail; the queue tries again", first) + if (repair) requestRepair() + } + } + + private fun isTimeout(error: Throwable) = + generateSequence(error) { it.cause }.any { it is java.util.concurrent.TimeoutException } + + /** Asks for one [repairRows] run later; one is pending at a time. The wait doubles after each failed repair. */ + private fun requestRepair() { + if (!repairPending.compareAndSet(false, true)) return + repairLater(repairDelayMs) { + repairPending.set(false) + repairRows() + } + } + + /** + * Starts every queued entry that has nothing to start it: after a pass + * whose WorkManager calls failed. One WorkManager query. An entry with + * no long wait and no row that can start now is scheduled as a repair + * (its main row when the network fits, else its doorbell); an entry with + * a long wait and no wake gets its wake (KEEP); a parked entry gets its + * expiry wake. It does not end a backoff wait. When it fails too, it + * asks for itself again, later each time, up to [REPAIR_MAX_MS]. + */ + fun repairRows() { + val now = clock() + val entries = store.all().filter { !it.legacy && (it.state == EntryState.QUEUED || it.state == EntryState.AWAITING_AUTH) } + val pass = SchedulePass("repair") + if (entries.isNotEmpty()) { + val rows = try { + scheduler.rowSummary() + } catch (error: Exception) { + pass.failed(error) + null + } + if (rows != null) { + val probe = NetworkProbe() + for (e in entries) { + val at = e.nextAttemptAt + val longWait = at != null && at - now > RetryClassifier.IN_WORKER_BACKOFF_MAX_MS + when { + e.state == EntryState.AWAITING_AUTH || longWait -> + if (e.id !in rows.withWake) pass.schedule { scheduleRun(e, keepWake = true) } + e.id !in rows.startable -> pass.schedule { scheduleRun(e, probe = probe) } + } + } + } + } + repairDelayMs = if (pass.anyFailed) minOf(repairDelayMs * 2, REPAIR_MAX_MS) else REPAIR_FIRST_MS + pass.finish() + } + /** * True when [entry]'s network does not fit now and its doorbell is * stored. False when it fits, on API 29, or when the doorbell failed. @@ -777,6 +893,8 @@ class QueueController( return runCatching { scheduler.ensureDoorbell(network, delay) } .onSuccess { probe.ensured += network } .onFailure { + // A WorkManager that does not answer: the pass stops (SchedulePass). + if (isTimeout(it)) throw it // One record per pass: the other entries of this rule skip the doorbell. probe.failed += network Diag.error("could not store the ${network.name} doorbell; its waiting entries start a run instead", it) diff --git a/android/src/main/java/ai/openspace/backgroundupload/Scheduler.kt b/android/src/main/java/ai/openspace/backgroundupload/Scheduler.kt index c2580cab..58085a29 100644 --- a/android/src/main/java/ai/openspace/backgroundupload/Scheduler.kt +++ b/android/src/main/java/ai/openspace/backgroundupload/Scheduler.kt @@ -428,7 +428,7 @@ private class WorkManagerRows(private val workManager: WorkManager) : WorkRows { override fun library(states: List): List = workManager.getWorkInfos( WorkQuery.Builder.fromTags(listOf(WorkManagerScheduler.WORK_TAG)).addStates(states).build(), - ).get().map(::row) + ).get(WorkManagerScheduler.WORK_CALL_TIMEOUT_MS, TimeUnit.MILLISECONDS).map(::row) override fun enqueue(name: String, policy: ExistingWorkPolicy, entry: QueueEntry, delayMs: Long) { workManager.beginUniqueWork(name, policy, request(entry, delayMs)).enqueue() diff --git a/android/src/main/java/ai/openspace/backgroundupload/UploaderModule.kt b/android/src/main/java/ai/openspace/backgroundupload/UploaderModule.kt index a49e3c03..26022767 100644 --- a/android/src/main/java/ai/openspace/backgroundupload/UploaderModule.kt +++ b/android/src/main/java/ai/openspace/backgroundupload/UploaderModule.kt @@ -43,6 +43,11 @@ class UploaderModule(context: ReactApplicationContext) : var instance: UploaderModule? = null private set + /** Waits out a repair delay, then hands the repair to [queueExecutor]. */ + private val repairTimer by lazy { + Executors.newSingleThreadScheduledExecutor { r -> Thread(r, "RNFileUploader.repair").apply { isDaemon = true } } + } + /** One thread for every module-side disk operation. It outlives a JS reload. */ val queueExecutor: ExecutorService = Executors.newSingleThreadExecutor { r -> Thread(r, "RNFileUploader.queue") @@ -58,6 +63,9 @@ class UploaderModule(context: ReactApplicationContext) : store, journal, settings, EventReporter, scheduler, networkFits = { wifiOnly -> networkFitsNow(context, wifiOnly) }, doorbells = Build.VERSION.SDK_INT >= EntryRun.DOORBELL_MIN_SDK, + // A pass whose WorkManager calls failed asks for a repair: it runs on + // the queue executor after the wait. + repairLater = { delayMs, task -> repairTimer.schedule({ queueExecutor.execute(task) }, delayMs, TimeUnit.MILLISECONDS) }, ) // Before init: the lifecycle listener it registers can fire at once. diff --git a/android/src/test/java/ai/openspace/backgroundupload/NetworkDoorbellTest.kt b/android/src/test/java/ai/openspace/backgroundupload/NetworkDoorbellTest.kt index 0ac6f9af..bdfb14fb 100644 --- a/android/src/test/java/ai/openspace/backgroundupload/NetworkDoorbellTest.kt +++ b/android/src/test/java/ai/openspace/backgroundupload/NetworkDoorbellTest.kt @@ -199,18 +199,19 @@ class NetworkDoorbellTest { } @Test - fun `a doorbell whose library check keeps refusing rings at most once every 30 s`() { + fun `while the library check keeps refusing, each ring is followed by the next one at least 30 s later`() { // Review probe P2: WorkManager says the rule is met; the library's check says no. network(RequiredNetwork.NONE) handedOver("a") network(RequiredNetwork.CONNECTED) fitsThrows = true val rings = mutableListOf() - repeat(20) { + // Ten minutes, a WorkManager look every second. + repeat(600) { if (ringDoorbells().isNotEmpty()) rings += work.now work.now += 1_000 } - assertTrue("rings at $rings", rings.size >= 1) + assertTrue("rings at $rings", rings.size in 10..21) assertTrue("rings at $rings", rings.zipWithNext().all { (x, y) -> y - x >= 30_000 }) assertTrue("rows: ${work.rows.size}", work.rows.count { it.doorbell != null } <= rings.size + 1) assertEquals(1, doorbellRows().size) @@ -334,6 +335,64 @@ class NetworkDoorbellTest { assertEquals(1, startable("a").size) } + // MARK: - a WorkManager that does not answer (delta review) + + /** The controller's later repairs, in order: the delay and the task. */ + private val repairs = mutableListOf Unit>>() + + private fun controllerWithRepairs() = QueueController( + store, journal, settings, events, scheduler, { it in running }, { work.now }, + networkFits = { wifiOnly -> work.network.ordinal >= (if (wifiOnly) RequiredNetwork.WIFI else RequiredNetwork.CONNECTED).ordinal }, + repairLater = { delayMs, task -> repairs += delayMs to task }, + ) + + @Test + fun `a doorbell ring stops at the first WorkManager timeout, writes one record and asks to ring again`() { + val file = File(tmp.newFolder(), "diagnostics.json") + Diag.log = DiagnosticLog(file) + try { + for (n in 1..250) store.save(entry(id = "q$n")) + network(RequiredNetwork.CONNECTED) + work.unfinishedTimeouts = 1_000 + assertFalse(newController().answerDoorbell(RequiredNetwork.CONNECTED)) + assertEquals(1, work.timedOutQueries) + assertEquals(1, DiagnosticLog(file).drain().count { it.level == DiagnosticLog.ERROR }) + } finally { + Diag.log = null + } + } + + @Test + fun `an entry saved while WorkManager does not answer is started later with no app call`() { + val c = controllerWithRepairs() + network(RequiredNetwork.CONNECTED) + work.unfinishedTimeouts = 1 + // mutate() resolves: the entry is saved. + assertEquals("e1", c.enqueue(parsed())) + assertEquals(EntryState.QUEUED, store.load("e1")!!.state) + assertEquals(emptyList(), uploadRows()) + // The queue executor runs the repair it asked for. + val (delay, task) = repairs.single() + assertEquals(QueueController.REPAIR_FIRST_MS, delay) + work.now += delay + task() + assertEquals(1, startable().size) + } + + @Test + fun `a repair that fails again asks for itself again, later each time`() { + val c = controllerWithRepairs() + network(RequiredNetwork.CONNECTED) + work.unfinishedTimeouts = 3 + c.enqueue(parsed()) + repairs.removeAt(0).second() + repairs.removeAt(0).second() + assertEquals(listOf(4 * QueueController.REPAIR_FIRST_MS), repairs.map { it.first }) + repairs.removeAt(0).second() + assertEquals(1, startable().size) + assertEquals(emptyList Unit>>(), repairs) + } + // MARK: - the fake follows WorkManager 2.12.0 @Test diff --git a/android/src/test/java/ai/openspace/backgroundupload/QueueControllerTest.kt b/android/src/test/java/ai/openspace/backgroundupload/QueueControllerTest.kt index 52dcdd0e..0c3438af 100644 --- a/android/src/test/java/ai/openspace/backgroundupload/QueueControllerTest.kt +++ b/android/src/test/java/ai/openspace/backgroundupload/QueueControllerTest.kt @@ -828,6 +828,80 @@ class QueueControllerTest { private fun withNetwork(fits: Boolean, doorbells: Boolean = true) = QueueController(store, journal, settings, events, scheduler, { it in running }, { now }, { fits }, doorbells) + /** The controller's later repairs, and a controller that records them. */ + private val repairs = mutableListOf Unit>>() + + private fun withRepairs() = + QueueController(store, journal, settings, events, scheduler, { it in running }, { now }, { true }, true) { delay, task -> repairs += delay to task } + + private fun errorsIn(file: File) = DiagnosticLog(file).drain().filter { it.level == DiagnosticLog.ERROR } + + @Test + fun `enqueue resolves when the schedule times out, and the repair schedules the entry`() { + val file = File(tmp.newFolder(), "diagnostics.json") + Diag.log = DiagnosticLog(file) + val c = withRepairs() + scheduler.scheduleTimeouts = 1 + assertEquals("e1", c.enqueue(parsed())) + assertEquals(EntryState.QUEUED, store.load("e1")!!.state) + assertEquals(listOf("state:e1:queued"), events.log) + assertEquals(1, errorsIn(file).size) + repairs.single().second() + assertEquals(listOf("e1"), scheduler.scheduled) + } + + @Test + fun `resume stops scheduling at a timeout, does not reject, and the repair schedules the rest`() { + val file = File(tmp.newFolder(), "diagnostics.json") + Diag.log = DiagnosticLog(file) + for (n in 1..3) store.save(entry(id = "q$n", state = EntryState.QUEUED)) + val c = withRepairs() + scheduler.scheduleTimeouts = 1 + c.resume() + assertEquals(emptyList(), scheduler.scheduled) + assertEquals(1, errorsIn(file).size) + repairs.single().second() + assertEquals(setOf("q1", "q2", "q3"), scheduler.scheduled.toSet()) + } + + @Test + fun `updateHeaders does not reject on a schedule timeout, and the repair schedules the unparked entries`() { + val file = File(tmp.newFolder(), "diagnostics.json") + Diag.log = DiagnosticLog(file) + for (n in 1..2) store.save(entry(id = "p$n", state = EntryState.AWAITING_AUTH, parkedGeneration = 0)) + val c = withRepairs() + scheduler.scheduleTimeouts = 1 + c.updateHeaders(mapOf("Authorization" to "Bearer new")) + assertEquals(EntryState.QUEUED, store.load("p1")!!.state) + assertEquals(1, errorsIn(file).size) + repairs.single().second() + assertEquals(setOf("p1", "p2"), scheduler.scheduled.toSet()) + } + + @Test + fun `the boot sweep writes its snapshot after a schedule timeout, and the repair schedules the rest`() { + val file = File(tmp.newFolder(), "diagnostics.json") + Diag.log = DiagnosticLog(file) + for (n in 1..3) store.save(entry(id = "q$n", state = EntryState.QUEUED)) + val c = withRepairs() + scheduler.scheduleTimeouts = 1 + c.sweep() + val records = DiagnosticLog(file).drain() + assertEquals(1, records.count { it.message == BootSnapshot.MESSAGE }) + assertEquals(1, records.count { it.level == DiagnosticLog.ERROR }) + repairs.single().second() + assertEquals(setOf("q1", "q2", "q3"), scheduler.scheduled.toSet()) + } + + @Test + fun `a failed pass asks for one repair at a time`() { + val c = withRepairs() + scheduler.scheduleTimeouts = 2 + c.enqueue(parsed(id = "a")) + c.enqueue(parsed(id = "b")) + assertEquals(1, repairs.size) + } + @Test fun `setWifiOnly(false) starts the queued entries that follow the setting when the network fits`() { val c = withNetwork(fits = true) diff --git a/android/src/test/java/ai/openspace/backgroundupload/TestSupport.kt b/android/src/test/java/ai/openspace/backgroundupload/TestSupport.kt index 83319aa0..8faa9098 100644 --- a/android/src/test/java/ai/openspace/backgroundupload/TestSupport.kt +++ b/android/src/test/java/ai/openspace/backgroundupload/TestSupport.kt @@ -135,7 +135,14 @@ internal class FakeScheduler : WorkScheduler { /** How many rowSummary() calls were made. */ var startableQueries = 0 + /** How many of the next schedule() calls time out, as a WorkManager that does not answer. */ + var scheduleTimeouts = 0 + override fun schedule(entry: QueueEntry) { + if (scheduleTimeouts > 0) { + scheduleTimeouts-- + throw java.util.concurrent.TimeoutException("WorkManager did not answer") + } scheduled += entry.id } @@ -277,7 +284,18 @@ internal class FakeWorkManager(var now: Long = 10_000L) : WorkRows { /** The rows that start now with the app process dead. */ fun startableWhileDead(): List = jobSchedulerRows().filter { it in startableNow() } + /** How many of the next unfinished(name) queries time out, as a WorkManager that does not answer. */ + var unfinishedTimeouts = 0 + + /** How many unfinished(name) queries timed out. */ + var timedOutQueries = 0 + override fun unfinished(name: String): List { + if (unfinishedTimeouts > 0) { + unfinishedTimeouts-- + timedOutQueries++ + throw java.util.concurrent.TimeoutException("WorkManager did not answer") + } nameQueries++ return open(name).map(::view) } From a508bf029378a6d234253363320a7c4190860c66 Mon Sep 17 00:00:00 2001 From: Dylan Murphy Date: Wed, 18 Nov 2026 13:05:47 -0500 Subject: [PATCH 10/12] Docs: WorkManager timeouts do not reject, and the ring-spacing exceptions Co-Authored-By: Claude Fable 5.2 --- CHANGELOG.md | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b2826e03..b7b8b742 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -134,9 +134,11 @@ way: Android documents the `dataSync` limit of 6 hours in 24 for apps that target Android 15, and each minute of the poll uses it. In other apps, and below Android 15, such a run polls as before. -A doorbell whose network WorkManager sees, but the library's check does -not, queues the next doorbell with a 30 s delay, so it rings at most once -every 30 s. Each attempt now checks the network again once it has a +A ring that the library's own check refuses (WorkManager sees the +network, the library does not) is followed by the next ring at least 30 s +later. Two bounded exceptions can add one ring with no delay: a hand-over +that lands while such a ring runs, and a repair that replaces a doorbell +WorkManager holds back. Each attempt now checks the network again once it has a transmit slot, just before it is written ahead. When the check fails, the request leaves the slot with no attempt counted and waits for the network again. Before, a Wi-Fi-only request that waited for a slot could go out @@ -194,6 +196,20 @@ entries that wait for a doorbell, so by itself it no longer means an entry is stuck: compare it with `entriesByWake`. The `Diagnostic` type has the three new fields. +Android: a WorkManager call that fails or times out no longer makes +`mutate()`, `resume()` or `updateHeaders()` reject, and no longer stops +the boot sweep. WorkManager queries and awaited inserts now time out after +10 s. A pass stops scheduling at the first timeout, because each further +call would hold the queue thread (which also serves every JS call) for +10 s more. It writes one diagnostics record and goes on with the rest of +its work: `mutate()` resolves once the entry is saved, and the boot sweep +still writes its boot record. Then the library itself runs a repair on its +queue thread, 30 s later at first, and later each time it fails again (up +to 5 minutes): it starts every queued entry that has no row that can start +now (its main row when the network fits, else its doorbell) and makes the +wakes that are missing. No app call is needed. A doorbell ring that meets +a timeout stops, writes one record and rings again later. + ## 11.3.0 Added `useRequests(filter?)` and `useRequestsWithProgress(filter?)`, React From 3553c8e986a62800333d8a40512dd0024050d27d Mon Sep 17 00:00:00 2001 From: Dylan Murphy Date: Wed, 18 Nov 2026 13:20:29 -0500 Subject: [PATCH 11/12] Android: wait for a WorkManager that does not answer once per repair From the third check of the doorbell design: - After a WorkManager timeout, while its repair is pending, a new pass makes no WorkManager call and leaves its entries to that repair, which has not read the store yet. Five enqueues now cost one timed-out query. A repair that schedules with no failure clears the state. - The boot sweep feeds a timeout of its run query into its pass, so it waits once, not twice. - CHANGELOG: the repair timer lives in memory; after a process death the boot sweep or a doorbell ring starts the entry. Co-Authored-By: Claude Fable 5.2 --- CHANGELOG.md | 9 +++- .../backgroundupload/QueueController.kt | 54 ++++++++++++++++--- .../backgroundupload/NetworkDoorbellTest.kt | 41 ++++++++++++++ .../openspace/backgroundupload/TestSupport.kt | 8 +++ 4 files changed, 103 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b7b8b742..c9a3e8a6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -207,8 +207,13 @@ still writes its boot record. Then the library itself runs a repair on its queue thread, 30 s later at first, and later each time it fails again (up to 5 minutes): it starts every queued entry that has no row that can start now (its main row when the network fits, else its doorbell) and makes the -wakes that are missing. No app call is needed. A doorbell ring that meets -a timeout stops, writes one record and rings again later. +wakes that are missing. No app call is needed while the process lives: +the repair timer is in memory, so if the process ends first, the next app +start (the boot sweep) or a doorbell ring starts the entry. While that +repair is pending, a new pass makes no WorkManager call and leaves its +entries to the repair, so each `mutate()` does not wait 10 s again. A +doorbell ring that meets a timeout stops, writes one record and rings +again later. ## 11.3.0 diff --git a/android/src/main/java/ai/openspace/backgroundupload/QueueController.kt b/android/src/main/java/ai/openspace/backgroundupload/QueueController.kt index d6437a26..65fde3dd 100644 --- a/android/src/main/java/ai/openspace/backgroundupload/QueueController.kt +++ b/android/src/main/java/ai/openspace/backgroundupload/QueueController.kt @@ -47,6 +47,16 @@ class QueueController( @Volatile private var repairDelayMs = REPAIR_FIRST_MS + /** + * The last WorkManager call of a pass timed out, and no pass has + * scheduled since. While a repair is also pending, a new pass makes no + * WorkManager call: the pending repair has not read the store yet (its + * task clears [repairPending] first), so it finds the entries this pass + * saved. A repair that schedules with no failure clears it. + */ + @Volatile + private var workManagerTimedOut = false + // MARK: - enqueue /** [repair]: the entry was queued already (a same-id enqueue of the same body). */ @@ -326,7 +336,7 @@ class QueueController( val idle = queued.filter { !longWait(it) && it.id !in clearedIds } val waiting = queued.filter { longWait(it) && it.id !in clearedIds } if (idle.isNotEmpty() || waiting.isNotEmpty()) { - val rows = runCatching { scheduler.rowSummary() }.onFailure(::failed).getOrNull() + val rows = if (pass.isStopped) null else runCatching { scheduler.rowSummary() }.onFailure(::failed).getOrNull() idle.filter { rows == null || it.id !in rows.startable } .forEach { e -> pass.schedule { scheduleRun(e, probe = probe) } } // A wait that stays needs its wake. KEEP leaves one that exists, so @@ -715,10 +725,22 @@ class QueueController( } // Read before the cleanup below changes it: this is what an earlier // process left in WorkManager. - val runs = runCatching { scheduler.libraryRuns() }.getOrNull() + // A timeout here stops the pass's scheduling too, so the sweep waits + // for WorkManager once, not twice. + val pass = SchedulePass("boot sweep") + val runs = if (pass.isStopped) { + null + } else { + try { + scheduler.libraryRuns() + } catch (error: Exception) { + // Any other failure only leaves the record without its run counts, as before. + if (isTimeout(error)) pass.failed(error) + null + } + } toForget.forEach { scheduler.cancel(it) } val probe = NetworkProbe() - val pass = SchedulePass("boot sweep") toSchedule.forEach { e -> pass.schedule { scheduleRun(e, keepWake = true, probe = probe) } } // Every start, even an idle one: a v9 leftover count of zero is a value // the app reports. It replaces a snapshot JS has not taken yet, so @@ -796,18 +818,30 @@ class QueueController( * for the pass, and asks for [repairRows] later, which starts the entries * the pass left with no row. */ - private inner class SchedulePass(val what: String) { + private inner class SchedulePass(val what: String, isRepair: Boolean = false) { private var failures = 0 private var skipped = 0 private var first: Throwable? = null - private var stopped = false + + /** + * True from the start while an earlier timeout is not cleared and its + * repair is pending: this pass leaves its entries to that repair. + */ + private val leftToRepair = !isRepair && workManagerTimedOut && repairPending.get() + private var stopped = leftToRepair val anyFailed get() = failures + skipped > 0 + /** No WorkManager call is made in this pass any more. */ + val isStopped get() = stopped + fun failed(error: Throwable) { failures++ if (first == null) first = error - if (isTimeout(error)) stopped = true + if (isTimeout(error)) { + stopped = true + workManagerTimedOut = true + } } fun schedule(block: () -> Unit) { @@ -824,6 +858,11 @@ class QueueController( fun finish(repair: Boolean = true) { if (!anyFailed) return + // Left to a pending repair: the timeout is recorded already. + if (leftToRepair && failures == 0) { + if (repair) requestRepair() + return + } val tail = if (stopped) "; stopped at a WorkManager timeout with $skipped left" else "" Diag.error("$what: $failures entry saves or WorkManager calls failed$tail; the queue tries again", first) if (repair) requestRepair() @@ -854,7 +893,7 @@ class QueueController( fun repairRows() { val now = clock() val entries = store.all().filter { !it.legacy && (it.state == EntryState.QUEUED || it.state == EntryState.AWAITING_AUTH) } - val pass = SchedulePass("repair") + val pass = SchedulePass("repair", isRepair = true) if (entries.isNotEmpty()) { val rows = try { scheduler.rowSummary() @@ -876,6 +915,7 @@ class QueueController( } } repairDelayMs = if (pass.anyFailed) minOf(repairDelayMs * 2, REPAIR_MAX_MS) else REPAIR_FIRST_MS + if (!pass.anyFailed) workManagerTimedOut = false pass.finish() } diff --git a/android/src/test/java/ai/openspace/backgroundupload/NetworkDoorbellTest.kt b/android/src/test/java/ai/openspace/backgroundupload/NetworkDoorbellTest.kt index bdfb14fb..66b37963 100644 --- a/android/src/test/java/ai/openspace/backgroundupload/NetworkDoorbellTest.kt +++ b/android/src/test/java/ai/openspace/backgroundupload/NetworkDoorbellTest.kt @@ -393,6 +393,47 @@ class NetworkDoorbellTest { assertEquals(emptyList Unit>>(), repairs) } + @Test + fun `five enqueues while WorkManager does not answer wait for it once, and the repair starts all five`() { + val c = controllerWithRepairs() + network(RequiredNetwork.CONNECTED) + work.unfinishedTimeouts = 1_000 + work.libraryTimeouts = 1_000 + for (n in 1..5) assertEquals("q$n", c.enqueue(parsed(id = "q$n"))) + assertEquals(1, work.timedOutQueries) + assertEquals(1, repairs.size) + // WorkManager answers again; the pending repair reads every queued entry. + work.unfinishedTimeouts = 0 + work.libraryTimeouts = 0 + repairs.removeAt(0).second() + for (n in 1..5) assertEquals("q$n", 1, startable("q$n").size) + // The repair cleared the timeout: the next enqueue schedules at once. + c.enqueue(parsed(id = "q6")) + assertEquals(1, startable("q6").size) + assertEquals(emptyList Unit>>(), repairs) + } + + @Test + fun `the boot sweep waits for a WorkManager that does not answer once, not twice`() { + val file = File(tmp.newFolder(), "diagnostics.json") + Diag.log = DiagnosticLog(file) + try { + val c = controllerWithRepairs() + for (n in 1..3) store.save(entry(id = "q$n")) + network(RequiredNetwork.CONNECTED) + work.unfinishedTimeouts = 1_000 + work.libraryTimeouts = 1_000 + c.sweep() + assertEquals(1, work.timedOutQueries) + val records = DiagnosticLog(file).drain() + assertEquals(true, records.single { it.message == BootSnapshot.MESSAGE }.data!!["runsUnavailable"]) + assertEquals(1, records.count { it.level == DiagnosticLog.ERROR }) + assertEquals(1, repairs.size) + } finally { + Diag.log = null + } + } + // MARK: - the fake follows WorkManager 2.12.0 @Test diff --git a/android/src/test/java/ai/openspace/backgroundupload/TestSupport.kt b/android/src/test/java/ai/openspace/backgroundupload/TestSupport.kt index 8faa9098..c9d28f2a 100644 --- a/android/src/test/java/ai/openspace/backgroundupload/TestSupport.kt +++ b/android/src/test/java/ai/openspace/backgroundupload/TestSupport.kt @@ -306,7 +306,15 @@ internal class FakeWorkManager(var now: Long = 10_000L) : WorkRows { /** How many unfinished(name) queries were made. */ var nameQueries = 0 + /** How many of the next library() queries time out, as a WorkManager that does not answer. They count in [timedOutQueries]. */ + var libraryTimeouts = 0 + override fun library(states: List): List { + if (libraryTimeouts > 0) { + libraryTimeouts-- + timedOutQueries++ + throw java.util.concurrent.TimeoutException("WorkManager did not answer") + } libraryQueries += states return rows.filter { it.state in states }.map(::view) } From 33c9fda41e412f2ee6e401d323d14444e6bdb512 Mon Sep 17 00:00:00 2001 From: Dylan Murphy Date: Wed, 18 Nov 2026 13:24:18 -0500 Subject: [PATCH 12/12] Test that a repair clears the timeout, and drop a guard no path needs Two mutations survived the previous commit: the repair pass never skips, because its task clears the pending flag first, so its extra guard did nothing; and no test showed the timeout being cleared. The guard is removed (the order is now the tested rule), and the test adds a later failure that is not a timeout: passes meanwhile still schedule. Co-Authored-By: Claude Fable 5.2 --- .../ai/openspace/backgroundupload/QueueController.kt | 10 ++++++---- .../openspace/backgroundupload/NetworkDoorbellTest.kt | 7 +++++++ .../java/ai/openspace/backgroundupload/TestSupport.kt | 7 +++++++ 3 files changed, 20 insertions(+), 4 deletions(-) diff --git a/android/src/main/java/ai/openspace/backgroundupload/QueueController.kt b/android/src/main/java/ai/openspace/backgroundupload/QueueController.kt index 65fde3dd..009c5ddd 100644 --- a/android/src/main/java/ai/openspace/backgroundupload/QueueController.kt +++ b/android/src/main/java/ai/openspace/backgroundupload/QueueController.kt @@ -818,16 +818,18 @@ class QueueController( * for the pass, and asks for [repairRows] later, which starts the entries * the pass left with no row. */ - private inner class SchedulePass(val what: String, isRepair: Boolean = false) { + private inner class SchedulePass(val what: String) { private var failures = 0 private var skipped = 0 private var first: Throwable? = null /** * True from the start while an earlier timeout is not cleared and its - * repair is pending: this pass leaves its entries to that repair. + * repair is pending: this pass leaves its entries to that repair. The + * repair's own pass never is: its task clears [repairPending] before + * [repairRows] starts. */ - private val leftToRepair = !isRepair && workManagerTimedOut && repairPending.get() + private val leftToRepair = workManagerTimedOut && repairPending.get() private var stopped = leftToRepair val anyFailed get() = failures + skipped > 0 @@ -893,7 +895,7 @@ class QueueController( fun repairRows() { val now = clock() val entries = store.all().filter { !it.legacy && (it.state == EntryState.QUEUED || it.state == EntryState.AWAITING_AUTH) } - val pass = SchedulePass("repair", isRepair = true) + val pass = SchedulePass("repair") if (entries.isNotEmpty()) { val rows = try { scheduler.rowSummary() diff --git a/android/src/test/java/ai/openspace/backgroundupload/NetworkDoorbellTest.kt b/android/src/test/java/ai/openspace/backgroundupload/NetworkDoorbellTest.kt index 66b37963..3eb1bf3d 100644 --- a/android/src/test/java/ai/openspace/backgroundupload/NetworkDoorbellTest.kt +++ b/android/src/test/java/ai/openspace/backgroundupload/NetworkDoorbellTest.kt @@ -411,6 +411,13 @@ class NetworkDoorbellTest { c.enqueue(parsed(id = "q6")) assertEquals(1, startable("q6").size) assertEquals(emptyList Unit>>(), repairs) + // A later failure that is not a timeout asks for a repair, and the + // passes meanwhile still schedule: no stale timeout makes them wait. + work.unfinishedErrors = 1 + c.enqueue(parsed(id = "q7")) + assertEquals(1, repairs.size) + c.enqueue(parsed(id = "q8")) + assertEquals(1, startable("q8").size) } @Test diff --git a/android/src/test/java/ai/openspace/backgroundupload/TestSupport.kt b/android/src/test/java/ai/openspace/backgroundupload/TestSupport.kt index c9d28f2a..b4923d3e 100644 --- a/android/src/test/java/ai/openspace/backgroundupload/TestSupport.kt +++ b/android/src/test/java/ai/openspace/backgroundupload/TestSupport.kt @@ -290,12 +290,19 @@ internal class FakeWorkManager(var now: Long = 10_000L) : WorkRows { /** How many unfinished(name) queries timed out. */ var timedOutQueries = 0 + /** How many of the next unfinished(name) queries fail with an error that is not a timeout. */ + var unfinishedErrors = 0 + override fun unfinished(name: String): List { if (unfinishedTimeouts > 0) { unfinishedTimeouts-- timedOutQueries++ throw java.util.concurrent.TimeoutException("WorkManager did not answer") } + if (unfinishedErrors > 0) { + unfinishedErrors-- + throw IllegalStateException("WorkManager failed") + } nameQueries++ return open(name).map(::view) }