diff --git a/CHANGELOG.md b/CHANGELOG.md index 574795a4..63dc4886 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,26 @@ +## Unreleased + +Android now sets its HTTP timeouts: connect 15 s, read 60 s, write 60 s. +Before, only the 24 h call cap was set, so OkHttp's 10 s defaults applied. +A server that took more than 10 s to answer after the body was sent made the +phone give up while the server still stored the request. The phone then sent +it again, and the server could store it twice. The 24 h call cap is +unchanged. + +Behavior change: the read timeout now allows 60 s of silence after the +body, before the attempt fails as a transport failure. A dead connection is +found after 60 s, not 10 s, and holds its transfer slot for that time. The +connect timeout is now 15 s per address, not 10 s. + +Android now keeps a response it finished reading before a stop (a system +stop, or `pause()`) took effect. Before, the response was dropped and the +request was sent again (at once after a system stop, at `resume()` after a +pause). Now the response goes through the usual rules: an accepted response +settles the entry, and an accepted chunked part keeps its accepted flag. A +response that has not been read when the stop lands is still lost, and the +request is still sent again. `cancel()` is unchanged: a cancelled entry +still settles `cancelled`. + ## 11.3.0 Added `useRequests(filter?)` and `useRequestsWithProgress(filter?)`, React diff --git a/android/src/main/java/ai/openspace/backgroundupload/ChunkedUploadWorker.kt b/android/src/main/java/ai/openspace/backgroundupload/ChunkedUploadWorker.kt index d236ec9a..574be011 100644 --- a/android/src/main/java/ai/openspace/backgroundupload/ChunkedUploadWorker.kt +++ b/android/src/main/java/ai/openspace/backgroundupload/ChunkedUploadWorker.kt @@ -90,6 +90,8 @@ internal class ChunkedTransfer(private val run: EntryRun) { ) when (val r = a.result) { is EntryRun.AttemptResult.Accepted -> { + // No suspension before this save: the part may have landed as the + // run was stopped (see awaitLanded). markAccepted(index, a.entry.descriptor!!.parts!![index]) return } diff --git a/android/src/main/java/ai/openspace/backgroundupload/EntryRun.kt b/android/src/main/java/ai/openspace/backgroundupload/EntryRun.kt index 27301058..bcdd45f3 100644 --- a/android/src/main/java/ai/openspace/backgroundupload/EntryRun.kt +++ b/android/src/main/java/ai/openspace/backgroundupload/EntryRun.kt @@ -280,6 +280,11 @@ internal class EntryRun( return Attempt(entry, url, policy, result) } + // The response can arrive on a stopped job (see awaitLanded). From here to + // ops.settle in run(), or to markAccepted for a part, nothing may suspend: + // a suspension on a stopped job throws, and the response is lost. The + // "lands as the run is stopped" tests in EntryRunTest pin this. + // A part counts on its own, so each part gets a rule's full maxRetries. val hits = (if (part != null) part.transientHitsByRule else entry.transientHitsByRule) ?: emptyMap() var verdict = RetryClassifier.classifyResponse( diff --git a/android/src/main/java/ai/openspace/backgroundupload/UploadTransport.kt b/android/src/main/java/ai/openspace/backgroundupload/UploadTransport.kt index 879a2208..273aef96 100644 --- a/android/src/main/java/ai/openspace/backgroundupload/UploadTransport.kt +++ b/android/src/main/java/ai/openspace/backgroundupload/UploadTransport.kt @@ -9,12 +9,32 @@ import java.util.concurrent.TimeUnit // chunked workers share them. Thus "requests transmitting at one time" means // one thing across all uploads. -// Max total time for a single request to complete -// This is 24hrs so plenty of time for large uploads -// Worst case is the time maxes out and the upload gets restarted. -// Not using unlimited time to prevent unexpected behaviors. -private const val REQUEST_TIMEOUT = 24L -private val REQUEST_TIMEOUT_UNIT = TimeUnit.HOURS +// OkHttp's defaults for the three timeouts below are 10 s each. 10 s of +// silence is too short: a server that takes longer to answer still stores +// the request, the phone sends it again, and the server can store it twice. + +// The time to open a TCP connection to one address. OkHttp tries the next +// address of the host when one does not answer only after this time, so a +// host with several addresses that do not answer (for example a broken IPv6 +// path) holds a transfer slot for that many times this value. Nothing was sent yet, so a connect timeout cannot make a duplicate. +// That is why it is shorter than the read and write timeouts. +private const val CONNECT_TIMEOUT_SECONDS = 15L + +// The longest silence while reading. This includes the wait for the server's +// answer after the last body byte. 60 s matches iOS's default request timeout. +// A dead connection is found after this time, and holds its transfer slot +// until then. +private const val READ_TIMEOUT_SECONDS = 60L + +// The longest a single write of the body may block. +private const val WRITE_TIMEOUT_SECONDS = 60L + +// A cap on the whole call: DNS, connect, the body, the server's work, and the +// response. OkHttp counts any retries it makes on its own inside the same cap. +// It is not a silence limit; the read and write timeouts above are. 24 h lets +// a large body finish on a slow network and still ends a call that never +// finishes. When it expires, the attempt fails and the retry rules apply. +private const val CALL_TIMEOUT_HOURS = 24L // The number of requests transmitting at one time across ALL uploads, chunked // parts included. This is the design's library-wide cap of 4. Every request, @@ -53,5 +73,8 @@ internal val transferGate = TransferGate(MAX_TRANSFER_CONCURRENCY, HIGH_PRIORITY // Use Okhttp as it provides the most standard behaviors even though it's not coroutine friendly internal val uploadHttpClient = OkHttpClient.Builder() - .callTimeout(REQUEST_TIMEOUT, REQUEST_TIMEOUT_UNIT) + .connectTimeout(CONNECT_TIMEOUT_SECONDS, TimeUnit.SECONDS) + .readTimeout(READ_TIMEOUT_SECONDS, TimeUnit.SECONDS) + .writeTimeout(WRITE_TIMEOUT_SECONDS, TimeUnit.SECONDS) + .callTimeout(CALL_TIMEOUT_HOURS, TimeUnit.HOURS) .build() diff --git a/android/src/main/java/ai/openspace/backgroundupload/UploadUtils.kt b/android/src/main/java/ai/openspace/backgroundupload/UploadUtils.kt index e17c37ec..0cd4a626 100644 --- a/android/src/main/java/ai/openspace/backgroundupload/UploadUtils.kt +++ b/android/src/main/java/ai/openspace/backgroundupload/UploadUtils.kt @@ -1,5 +1,6 @@ package ai.openspace.backgroundupload +import kotlinx.coroutines.CancellationException import kotlinx.coroutines.suspendCancellableCoroutine import okhttp3.Call import okhttp3.Callback @@ -18,7 +19,7 @@ import okio.buffer import java.io.File import java.io.IOException import java.io.RandomAccessFile -import kotlin.coroutines.resumeWithException +import java.util.concurrent.atomic.AtomicReference // Throttling interval of the raw progress callback. ProgressThrottle limits // the JS events above this. @@ -97,38 +98,84 @@ fun rangeRequestBody(file: File, start: Long, end: Long): RequestBody = object : } } -private suspend fun awaitResponse(client: OkHttpClient, request: Request): UploadResponse = - suspendCancellableCoroutine { continuation -> - val call = client.newCall(request) - continuation.invokeOnCancellation { call.cancel() } - call.enqueue(object : Callback { - override fun onFailure(call: Call, e: IOException) = - continuation.resumeWithException(e) - - override fun onResponse(call: Call, response: Response) { - val result = try { - response.use { res -> // close the response asap - // The body unchanged: an empty body stays empty. A substituted - // reason phrase would make accept `bodyIncludes` rules match text - // the server never sent. The cap applies while it streams in. - val body = res.body?.let { - BodyCap.read(it.source(), BodyCap.SETTLED_MAX_BYTES, it.contentType()?.charset() ?: Charsets.UTF_8) +private suspend fun awaitResponse(client: OkHttpClient, request: Request): UploadResponse { + val call = client.newCall(request) + return awaitLanded( + register = { onResult -> + call.enqueue(object : Callback { + override fun onFailure(call: Call, e: IOException) = onResult(Result.failure(e)) + + override fun onResponse(call: Call, response: Response) { + val result = try { + response.use { res -> // close the response asap + // The body unchanged: an empty body stays empty. A substituted + // reason phrase would make accept `bodyIncludes` rules match text + // the server never sent. The cap applies while it streams in. + val body = res.body?.let { + BodyCap.read(it.source(), BodyCap.SETTLED_MAX_BYTES, it.contentType()?.charset() ?: Charsets.UTF_8) + } + UploadResponse( + res.code, + body?.text.orEmpty(), + res.headers.toMultimap().mapValues { it.value.joinToString(", ") }, + body?.truncated ?: false, + ) } - UploadResponse( - res.code, - body?.text.orEmpty(), - res.headers.toMultimap().mapValues { it.value.joinToString(", ") }, - body?.truncated ?: false, - ) + } catch (e: IOException) { + onResult(Result.failure(e)) + return } - } catch (e: IOException) { - continuation.resumeWithException(e) - return + onResult(Result.success(result)) } - continuation.resumeWith(Result.success(result)) + }) + }, + cancel = { call.cancel() }, + ) +} + +/** + * Waits for one callback result and keeps a result that landed before a stop. + * + * [register] starts the work and gets the callback for its result. [cancel] + * stops the work. A stop (WorkManager stops the worker, pause() or cancel() + * cancels its work) with no result cancels the work and throws. + * + * Why: a plain suspendCancellableCoroutine drops a value when the stop comes + * after the resume and before the coroutine runs again. Its docs say it "will + * not resume successfully, even if CancellableContinuation.resume was already + * invoked". For an upload, that throws away a response OkHttp already read, + * and the request is sent again (at once after a system stop, at resume() + * after a pause). So the result is saved before the resume, and a stop + * returns it when it is there. A response not yet read when the stop lands + * is still lost: the stop cancels the call. + * + * A failure that landed first is thrown as it is (kotlinx delivers it). + * + * The caller's job is then cancelled, but plain code still runs. Each caller + * must reach its save (ops.settle for a simple entry, markAccepted for a part) + * with no suspension in between, because a suspension on a cancelled job + * throws. EntryRunTest pins this for both callers. + */ +internal suspend fun awaitLanded( + register: (onResult: (Result) -> Unit) -> Unit, + cancel: () -> Unit, +): T { + val landed = AtomicReference(null) + try { + return suspendCancellableCoroutine { continuation -> + // Runs at a stop, also after a resume that waits for dispatch. A cancel + // after the response was read and closed does nothing. + continuation.invokeOnCancellation { cancel() } + register { result -> + result.getOrNull()?.let { landed.set(it) } + continuation.resumeWith(result) } - }) + } + } catch (stop: CancellationException) { + landed.get()?.let { return it } + throw stop } +} private fun throttled(onProgress: (Long) -> Unit): (Long) -> Unit { var lastProgressReport = 0L diff --git a/android/src/main/java/ai/openspace/backgroundupload/UploadWorker.kt b/android/src/main/java/ai/openspace/backgroundupload/UploadWorker.kt index a49f7548..468899f2 100644 --- a/android/src/main/java/ai/openspace/backgroundupload/UploadWorker.kt +++ b/android/src/main/java/ai/openspace/backgroundupload/UploadWorker.kt @@ -39,6 +39,8 @@ internal class SimpleTransfer(private val run: EntryRun) { fileExists = { file == null || file.exists() }, ) when (val r = a.result) { + // No suspension between here and ops.settle: the response may have + // landed as the run was stopped (see awaitLanded). is EntryRun.AttemptResult.Accepted -> return Settlement.Completed(r.response, a.url, a.method) is EntryRun.AttemptResult.Auth -> { if (!r.reissue) throw EntryRun.ParkException(r.headerGeneration) diff --git a/android/src/test/java/ai/openspace/backgroundupload/AwaitResponseTest.kt b/android/src/test/java/ai/openspace/backgroundupload/AwaitResponseTest.kt new file mode 100644 index 00000000..2cf30c4d --- /dev/null +++ b/android/src/test/java/ai/openspace/backgroundupload/AwaitResponseTest.kt @@ -0,0 +1,216 @@ +package ai.openspace.backgroundupload + +import kotlinx.coroutines.CancellationException +import kotlinx.coroutines.CoroutineDispatcher +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.CoroutineStart +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.Job +import kotlinx.coroutines.currentCoroutineContext +import kotlinx.coroutines.launch +import kotlinx.coroutines.runBlocking +import kotlinx.coroutines.withContext +import kotlinx.coroutines.withTimeout +import okhttp3.ConnectionPool +import okhttp3.Dispatcher +import okhttp3.OkHttpClient +import okhttp3.RequestBody.Companion.toRequestBody +import org.junit.After +import org.junit.Assert.assertEquals +import org.junit.Assert.assertNull +import org.junit.Assert.assertTrue +import org.junit.Test +import java.io.IOException +import java.net.ServerSocket +import java.net.Socket +import java.util.concurrent.ConcurrentLinkedQueue +import java.util.concurrent.CountDownLatch +import java.util.concurrent.TimeUnit +import java.util.concurrent.atomic.AtomicInteger +import kotlin.concurrent.thread +import kotlin.coroutines.CoroutineContext + +/** + * A plain HTTP/1.1 server on a local socket. It reads one request at a time + * on a connection, waits [delayMs], and answers 200 "ok". With [truncate] it + * promises 10 body bytes, sends 2, and closes. + */ +private class TinyServer(private val delayMs: Long = 0, private val truncate: Boolean = false) : AutoCloseable { + private val server = ServerSocket(0) + val port: Int get() = server.localPort + val connections = AtomicInteger(0) + val requests = AtomicInteger(0) + + @Volatile + var answered = CountDownLatch(1) + + init { + thread(isDaemon = true) { + while (true) { + val s = try { server.accept() } catch (e: IOException) { return@thread } + connections.incrementAndGet() + thread(isDaemon = true) { serve(s) } + } + } + } + + private fun serve(s: Socket) { + try { + val input = s.getInputStream() + val out = s.getOutputStream() + while (true) { + val head = StringBuilder() + while (!head.endsWith("\r\n\r\n")) { + val b = input.read() + if (b < 0) return + head.append(b.toChar()) + } + var left = Regex("(?i)content-length: (\\d+)").find(head)?.groupValues?.get(1)?.toInt() ?: 0 + val buf = ByteArray(8192) + while (left > 0) { + val n = input.read(buf, 0, minOf(left, buf.size)) + if (n < 0) return + left -= n + } + requests.incrementAndGet() + if (delayMs > 0) Thread.sleep(delayMs) + val length = if (truncate) 10 else 2 + out.write("HTTP/1.1 200 OK\r\nContent-Length: $length\r\nContent-Type: text/plain\r\n\r\nok".toByteArray()) + out.flush() + answered.countDown() + if (truncate) return + } + } catch (e: IOException) { + // The client went away. + } finally { + runCatching { s.close() } + } + } + + override fun close() = server.close() +} + +/** A dispatcher that only queues. The test runs the tasks by hand, so it decides when the coroutine runs again. */ +private class ManualDispatcher : CoroutineDispatcher() { + private val queue = ConcurrentLinkedQueue() + + override fun dispatch(context: CoroutineContext, block: Runnable) { + queue += block + } + + fun pump() { + while (true) (queue.poll() ?: return).run() + } + + /** Waits until another thread queued a task: the OkHttp callback resumed the coroutine. */ + fun awaitTask(ms: Long = 10_000) { + val end = System.currentTimeMillis() + ms + while (queue.isEmpty()) { + check(System.currentTimeMillis() < end) { "no task queued" } + Thread.sleep(5) + } + } +} + +/** awaitResponse (through okhttpSend) against real OkHttp and a local socket server. */ +class AwaitResponseTest { + // The library's client with its own dispatcher and pool, so the counts here are this test's own. + private val client: OkHttpClient = uploadHttpClient.newBuilder() + .dispatcher(Dispatcher()) + .connectionPool(ConnectionPool()) + .build() + + @After + fun tearDown() { + client.dispatcher.cancelAll() + client.connectionPool.evictAll() + } + + private fun request(port: Int) = TransferRequest( + "http://127.0.0.1:$port/items", "POST", mapOf("Content-Type" to "application/json"), + """{"a":1}""".toRequestBody(null), + ) + + @Test + fun `a landed response survives a stop, and the connection is reused`() { + TinyServer().use { server -> + val dispatcher = ManualDispatcher() + var got: UploadResponse? = null + val job = CoroutineScope(dispatcher + Job()).launch { + got = okhttpSend(client, request(server.port)) {} + } + dispatcher.pump() // runs to the suspension in awaitLanded; the call is enqueued + assertTrue(server.answered.await(10, TimeUnit.SECONDS)) + dispatcher.awaitTask() // OkHttp read the response and resumed; the coroutine waits to run + job.cancel() // the stop, before the coroutine runs again + dispatcher.pump() + assertEquals(200, got?.code) + assertEquals("ok", got?.body) + assertEquals(1, server.requests.get()) + + // The call.cancel() from the stop ran after the response was read and + // closed, so the pooled connection still works. + val again = runBlocking { okhttpSend(client, request(server.port)) {} } + assertEquals(200, again.code) + assertEquals(2, server.requests.get()) + assertEquals(1, server.connections.get()) + } + } + + @Test + fun `a stop with no response cancels the call and throws the stop`() { + TinyServer(delayMs = 30_000).use { server -> + var thrown: Throwable? = null + runBlocking { + val job = launch(Dispatchers.IO) { + try { okhttpSend(client, request(server.port)) {} } catch (e: Throwable) { thrown = e; throw e } + } + withContext(Dispatchers.IO) { while (server.requests.get() == 0) Thread.sleep(10) } + job.cancel() + withTimeout(5_000) { job.join() } + } + assertTrue(thrown is CancellationException) + // call.cancel() ended the call: OkHttp runs nothing for it. + val end = System.currentTimeMillis() + 5_000 + while (client.dispatcher.runningCallsCount() > 0 && System.currentTimeMillis() < end) Thread.sleep(10) + assertEquals(0, client.dispatcher.runningCallsCount()) + } + } + + @Test + fun `a stop before the call starts sends nothing`() { + TinyServer().use { server -> + var thrown: Throwable? = null + runBlocking { + launch(Dispatchers.IO, start = CoroutineStart.UNDISPATCHED) { + currentCoroutineContext()[Job]!!.cancel() + try { okhttpSend(client, request(server.port)) {} } catch (e: Throwable) { thrown = e; throw e } + }.join() + } + Thread.sleep(300) + assertTrue(thrown is CancellationException) + assertEquals(0, server.requests.get()) + assertEquals(0, server.connections.get()) + } + } + + @Test + fun `a body that fails to read is not saved as landed`() { + TinyServer(truncate = true).use { server -> + val dispatcher = ManualDispatcher() + var got: UploadResponse? = null + var thrown: Throwable? = null + val job = CoroutineScope(dispatcher + Job()).launch { + try { got = okhttpSend(client, request(server.port)) {} } catch (e: Throwable) { thrown = e } + } + dispatcher.pump() + assertTrue(server.answered.await(10, TimeUnit.SECONDS)) + dispatcher.awaitTask() // the read failed and resumed the coroutine with it + job.cancel() + dispatcher.pump() + // The failure comes through as a transport failure, not as a response. + assertNull(got) + assertTrue("expected an IOException, got $thrown", thrown is IOException) + } + } +} diff --git a/android/src/test/java/ai/openspace/backgroundupload/EntryRunTest.kt b/android/src/test/java/ai/openspace/backgroundupload/EntryRunTest.kt index 82172bf7..72c06764 100644 --- a/android/src/test/java/ai/openspace/backgroundupload/EntryRunTest.kt +++ b/android/src/test/java/ai/openspace/backgroundupload/EntryRunTest.kt @@ -1,6 +1,10 @@ package ai.openspace.backgroundupload import kotlinx.coroutines.CancellationException +import kotlinx.coroutines.Job +import kotlinx.coroutines.currentCoroutineContext +import kotlinx.coroutines.ensureActive +import kotlinx.coroutines.launch import kotlinx.coroutines.runBlocking import org.junit.Assert.assertEquals import org.junit.Assert.assertFalse @@ -34,6 +38,8 @@ internal class FakeHost(var clock: Long = 10_000L) : TransferHost { sleeps += ms clock += ms onSleep() + // A real delay throws here when the run's job was cancelled. + currentCoroutineContext().ensureActive() } override suspend fun send(request: TransferRequest, onProgress: (Long) -> Unit): UploadResponse { @@ -367,6 +373,57 @@ class EntryRunTest { assertEquals("end:false", host.progress.last()) } + /** + * Runs e1 in its own job. [stopOn] decides, per request, whether the send + * stops that job (as WorkManager stops a worker) and then still returns + * its response: what awaitLanded does when a response landed before the stop. + */ + private fun runStoppedAt(code: Int = 200, stopOn: (TransferRequest) -> Boolean) = runBlocking { + var job: Job? = null + host.handler = { r, _ -> + if (stopOn(r)) job!!.cancel() + UploadResponse(code, "ok", mapOf()) + } + job = launch { EntryRun("e1", store, ops, host).run() } + job.join() + } + + @Test + fun `a response that lands as the run is stopped still settles completed`() { + // Pins that nothing suspends between the send's return and ops.settle. + // A suspension there throws on a stopped job, and the entry would go + // back to queued and be sent again. + controller.enqueue(parsed()) + runStoppedAt { true } + assertEquals(EntryState.COMPLETED, store.load("e1")!!.state) + assertEquals(EventJournal.KIND_COMPLETED, outcome().kind) + assertEquals(listOf("completed"), host.attempts.map { it.outcome }) + } + + @Test + fun `a 500 that lands at a stop is counted once and the entry goes back to queued`() { + // The backoff sleep throws on the stopped job, as a real delay does. + controller.enqueue(parsed()) + runStoppedAt(code = 500) { true } + val e = store.load("e1")!! + assertEquals(EntryState.QUEUED, e.state) + assertEquals(1, e.attempts) + assertEquals(1, host.requests.size) + assertEquals(listOf("error"), host.attempts.map { it.outcome }) + assertEquals(emptyList(), journal.unacknowledged()) + } + + @Test + fun `a part that lands as the run is stopped keeps its accepted flag`() { + // The same pin for a part: nothing suspends between the send's return and markAccepted. + chunked() + runStoppedAt { it.url.endsWith("start=0") } + val e = store.load("e1")!! + assertEquals(EntryState.QUEUED, e.state) + assertEquals(listOf(true, false, false), e.descriptor!!.parts!!.map { it.accepted }) + assertEquals(emptyList(), journal.unacknowledged()) + } + @Test fun `a timeout stop takes one more backoff step instead of restarting at once`() { controller.enqueue(parsed()) diff --git a/android/src/test/java/ai/openspace/backgroundupload/UploadTransportTest.kt b/android/src/test/java/ai/openspace/backgroundupload/UploadTransportTest.kt new file mode 100644 index 00000000..491f886f --- /dev/null +++ b/android/src/test/java/ai/openspace/backgroundupload/UploadTransportTest.kt @@ -0,0 +1,142 @@ +package ai.openspace.backgroundupload + +import kotlinx.coroutines.CancellationException +import kotlinx.coroutines.CoroutineStart +import kotlinx.coroutines.Job +import kotlinx.coroutines.currentCoroutineContext +import kotlinx.coroutines.launch +import kotlinx.coroutines.runBlocking +import org.junit.Assert.assertEquals +import org.junit.Assert.assertFalse +import org.junit.Assert.assertNotNull +import org.junit.Assert.assertNull +import org.junit.Assert.assertSame +import org.junit.Assert.assertTrue +import org.junit.Test +import java.io.IOException +import java.util.concurrent.TimeUnit + +class UploadHttpClientTest { + @Test + fun `the client sets connect 15 s, read 60 s, write 60 s, and keeps the 24 h call cap`() { + assertEquals(15_000, uploadHttpClient.connectTimeoutMillis) + assertEquals(60_000, uploadHttpClient.readTimeoutMillis) + assertEquals(60_000, uploadHttpClient.writeTimeoutMillis) + assertEquals(TimeUnit.HOURS.toMillis(24).toInt(), uploadHttpClient.callTimeoutMillis) + } +} + +class AwaitLandedTest { + private val response = UploadResponse(201, "created", mapOf()) + + /** A fake call: the test delivers the result by hand and sees whether the call was cancelled. */ + private class FakeCall { + var onResult: ((Result) -> Unit)? = null + var cancels = 0 + + suspend fun await(): UploadResponse = + awaitLanded(register = { onResult = it }, cancel = { cancels++ }) + } + + @Test + fun `a result with no stop is returned`() = runBlocking { + val call = FakeCall() + var got: UploadResponse? = null + val job = launch(start = CoroutineStart.UNDISPATCHED) { got = call.await() } + call.onResult!!(Result.success(response)) + job.join() + assertSame(response, got) + assertEquals(0, call.cancels) + } + + @Test + fun `a failure with no stop is thrown`() = runBlocking { + val call = FakeCall() + var thrown: Throwable? = null + val job = launch(start = CoroutineStart.UNDISPATCHED) { + try { call.await() } catch (e: IOException) { thrown = e } + } + call.onResult!!(Result.failure(IOException("reset"))) + job.join() + assertEquals("reset", thrown!!.message) + } + + @Test + fun `a result that lands before a stop is returned`() = runBlocking { + val call = FakeCall() + var got: UploadResponse? = null + val job = launch(start = CoroutineStart.UNDISPATCHED) { got = call.await() } + // The response lands, and the stop arrives before the coroutine runs again. + // This is the window where a plain suspendCancellableCoroutine drops the value. + call.onResult!!(Result.success(response)) + job.cancel() + job.join() + assertSame(response, got) + } + + @Test + fun `a stop with no result cancels the call and throws`() = runBlocking { + val call = FakeCall() + var thrown: Throwable? = null + var got: UploadResponse? = null + val job = launch(start = CoroutineStart.UNDISPATCHED) { + try { got = call.await() } catch (e: CancellationException) { thrown = e; throw e } + } + job.cancel() + job.join() + assertEquals(1, call.cancels) + assertTrue(thrown is CancellationException) + assertNull(got) + } + + @Test + fun `a stop before the call starts cancels it at once`() = runBlocking { + val call = FakeCall() + var thrown: Throwable? = null + val job = launch(start = CoroutineStart.UNDISPATCHED) { + currentCoroutineContext()[Job]!!.cancel() + try { call.await() } catch (e: CancellationException) { thrown = e; throw e } + } + job.join() + assertNotNull(thrown) + assertEquals(1, call.cancels) + } + + @Test + fun `a failure that lands before a stop is thrown as it is`() = runBlocking { + // Unchanged behavior: kotlinx delivers an exception that landed first. + // The attempt then counts a transport failure, and its backoff stops the run. + val call = FakeCall() + var thrown: Throwable? = null + val job = launch(start = CoroutineStart.UNDISPATCHED) { + try { call.await() } catch (e: IOException) { thrown = e } + } + call.onResult!!(Result.failure(IOException("reset"))) + job.cancel() + job.join() + assertEquals("reset", thrown!!.message) + assertFalse(job.isActive) + } +} + +class TransferGateStopTest { + @Test + fun `the gate returns the block's value on a stopped job, in both lanes`() = runBlocking { + // EntryWorker sends inside transferGate.withPermit. A response that + // landed as the run stopped must come out of the gate: nothing in it may + // suspend after the block. + for (high in listOf(false, true)) { + val gate = TransferGate(total = 4, reserved = 1) + var got: String? = null + val job = launch { + got = gate.withPermit(high) { + currentCoroutineContext()[Job]!!.cancel() + "landed" + } + } + job.join() + assertEquals("landed", got) + assertEquals(4, gate.availableSlots) + } + } +}