From 15dbca3c7711a1a253f1d04a54b48042a1a42d5a Mon Sep 17 00:00:00 2001 From: Dylan Murphy Date: Tue, 17 Nov 2026 16:38:06 -0500 Subject: [PATCH 1/4] iOS: keep the first attempt's result when it arrives after a re-send After a relaunch, the library waits 10 s for the result of a task that finished while the app was not running. When none came, it deleted the task's key and sent the request again as a new attempt. A late result of the first attempt was then dropped, a 2xx included, and the second send ran to the end. Now the grace marks the old keys superseded instead of deleting them. A late accepted result completes the entry with the first attempt's response, and settle cancels the new attempt's task. A late failure or cancel of the old attempt is dropped and does not disturb the new one. settle (and the relaunch repair of a lost settle) removes an entry's superseded keys of older attempts, so the task map does not grow. Co-Authored-By: Claude Fable 5.2 --- CHANGELOG.md | 21 +++ README.md | 6 +- ios/QueueCoordinator+Outcomes.swift | 15 +++ ios/QueueCoordinator+Reconcile.swift | 19 ++- ios/QueueCoordinator+Simple.swift | 11 +- ios/Tests/CoordinatorRelaunchTests.swift | 161 +++++++++++++++++++++-- 6 files changed, 212 insertions(+), 21 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 574795a4..9aa9b867 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,24 @@ +## Unreleased + +Behavior change (iOS): a late 2xx now completes an entry that earlier +versions left running and sent again. After a relaunch, iOS waits 10 s for +the result of an upload that finished while the app was not running. When +no result came in that time, earlier versions sent the request again as a +new attempt and dropped the first attempt's result if it arrived later. So +a first attempt that the server accepted was always followed by a second +send. Now the library keeps the first attempt's task key, marked as +replaced. When that result arrives later and is accepted (a 2xx, or a +status the definition's `accept` rules allow), the entry completes with the first attempt's response and the second attempt's +task is cancelled. If the second task had not started yet, nothing is sent +twice. If it already sent its request, the server still gets two requests, +as before. A late failure or cancel of the first attempt is ignored, and +the second attempt goes on. A first attempt's result that arrives after +the entry already settled is ignored, so the app gets one outcome. The +kept keys are removed when the entry settles. The `attempt` event of the +late result names the first attempt and its `X-Request-Id`; the settled +outcome's `attempts` and `requestId` still name the last attempt. Android +and chunked parts are unchanged. + ## 11.3.0 Added `useRequests(filter?)` and `useRequestsWithProgress(filter?)`, React diff --git a/README.md b/README.md index 525c964f..57e22803 100644 --- a/README.md +++ b/README.md @@ -282,7 +282,11 @@ above lets the library journal their outcomes. **Relaunch.** When the app comes back and an entry's task is gone but a completion may still be in flight from the daemon, the library waits up to 10 s for it before it sends again. This is what stops a request that finished -while the app was dead from being sent twice. +while the app was dead from being sent twice. On iOS, when that result comes +after the wait, it still counts: an accepted result completes the entry with +the first attempt's response and cancels the new attempt's task. A failure +that comes late is ignored, and the new attempt goes on. If the new task +already sent its request, the server gets the request twice. # Reliable delivery diff --git a/ios/QueueCoordinator+Outcomes.swift b/ios/QueueCoordinator+Outcomes.swift index 1e141a21..f51b7e82 100644 --- a/ios/QueueCoordinator+Outcomes.swift +++ b/ios/QueueCoordinator+Outcomes.swift @@ -63,6 +63,7 @@ extension QueueCoordinator { journaled = false } cancelTasks(id, purpose: .superseded) + removeOlderSupersededKeys(e) chunked.stop(id) emitProgress(id, sent: e.bytesSent, total: e.totalBytes) e.settledEventId = event.eventId @@ -108,12 +109,26 @@ extension QueueCoordinator { n.nextAttemptAt = nil n.authParked = false cancelTasks(e.id, purpose: .superseded) + removeOlderSupersededKeys(e) chunked.stop(e.id) disarmExpiry(e.id) commit(n, emit: false) return n } + /// A simple entry keeps the keys of an older attempt that the relaunch + /// grace gave up on, so a late result can still complete it (withoutTask). + /// Once the entry settles, such a result is dropped anyway, so the keys + /// go. Keys of the current attempt stay: a task cancelled at this settle + /// reads its purpose in its cancel callback, which removes the key. + func removeOlderSupersededKeys(_ e: QueueEntry) { + guard !e.isChunked else { return } + taskMap.removeAll { _, m in + m.id == e.id && m.partIndex == nil && m.purpose == .superseded && m.generation == e.generation + && (m.attempt.map { $0 < e.attempts } ?? false) + } + } + /// Relaunch, before any task is matched: every live row whose own /// generation already has an unacked outcome takes that outcome, so it is /// never sent again. diff --git a/ios/QueueCoordinator+Reconcile.swift b/ios/QueueCoordinator+Reconcile.swift index b9d3252b..7787b975 100644 --- a/ios/QueueCoordinator+Reconcile.swift +++ b/ios/QueueCoordinator+Reconcile.swift @@ -182,13 +182,18 @@ extension QueueCoordinator { guard let self, let current = self.index.entry(e.id), current.state == e.state, current.generation == e.generation, current.attempts == e.attempts, !self.liveTasks.values.contains(where: { $0.id == e.id }) else { return } - // No replay moved the entry: the task is lost. Its key would send - // every later launch through this wait again. - self.taskMap.removeAll { _, m in - m.id == e.id && m.generation == e.generation && m.attempt == e.attempts - } - // It may have run, so the next attempt gets a new ordinal: a late - // replay of this one is then dropped as stale. + // No replay came in time. The task may be lost, or its result may + // still arrive later. Mark its keys superseded instead of deleting + // them. A late accepted result then completes the entry and cancels + // the new attempt's task, and a late failure or cancel is dropped + // (taskCompletedLocked). The wait covers only the current attempt, so + // once the re-issue below moves the entry to a new attempt, a later + // launch does not wait for these keys again. settle removes them. + let stale = self.taskMap.keys(where: { + $0.id == e.id && $0.generation == e.generation && $0.attempt == e.attempts + }) + self.taskMap.setPurpose(.superseded, for: stale.map { (key: $0, id: e.id) }) + // It may have run, so the next attempt gets a new ordinal and request id. self.reissue(current, advanceAttempt: true) } } diff --git a/ios/QueueCoordinator+Simple.swift b/ios/QueueCoordinator+Simple.swift index 032a119a..37cf6208 100644 --- a/ios/QueueCoordinator+Simple.swift +++ b/ios/QueueCoordinator+Simple.swift @@ -178,10 +178,13 @@ extension QueueCoordinator { if cancelled, meta?.purpose == .pause || meta?.purpose == .superseded { return } let accepted = c.error == nil && c.statusCode.map { UploadOutcome.isAccepted($0, body: c.body, accept: e.accept) } == true - // A replaced task that finished before its cancel took effect. Its - // replacement drives the entry, unless this one landed while the entry - // is live. The server has the body then, so the entry completes, even - // when the replacement failed and the entry moved to a later attempt. + // A replaced task that finished before its cancel took effect, or an + // older attempt whose result came after the relaunch grace gave up on it + // (withoutTask). The newer task drives the entry, unless this one landed + // while the entry is live. The server has the body then, so the entry + // completes and settle cancels the newer task, even when the entry moved + // to a later attempt. A failure is dropped: it must not disturb the + // newer attempt. if meta?.purpose == .superseded && (!accepted || !e.isLive) { return } // A delayed task that ended before its begin date, with no response and // no body byte sent, never reached the daemon: nsurlsessiond refused the diff --git a/ios/Tests/CoordinatorRelaunchTests.swift b/ios/Tests/CoordinatorRelaunchTests.swift index e0e7fecf..6faba09a 100644 --- a/ios/Tests/CoordinatorRelaunchTests.swift +++ b/ios/Tests/CoordinatorRelaunchTests.swift @@ -64,7 +64,7 @@ final class CoordinatorRelaunchTests: XCTestCase { XCTAssertTrue(fresh.transport.created.isEmpty, "no duplicate request") } - func testRunningWithALostTaskReissuesAfterTheGraceAndPrunesItsKey() throws { + func testRunningWithALostTaskReissuesAfterTheGraceAndKeepsItsKeyForALateResult() throws { h.boot() _ = try h.enqueue(h.dataRaw(id: "a")).get() let lost = h.transport.live[0] @@ -73,12 +73,20 @@ final class CoordinatorRelaunchTests: XCTestCase { XCTAssertTrue(fresh.transport.created.isEmpty) fresh.advance(Double(QueueCoordinator.graceMs)) XCTAssertEqual(fresh.transport.created.count, 1) - XCTAssertNil(fresh.map.meta(forKey: lost.key)) - XCTAssertTrue(fresh.map.keys(where: { $0.id == "a" && $0.attempt == 1 }).isEmpty, "\(lost.key) pruned") - // The next launch does not wait the grace again for the same lost task. + XCTAssertEqual(fresh.map.meta(forKey: lost.key)?.purpose, .superseded, "kept, so a late result can win") + let retry = try XCTUnwrap(fresh.transport.live.first) + // The next launch does not wait the grace again for the same lost task: + // the wait covers only attempt 2, so its replay ends the wait at once. let third = Harness(root: h.root) - third.boot() - XCTAssertTrue(third.map.keys(where: { $0.attempt == 1 }).isEmpty) + var released = false + third.coordinator.reconcileAll { released = true } + third.drain() + third.drain() + XCTAssertFalse(released) + third.complete(FakeTask(key: retry.key, description: retry.taskDescription, request: retry.request)) + XCTAssertTrue(released, "attempt 1's key does not hold the wait") + XCTAssertEqual(third.entry("a")?.state, .completed) + XCTAssertTrue(third.map.keys(where: { $0.id == "a" }).isEmpty, "the settle removes the kept key") } func testReconcileDropsKeysWhoseIdHasNoEntry() throws { @@ -111,7 +119,8 @@ final class CoordinatorRelaunchTests: XCTestCase { } // The key says the delayed task may have run while the app was dead: wait - // for its replay, then mint a new attempt so a late replay is stale. + // for its replay, then mint a new attempt. A late accepted replay of the + // old attempt still completes the entry. func testQueuedEntryWithAPendingKeyWaitsTheGraceThenMintsANewAttempt() throws { h.boot() _ = try h.enqueue(h.dataRaw(id: "a")).get() @@ -124,10 +133,144 @@ final class CoordinatorRelaunchTests: XCTestCase { let task = try XCTUnwrap(fresh.transport.live.first) XCTAssertEqual(fresh.entry("a")?.attempts, 3) XCTAssertNotEqual(task.header("X-Request-Id"), waiting.header("X-Request-Id")) - XCTAssertTrue(fresh.map.keys(where: { $0.id == "a" && $0.attempt == 2 }).isEmpty, "the lost task's key is pruned") - // A late replay of the lost attempt is dropped. + XCTAssertEqual(fresh.map.meta(forKey: waiting.key)?.purpose, .superseded, "the old key is kept, marked") + // A late accepted replay of the old attempt wins over the new one. fresh.complete(FakeTask(key: waiting.key, description: waiting.taskDescription, request: waiting.request)) + XCTAssertEqual(fresh.entry("a")?.state, .completed) + XCTAssertTrue(task.cancelled) + XCTAssertEqual(fresh.sink.settled.count, 1) + } + + // MARK: - A late result wins + + /// Attempt 1's task finished while the app was dead, and its callback is + /// late. The grace ends, attempt 2 (task B) is issued, and then attempt 1's + /// result arrives. + private func reissuedAfterTheGrace() throws -> (fresh: Harness, original: FakeTask, b: FakeTask) { + h.boot() + _ = try h.enqueue(h.dataRaw(id: "a")).get() + let original = h.transport.live[0] + let fresh = Harness(root: h.root) + fresh.boot() + fresh.advance(Double(QueueCoordinator.graceMs)) + let b = try XCTUnwrap(fresh.transport.live.first) + XCTAssertEqual(fresh.entry("a")?.attempts, 2) + return (fresh, original, b) + } + + private func replay(_ task: FakeTask) -> FakeTask { + FakeTask(key: task.key, description: task.taskDescription, request: task.request) + } + + func testALateReplayAfterAReissueCompletesTheEntryAndStopsTheNewTask() throws { + let (fresh, original, b) = try reissuedAfterTheGrace() + fresh.complete(replay(original), status: 201, body: #"{"id":7}"#) + XCTAssertEqual(fresh.entry("a")?.state, .completed) + XCTAssertTrue(b.cancelled, "the second send stops") + XCTAssertEqual(fresh.sink.settled.count, 1) + let response = fresh.sink.settled.first?["response"] as? [String: Any] + XCTAssertEqual(response?["status"] as? Int, 201, "the app gets the first attempt's response") + XCTAssertEqual(response?["body"] as? String, #"{"id":7}"#) + XCTAssertEqual(fresh.sink.settled.first?["attempts"] as? Int, 2, "the outcome counts both attempts") + XCTAssertEqual(fresh.sink.attempts.last?["attempt"] as? Int, 1) + XCTAssertEqual(fresh.sink.attempts.last?["requestId"] as? String, original.header("X-Request-Id")) + // B's cancel callback adds nothing. + fresh.deliverCancel(b) + XCTAssertEqual(fresh.sink.settled.count, 1) + XCTAssertEqual(fresh.transport.created.count, 1, "no third send") + } + + func testALateFailureAfterAReissueLeavesTheNewAttemptAlone() throws { + let (fresh, original, b) = try reissuedAfterTheGrace() + let attemptsBefore = fresh.sink.attempts.count + fresh.complete(replay(original), status: 500) + fresh.complete(replay(original), error: NSError(domain: NSURLErrorDomain, code: NSURLErrorTimedOut)) + fresh.deliverCancel(replay(original)) XCTAssertEqual(fresh.entry("a")?.state, .running) + XCTAssertEqual(fresh.entry("a")?.attempts, 2) + XCTAssertFalse(b.cancelled) + XCTAssertEqual(fresh.transport.created.count, 1, "no retry is scheduled for the old attempt") + XCTAssertEqual(fresh.sink.attempts.count, attemptsBefore, "no event for the old attempt") + XCTAssertTrue(fresh.sink.settled.isEmpty) + fresh.complete(b, status: 201) + XCTAssertEqual(fresh.entry("a")?.state, .completed) + XCTAssertEqual(fresh.sink.settled.count, 1) + } + + func testALate2xxAfterTheNewAttemptCompletedIsDropped() throws { + let (fresh, original, b) = try reissuedAfterTheGrace() + fresh.complete(b, status: 200) + XCTAssertEqual(fresh.entry("a")?.state, .completed) + let attemptsBefore = fresh.sink.attempts.count + fresh.complete(replay(original), status: 201) + XCTAssertEqual(fresh.entry("a")?.state, .completed) + XCTAssertEqual(fresh.sink.settled.count, 1, "one outcome") + XCTAssertEqual(fresh.sink.attempts.count, attemptsBefore) + XCTAssertTrue(fresh.transport.live.isEmpty) + } + + func testTheSupersededKeysAreRemovedWhenTheEntrySettles() throws { + let (fresh, original, b) = try reissuedAfterTheGrace() + XCTAssertEqual(fresh.map.meta(forKey: original.key)?.purpose, .superseded, "kept for a late result") + fresh.complete(b, status: 400) + XCTAssertEqual(fresh.entry("a")?.state, .error) + XCTAssertNil(fresh.map.meta(forKey: original.key)) + XCTAssertTrue(fresh.map.keys(where: { $0.id == "a" }).isEmpty, "nothing left for the entry") + } + + /// The settle's journal write landed and its entry save did not, and the + /// process died before the kept key was removed. The relaunch repair of + /// the lost settle removes it. + func testTheRepairOfALostSettleRemovesTheSupersededKeys() throws { + let (fresh, original, b) = try reissuedAfterTheGrace() + let kept = try XCTUnwrap(fresh.map.meta(forKey: original.key)) + let dir = fresh.store.dir("a") + setReadOnly(dir, true) + fresh.complete(b, status: 201) + setReadOnly(dir, false) + XCTAssertEqual(fresh.store.load("a")?.state, .running, "the disk still says running") + fresh.map.set(kept, forKey: original.key) // the removal did not land + let third = Harness(root: h.root) + third.boot() + XCTAssertEqual(third.entry("a")?.state, .completed) + XCTAssertNil(third.map.meta(forKey: original.key)) + XCTAssertTrue(third.transport.created.isEmpty) + } + + /// A second relaunch between the re-issue and the late result. Attempt + /// 1's task is listed as finished (not live): reconcile neither adopts nor + /// cancels it, and its result still completes the entry. + func testALateReplayAfterASecondRelaunchStillCompletesTheEntry() throws { + let (_, original, b) = try reissuedAfterTheGrace() + let listed = replay(original) + listed.isLive = false + let survivor = replay(b) + let third = Harness(root: h.root) + third.relaunch(daemonTasks: [listed, survivor]) + third.boot() + XCTAssertFalse(listed.cancelled) + XCTAssertTrue(third.transport.created.isEmpty, "B is adopted") + XCTAssertEqual(third.map.meta(forKey: original.key)?.purpose, .superseded) + third.complete(listed, status: 201) + XCTAssertEqual(third.entry("a")?.state, .completed) + XCTAssertTrue(survivor.cancelled) + XCTAssertEqual(third.sink.settled.count, 1) + } + + /// Same, but B is lost too: the second grace issues attempt 3, and attempt + /// 1's late result still wins over it. + func testALateReplayAfterTwoReissuesCompletesTheEntry() throws { + let (_, original, _) = try reissuedAfterTheGrace() + let third = Harness(root: h.root) + third.boot() + third.advance(Double(QueueCoordinator.graceMs)) + let c = try XCTUnwrap(third.transport.live.first) + XCTAssertEqual(third.entry("a")?.attempts, 3) + third.complete(replay(original), status: 201) + XCTAssertEqual(third.entry("a")?.state, .completed) + XCTAssertTrue(c.cancelled) + XCTAssertEqual(third.sink.settled.count, 1) + XCTAssertTrue(third.map.keys(where: { $0.id == "a" && $0.purpose == .superseded && $0.attempt ?? 0 < 3 }).isEmpty) } // MARK: - Review fixes: lost saves, early completions, pending replays From bb4022a83f8cc087562d134348b5a217b6b98ff4 Mon Sep 17 00:00:00 2001 From: Dylan Murphy Date: Tue, 17 Nov 2026 17:00:07 -0500 Subject: [PATCH 2/4] iOS: give kept late-result keys their own purpose and name the winning request The relaunch grace kept the old attempt's key as superseded, and settle removed only keys of the same generation with a lower attempt. A key stayed when the entry settled inside the grace's own re-issue, or when a different body replaced the entry. A later use of the id, which starts again at generation 1, could then be completed by the old late result. The settle cleanup also removed superseded keys of tasks that reconcile had cancelled, so their cancel callback could reach a later use of the id as a system cancel and send it twice. Now a kept key has its own purpose, lateResult. It is handled like superseded when a late result is evaluated. Reconcile turns it into superseded when it finds the task live, because the task then owes a cancel callback. Every lateResult key of an id is removed when the id settles, is forgotten (ack or cancel of a settled row), or gets a different body. Keys of other purposes stay until their callback. A simple request's outcome now carries the request id of the attempt whose result settled it, as the chunked path already does. The late-result tests move to CoordinatorLateResultTests. Each late failure kind gets its own kept key. Co-Authored-By: Claude Fable 5.2 --- ios/QueueCoordinator+Enqueue.swift | 2 + ios/QueueCoordinator+Outcomes.swift | 25 +- ios/QueueCoordinator+Reconcile.swift | 25 +- ios/QueueCoordinator+Simple.swift | 22 +- ios/TaskMap.swift | 12 + ios/Tests/CoordinatorLateResultTests.swift | 398 +++++++++++++++++++++ ios/Tests/CoordinatorRelaunchTests.swift | 136 +------ 7 files changed, 456 insertions(+), 164 deletions(-) create mode 100644 ios/Tests/CoordinatorLateResultTests.swift diff --git a/ios/QueueCoordinator+Enqueue.swift b/ios/QueueCoordinator+Enqueue.swift index 94941cce..e3210aa9 100644 --- a/ios/QueueCoordinator+Enqueue.swift +++ b/ios/QueueCoordinator+Enqueue.swift @@ -167,6 +167,8 @@ extension QueueCoordinator { n.headerGeneration = settings.headerGeneration n.state = paused ? .paused : .queued try saveOrDiscard(n, staged: staged) + // A late result of the old body must not complete the new one. + removeLateResultKeys(p.id) store.removeV9Manifest(p.id) store.sweep(n) // deletes the old body publish(n) diff --git a/ios/QueueCoordinator+Outcomes.swift b/ios/QueueCoordinator+Outcomes.swift index f51b7e82..42e7d946 100644 --- a/ios/QueueCoordinator+Outcomes.swift +++ b/ios/QueueCoordinator+Outcomes.swift @@ -63,7 +63,7 @@ extension QueueCoordinator { journaled = false } cancelTasks(id, purpose: .superseded) - removeOlderSupersededKeys(e) + removeLateResultKeys(id) chunked.stop(id) emitProgress(id, sent: e.bytesSent, total: e.totalBytes) e.settledEventId = event.eventId @@ -109,24 +109,21 @@ extension QueueCoordinator { n.nextAttemptAt = nil n.authParked = false cancelTasks(e.id, purpose: .superseded) - removeOlderSupersededKeys(e) + removeLateResultKeys(e.id) chunked.stop(e.id) disarmExpiry(e.id) commit(n, emit: false) return n } - /// A simple entry keeps the keys of an older attempt that the relaunch - /// grace gave up on, so a late result can still complete it (withoutTask). - /// Once the entry settles, such a result is dropped anyway, so the keys - /// go. Keys of the current attempt stay: a task cancelled at this settle - /// reads its purpose in its cancel callback, which removes the key. - func removeOlderSupersededKeys(_ e: QueueEntry) { - guard !e.isChunked else { return } - taskMap.removeAll { _, m in - m.id == e.id && m.partIndex == nil && m.purpose == .superseded && m.generation == e.generation - && (m.attempt.map { $0 < e.attempts } ?? false) - } + /// Removes every late-result key of `id` (see withoutTask), whatever its + /// attempt or generation. Called when the id settles, is forgotten, or + /// gets a different body: a late result can then no longer complete what + /// the id holds, and a kept key must not reach a later use of the id. No + /// task in this process holds such a key, so none owes a cancel callback + /// that would read it. Keys of other purposes stay until their callback. + func removeLateResultKeys(_ id: String) { + taskMap.removeAll { _, m in m.id == id && m.purpose == .lateResult } } /// Relaunch, before any task is matched: every live row whose own @@ -144,6 +141,7 @@ extension QueueCoordinator { /// unacked outcomes (cancel on a settled entry). func forget(_ id: String, dropEvents: Bool) { cancelTasks(id, purpose: .superseded) + removeLateResultKeys(id) chunked.stop(id) store.remove(id) index.remove(id) @@ -178,6 +176,7 @@ extension QueueCoordinator { } pendingJournal = pendingJournal.filter { $0.value.id != id } cancelTasks(id, purpose: .superseded) + removeLateResultKeys(id) chunked.stop(id) index.remove(id) disarmExpiry(id) diff --git a/ios/QueueCoordinator+Reconcile.swift b/ios/QueueCoordinator+Reconcile.swift index 7787b975..be9e1669 100644 --- a/ios/QueueCoordinator+Reconcile.swift +++ b/ios/QueueCoordinator+Reconcile.swift @@ -111,8 +111,12 @@ extension QueueCoordinator { let meta = taskMap.meta(forKey: task.key) // A task the library replaced or dropped, still live because the // process died before the daemon applied the cancel. Never adopt it: - // its replacement, or the re-issue below, drives the entry. - if meta?.purpose == .superseded { + // its replacement, or the re-issue below, drives the entry. A kept + // late-result key on a live task becomes superseded: the task now owes + // a cancel callback, so its key must stay until that callback, and a + // settle removes only late-result keys. + if let purpose = meta?.purpose, purpose.isOutdated { + if purpose == .lateResult { taskMap.setPurpose(.superseded, forKey: task.key, id: meta?.id ?? "") } task.cancel() continue } @@ -183,16 +187,19 @@ extension QueueCoordinator { current.generation == e.generation, current.attempts == e.attempts, !self.liveTasks.values.contains(where: { $0.id == e.id }) else { return } // No replay came in time. The task may be lost, or its result may - // still arrive later. Mark its keys superseded instead of deleting - // them. A late accepted result then completes the entry and cancels - // the new attempt's task, and a late failure or cancel is dropped - // (taskCompletedLocked). The wait covers only the current attempt, so - // once the re-issue below moves the entry to a new attempt, a later - // launch does not wait for these keys again. settle removes them. + // still arrive later. Keep its keys as late-result keys instead of + // deleting them. A late accepted result then completes the entry and + // cancels the new attempt's task, and a late failure or cancel is + // dropped (taskCompletedLocked). The wait covers only the current + // attempt, so once the re-issue below moves the entry to a new + // attempt, a later launch does not wait for these keys again. A key + // already marked superseded or pause keeps its purpose: its task may + // still owe a cancel callback. let stale = self.taskMap.keys(where: { $0.id == e.id && $0.generation == e.generation && $0.attempt == e.attempts + && ($0.purpose ?? .attempt) == .attempt }) - self.taskMap.setPurpose(.superseded, for: stale.map { (key: $0, id: e.id) }) + self.taskMap.setPurpose(.lateResult, for: stale.map { (key: $0, id: e.id) }) // It may have run, so the next attempt gets a new ordinal and request id. self.reissue(current, advanceAttempt: true) } diff --git a/ios/QueueCoordinator+Simple.swift b/ios/QueueCoordinator+Simple.swift index 37cf6208..67ac7ccd 100644 --- a/ios/QueueCoordinator+Simple.swift +++ b/ios/QueueCoordinator+Simple.swift @@ -172,20 +172,26 @@ extension QueueCoordinator { // entry, or an older one the library replaced (see below). guard case .request(let id, let generation, let attempt) = owner, var e = index.entry(id), !e.legacy, !e.isChunked, e.generation == generation, - e.attempts == attempt || (meta?.purpose == .superseded && attempt < e.attempts) else { return } + e.attempts == attempt || (meta?.purpose?.isOutdated == true && attempt < e.attempts) else { return } let cancelled = RetryClassifier.isCancellation(c.error) - if cancelled, meta?.purpose == .pause || meta?.purpose == .superseded { return } + if cancelled, meta?.purpose == .pause || meta?.purpose?.isOutdated == true { return } let accepted = c.error == nil && c.statusCode.map { UploadOutcome.isAccepted($0, body: c.body, accept: e.accept) } == true // A replaced task that finished before its cancel took effect, or an // older attempt whose result came after the relaunch grace gave up on it - // (withoutTask). The newer task drives the entry, unless this one landed - // while the entry is live. The server has the body then, so the entry - // completes and settle cancels the newer task, even when the entry moved - // to a later attempt. A failure is dropped: it must not disturb the - // newer attempt. - if meta?.purpose == .superseded && (!accepted || !e.isLive) { return } + // (a late-result key, see withoutTask). The newer task drives the entry, + // unless this one landed while the entry is live. The server has the + // body then, so the entry completes and settle cancels the newer task, + // even when the entry moved to a later attempt. A failure is dropped: it + // must not disturb the newer attempt. + if meta?.purpose?.isOutdated == true && (!accepted || !e.isLive) { return } + // The outcome names the request whose response settles it. A late + // result belongs to an older attempt than the entry's last one. + if let requestId = meta?.requestId, requestId != e.lastRequestId { + e.lastRequestId = requestId + index.upsert(e) + } // A delayed task that ended before its begin date, with no response and // no body byte sent, never reached the daemon: nsurlsessiond refused the // connection (a simulator build with no entitlements does this). It is diff --git a/ios/TaskMap.swift b/ios/TaskMap.swift index 32e8d6f4..c97c4631 100644 --- a/ios/TaskMap.swift +++ b/ios/TaskMap.swift @@ -18,6 +18,18 @@ final class TaskMap { case attempt case pause case superseded + /// A simple task the relaunch grace gave up on: it was not live, and + /// no result came in the wait. The entry moved to a new attempt, and the + /// key stays so a late accepted result can still complete the entry. No + /// task in this process holds it, so it owes no cancel callback, and + /// every key of this purpose for an id is removed when that id settles, + /// is forgotten, or gets a different body. An older build reads it as + /// nil and drops the late result. + case lateResult + + /// A task that no longer drives its entry: its failure and its cancel + /// produce nothing, and only an accepted result on a live entry counts. + var isOutdated: Bool { self == .superseded || self == .lateResult } } struct Meta: Codable, Equatable { diff --git a/ios/Tests/CoordinatorLateResultTests.swift b/ios/Tests/CoordinatorLateResultTests.swift new file mode 100644 index 00000000..472e01fb --- /dev/null +++ b/ios/Tests/CoordinatorLateResultTests.swift @@ -0,0 +1,398 @@ +import XCTest +@testable import RNBGUCore + +/// After a relaunch, the grace for a simple entry with no live task ends +/// and the entry moves to a new attempt. The old attempt's key is kept as a +/// late-result key, so its late accepted result still completes the entry. +final class CoordinatorLateResultTests: XCTestCase { + private var h: Harness! + + override func setUp() { + h = Harness() + } + + override func tearDown() { + try? FileManager.default.removeItem(at: h.root) + } + + /// Attempt 1's task finished while the app was dead, and its callback is + /// late. The grace ends, attempt 2 (task B) is issued, and then attempt 1's + /// result arrives. + private func reissuedAfterTheGrace(extra: [String: Any] = [:]) throws + -> (fresh: Harness, original: FakeTask, b: FakeTask) { + h.boot() + _ = try h.enqueue(h.dataRaw(id: "a", extra: extra)).get() + let original = h.transport.live[0] + let fresh = Harness(root: h.root) + fresh.boot() + fresh.advance(Double(QueueCoordinator.graceMs)) + let b = try XCTUnwrap(fresh.transport.live.first) + XCTAssertEqual(fresh.entry("a")?.attempts, 2) + XCTAssertEqual(fresh.map.meta(forKey: original.key)?.purpose, .lateResult) + return (fresh, original, b) + } + + private func replay(_ task: FakeTask) -> FakeTask { + FakeTask(key: task.key, description: task.taskDescription, request: task.request) + } + + private func settledResponse(_ x: Harness) -> [String: Any]? { + x.sink.settled.last?["response"] as? [String: Any] + } + + // MARK: - A late accepted result wins + + func testALateReplayAfterAReissueCompletesTheEntryAndStopsTheNewTask() throws { + let (fresh, original, b) = try reissuedAfterTheGrace() + fresh.complete(replay(original), status: 201, body: #"{"id":7}"#) + XCTAssertEqual(fresh.entry("a")?.state, .completed) + XCTAssertTrue(b.cancelled, "the second send stops") + XCTAssertEqual(fresh.sink.settled.count, 1) + XCTAssertEqual(settledResponse(fresh)?["status"] as? Int, 201, "the app gets the first attempt's response") + XCTAssertEqual(settledResponse(fresh)?["body"] as? String, #"{"id":7}"#) + XCTAssertEqual(fresh.sink.settled.first?["attempts"] as? Int, 2, "the outcome counts both attempts") + XCTAssertEqual(fresh.sink.settled.first?["requestId"] as? String, original.header("X-Request-Id"), + "the request id of the response's own request") + XCTAssertEqual(fresh.sink.attempts.last?["attempt"] as? Int, 1) + XCTAssertEqual(fresh.sink.attempts.last?["requestId"] as? String, original.header("X-Request-Id")) + // B's cancel callback adds nothing. + fresh.deliverCancel(b) + XCTAssertEqual(fresh.sink.settled.count, 1) + XCTAssertEqual(fresh.transport.created.count, 1, "no third send") + XCTAssertTrue(fresh.map.keys(where: { $0.id == "a" }).isEmpty) + } + + /// A late 2xx while the new attempt waits out a backoff. The waiting task + /// never began, so the outcome must not name its request id. + func testALate2xxWhileTheNewAttemptWaitsInBackoffCompletesTheEntry() throws { + let (fresh, original, b) = try reissuedAfterTheGrace() + fresh.complete(b, status: 503) + let delayed = try XCTUnwrap(fresh.transport.live.first) + XCTAssertEqual(fresh.entry("a")?.state, .queued) + fresh.complete(replay(original), status: 201) + XCTAssertEqual(fresh.entry("a")?.state, .completed) + XCTAssertTrue(delayed.cancelled) + XCTAssertEqual(fresh.sink.settled.first?["attempts"] as? Int, 3) + XCTAssertEqual(fresh.sink.settled.first?["requestId"] as? String, original.header("X-Request-Id")) + XCTAssertNotEqual(fresh.sink.settled.first?["requestId"] as? String, delayed.header("X-Request-Id")) + fresh.deliverCancel(delayed) + fresh.advance(60_000) + XCTAssertEqual(fresh.transport.created.count, 2, "B and the delayed task only") + XCTAssertEqual(fresh.sink.settled.count, 1) + XCTAssertTrue(fresh.map.keys(where: { $0.id == "a" }).isEmpty) + } + + func testALate2xxWhilePausedCompletesTheEntry() throws { + let (fresh, original, b) = try reissuedAfterTheGrace() + fresh.pause() + XCTAssertEqual(fresh.entry("a")?.state, .paused) + fresh.complete(replay(original), status: 201) + XCTAssertEqual(fresh.entry("a")?.state, .completed) + XCTAssertEqual(fresh.sink.settled.count, 1) + fresh.deliverCancel(b) + fresh.resume() + XCTAssertEqual(fresh.entry("a")?.state, .completed) + XCTAssertEqual(fresh.transport.created.count, 1, "the resume sends nothing") + XCTAssertEqual(fresh.sink.settled.count, 1) + XCTAssertTrue(fresh.map.keys(where: { $0.id == "a" }).isEmpty) + } + + func testALate2xxWhileAwaitingAuthCompletesTheEntry() throws { + let (fresh, original, b) = try reissuedAfterTheGrace() + fresh.complete(b, status: 401) + XCTAssertEqual(fresh.entry("a")?.state, .awaitingAuth) + fresh.complete(replay(original), status: 201) + XCTAssertEqual(fresh.entry("a")?.state, .completed) + XCTAssertEqual(fresh.sink.settled.count, 1) + fresh.updateHeaders(["Authorization": "new"]) + XCTAssertEqual(fresh.transport.created.count, 1, "new headers send nothing") + } + + /// The late 2xx lands at the next launch, before its reconcile, while B is + /// still live in the daemon. Reconcile then cancels B. + func testALate2xxBeforeTheNextReconcileCompletesTheEntryAndReconcileStopsB() throws { + let (_, original, b) = try reissuedAfterTheGrace() + let survivor = replay(b) + let third = Harness(root: h.root) + third.transport.deferAllTasks = true + third.relaunch(daemonTasks: [survivor]) + var released = false + third.coordinator.reconcileAll { released = true } + third.drain() + third.complete(replay(original), status: 201) + XCTAssertEqual(third.entry("a")?.state, .completed) + third.transport.releaseAllTasks() + third.drain() + third.drain() + XCTAssertTrue(survivor.cancelled) + XCTAssertTrue(released) + XCTAssertTrue(third.transport.created.isEmpty) + third.deliverCancel(survivor) + XCTAssertEqual(third.sink.settled.count, 1) + XCTAssertTrue(third.map.keys(where: { $0.id == "a" }).isEmpty) + } + + /// A second relaunch between the re-issue and the late result. Attempt + /// 1's task is listed as finished (not live): reconcile neither adopts nor + /// cancels it, and its result still completes the entry. + func testALateReplayAfterASecondRelaunchStillCompletesTheEntry() throws { + let (_, original, b) = try reissuedAfterTheGrace() + let listed = replay(original) + listed.isLive = false + let survivor = replay(b) + let third = Harness(root: h.root) + third.relaunch(daemonTasks: [listed, survivor]) + third.boot() + XCTAssertFalse(listed.cancelled) + XCTAssertTrue(third.transport.created.isEmpty, "B is adopted") + XCTAssertEqual(third.map.meta(forKey: original.key)?.purpose, .lateResult) + third.complete(listed, status: 201) + XCTAssertEqual(third.entry("a")?.state, .completed) + XCTAssertTrue(survivor.cancelled) + XCTAssertEqual(third.sink.settled.count, 1) + } + + /// Same, but B is lost too: the second grace issues attempt 3, and attempt + /// 1's late result still wins over it. + func testALateReplayAfterTwoReissuesCompletesTheEntry() throws { + let (_, original, b) = try reissuedAfterTheGrace() + let third = Harness(root: h.root) + third.boot() + third.advance(Double(QueueCoordinator.graceMs)) + let c = try XCTUnwrap(third.transport.live.first) + XCTAssertEqual(third.entry("a")?.attempts, 3) + XCTAssertEqual(third.map.meta(forKey: b.key)?.purpose, .lateResult) + third.complete(replay(original), status: 201) + XCTAssertEqual(third.entry("a")?.state, .completed) + XCTAssertTrue(c.cancelled) + XCTAssertEqual(third.sink.settled.count, 1) + XCTAssertTrue(third.map.keys(where: { $0.id == "a" && $0.purpose == .lateResult }).isEmpty) + } + + // MARK: - A late failure or cancel is dropped + + /// Each late result gets its own kept key: the first delivery removes it. + private func assertALateResultLeavesTheNewAttemptAlone( + status: Int? = nil, error: NSError? = nil, file: StaticString = #filePath, line: UInt = #line) throws { + let (fresh, original, b) = try reissuedAfterTheGrace() + let attemptsBefore = fresh.sink.attempts.count + fresh.complete(replay(original), status: status, error: error) + XCTAssertEqual(fresh.entry("a")?.state, .running, file: file, line: line) + XCTAssertEqual(fresh.entry("a")?.attempts, 2, file: file, line: line) + XCTAssertFalse(b.cancelled, file: file, line: line) + XCTAssertEqual(fresh.transport.created.count, 1, "no retry for the old attempt", file: file, line: line) + XCTAssertEqual(fresh.sink.attempts.count, attemptsBefore, "no event for the old attempt", file: file, line: line) + XCTAssertTrue(fresh.sink.settled.isEmpty, file: file, line: line) + fresh.complete(b, status: 201) + XCTAssertEqual(fresh.entry("a")?.state, .completed, file: file, line: line) + XCTAssertEqual(fresh.sink.settled.count, 1, file: file, line: line) + XCTAssertEqual(fresh.sink.settled.first?["requestId"] as? String, b.header("X-Request-Id"), file: file, line: line) + } + + func testALate500AfterAReissueLeavesTheNewAttemptAlone() throws { + try assertALateResultLeavesTheNewAttemptAlone(status: 500) + } + + func testALateTransportErrorAfterAReissueLeavesTheNewAttemptAlone() throws { + try assertALateResultLeavesTheNewAttemptAlone( + error: NSError(domain: NSURLErrorDomain, code: NSURLErrorTimedOut)) + } + + func testALateCancelAfterAReissueLeavesTheNewAttemptAlone() throws { + try assertALateResultLeavesTheNewAttemptAlone( + error: NSError(domain: NSURLErrorDomain, code: NSURLErrorCancelled)) + } + + func testALate2xxAfterTheNewAttemptCompletedIsDropped() throws { + let (fresh, original, b) = try reissuedAfterTheGrace() + fresh.complete(b, status: 200) + XCTAssertEqual(fresh.entry("a")?.state, .completed) + let attemptsBefore = fresh.sink.attempts.count + fresh.complete(replay(original), status: 201) + XCTAssertEqual(fresh.entry("a")?.state, .completed) + XCTAssertEqual(fresh.sink.settled.count, 1, "one outcome") + XCTAssertEqual(fresh.sink.attempts.count, attemptsBefore) + XCTAssertTrue(fresh.transport.live.isEmpty) + } + + func testALate2xxAfterAUserCancelIsDropped() throws { + let (fresh, original, b) = try reissuedAfterTheGrace() + fresh.cancel("a") + XCTAssertEqual(fresh.entry("a")?.state, .cancelled) + XCTAssertNil(fresh.map.meta(forKey: original.key)) + let attemptsBefore = fresh.sink.attempts.count + fresh.complete(replay(original), status: 201) + fresh.deliverCancel(b) + XCTAssertEqual(fresh.entry("a")?.state, .cancelled) + XCTAssertEqual(fresh.sink.settled.count, 1, "the cancel only") + XCTAssertEqual(fresh.sink.attempts.count, attemptsBefore) + } + + // MARK: - When the kept keys go + + func testTheLateResultKeysAreRemovedWhenTheEntrySettles() throws { + let (fresh, original, b) = try reissuedAfterTheGrace() + fresh.complete(b, status: 400) + XCTAssertEqual(fresh.entry("a")?.state, .error) + XCTAssertNil(fresh.map.meta(forKey: original.key)) + XCTAssertTrue(fresh.map.keys(where: { $0.id == "a" }).isEmpty, "nothing left for the entry") + } + + /// The settle's journal write landed and its entry save did not, and the + /// process died before the kept key was removed. The relaunch repair of + /// the lost settle removes it. + func testTheRepairOfALostSettleRemovesTheLateResultKeys() throws { + let (fresh, original, b) = try reissuedAfterTheGrace() + let kept = try XCTUnwrap(fresh.map.meta(forKey: original.key)) + let dir = fresh.store.dir("a") + setReadOnly(dir, true) + fresh.complete(b, status: 201) + setReadOnly(dir, false) + XCTAssertEqual(fresh.store.load("a")?.state, .running, "the disk still says running") + fresh.map.set(kept, forKey: original.key) // the removal did not land + let third = Harness(root: h.root) + third.boot() + XCTAssertEqual(third.entry("a")?.state, .completed) + XCTAssertNil(third.map.meta(forKey: original.key)) + XCTAssertTrue(third.transport.created.isEmpty) + } + + /// A settled row whose kept key is still there, as if the removal at the + /// settle did not land. Forgetting the row removes it. + private func settledWithAKeptKey(status: Int) throws -> (Harness, FakeTask) { + let (fresh, original, b) = try reissuedAfterTheGrace() + let kept = try XCTUnwrap(fresh.map.meta(forKey: original.key)) + fresh.complete(b, status: status) + fresh.map.set(kept, forKey: original.key) + return (fresh, original) + } + + func testTheAckThatForgetsTheRowRemovesTheLateResultKeys() throws { + let (fresh, original) = try settledWithAKeptKey(status: 200) + fresh.ack([try XCTUnwrap(fresh.sink.settled.last?["eventId"] as? String)]) + XCTAssertNil(fresh.row("a")) + XCTAssertNil(fresh.map.meta(forKey: original.key)) + } + + func testCancelOfASettledRowRemovesTheLateResultKeys() throws { + let (fresh, original) = try settledWithAKeptKey(status: 400) + XCTAssertNil(fresh.cancel("a")) + XCTAssertNil(fresh.row("a")) + XCTAssertNil(fresh.map.meta(forKey: original.key)) + } + + /// The entry expires inside the grace's own re-issue, so it settles before + /// a new attempt is added: the kept key has the entry's current attempt. + /// It still goes, so a later use of the id cannot take its late result. + /// The later use is on attempt 2 here. On attempt 1 the task description + /// alone matches it, with or without a key: that needs a per-use value, + /// which this change does not add. + func testAnEntryThatSettlesInsideTheGraceReissueDropsItsKeptKey() throws { + h.boot() + _ = try h.enqueue(h.dataRaw(id: "a", extra: ["expiresAt": h.clock + 60_000])).get() + let original = h.transport.live[0] + let fresh = Harness(root: h.root) + fresh.clock = h.clock + 120_000 + fresh.boot() + fresh.advance(Double(QueueCoordinator.graceMs)) + XCTAssertEqual(fresh.entry("a")?.state, .error, "expired") + XCTAssertEqual(fresh.entry("a")?.attempts, 1) + XCTAssertNil(fresh.map.meta(forKey: original.key)) + // The app cancels the error row and uses the id again. + fresh.cancel("a") + _ = try fresh.enqueue(fresh.dataRaw(id: "a", data: ["x": 2])).get() + fresh.complete(try XCTUnwrap(fresh.transport.live.first), status: 503) + XCTAssertEqual(fresh.entry("a")?.attempts, 2) + let delayed = try XCTUnwrap(fresh.transport.live.first) + let settledBefore = fresh.sink.settled.count + fresh.complete(replay(original), status: 201, body: "STALE") + XCTAssertEqual(fresh.entry("a")?.state, .queued, "the new body still goes out") + XCTAssertFalse(delayed.cancelled) + XCTAssertEqual(fresh.sink.settled.count, settledBefore) + } + + /// A different body replaces the entry: the old body's late result must + /// not complete the new one. + private func assertADifferentBodyDropsTheKeptKey( + file: StaticString = #filePath, line: UInt = #line, _ setUp: (Harness, FakeTask) throws -> Void) throws { + let (fresh, original, b) = try reissuedAfterTheGrace() + try setUp(fresh, b) + XCTAssertNotEqual(fresh.entry("a")?.state, .running, file: file, line: line) + _ = try fresh.enqueue(fresh.dataRaw(id: "a", data: ["x": 2])).get() + XCTAssertEqual(fresh.entry("a")?.generation, 2, file: file, line: line) + XCTAssertNil(fresh.map.meta(forKey: original.key), file: file, line: line) + let stateBefore = fresh.entry("a")?.state + let live = fresh.transport.live + fresh.complete(replay(original), status: 201, body: "STALE") + XCTAssertEqual(fresh.entry("a")?.state, stateBefore, file: file, line: line) + XCTAssertTrue(live.allSatisfy { !$0.cancelled }, file: file, line: line) + XCTAssertTrue(fresh.sink.settled.isEmpty, file: file, line: line) + } + + func testADifferentBodyInBackoffDropsTheKeptKey() throws { + try assertADifferentBodyDropsTheKeptKey { fresh, b in fresh.complete(b, status: 503) } + } + + func testADifferentBodyWhilePausedDropsTheKeptKey() throws { + try assertADifferentBodyDropsTheKeptKey { fresh, b in + fresh.pause() + fresh.deliverCancel(b) + } + } + + func testADifferentBodyWhileAwaitingAuthDropsTheKeptKey() throws { + try assertADifferentBodyDropsTheKeptKey { fresh, b in fresh.complete(b, status: 401) } + } + + /// Three uses of one id. Use 1's attempt 1 is kept; a different body + /// replaces it (use 1, generation 2), which completes and is acked; use 3 + /// starts again at generation 1 and its first attempt fails. Use 1's late + /// 2xx must not complete use 3 with an old response. + func testALateResultFromAnEarlierUseOfTheIdDoesNotCompleteALaterUse() throws { + let (fresh, original, b) = try reissuedAfterTheGrace() + fresh.complete(b, status: 503) + _ = try fresh.enqueue(fresh.dataRaw(id: "a", data: ["x": 2])).get() + let e = try XCTUnwrap(fresh.transport.live.last) + fresh.complete(e, status: 200) + let eventId = try XCTUnwrap(fresh.sink.settled.last?["eventId"] as? String) + fresh.ack([eventId]) + XCTAssertNil(fresh.row("a")) + _ = try fresh.enqueue(fresh.dataRaw(id: "a", data: ["x": 3])).get() + let c = try XCTUnwrap(fresh.transport.live.last) + fresh.complete(c, status: 503) + XCTAssertEqual(fresh.entry("a")?.generation, 1) + XCTAssertEqual(fresh.entry("a")?.attempts, 2) + let delayed = try XCTUnwrap(fresh.transport.live.last) + let settledBefore = fresh.sink.settled.count + fresh.complete(replay(original), status: 201, body: "STALE-FROM-USE-1") + XCTAssertEqual(fresh.entry("a")?.state, .queued, "use 3 still sends its own body") + XCTAssertFalse(delayed.cancelled) + XCTAssertEqual(fresh.sink.settled.count, settledBefore) + } + + /// A task that reconcile found live under a kept key is cancelled. It + /// owes a cancel callback, so its key must outlive the settle and the ack. + /// Without it, after the id is used again, the old cancel would count as + /// a system cancel of the new use and send it twice. + func testACancelCallbackOfAReconcileCancelledTaskDoesNothingAfterTheIdIsUsedAgain() throws { + let (_, original, b) = try reissuedAfterTheGrace() + let old = replay(original) + let survivor = replay(b) + let third = Harness(root: h.root) + third.relaunch(daemonTasks: [old, survivor]) + third.boot() + XCTAssertTrue(old.cancelled) + XCTAssertFalse(survivor.cancelled, "B is adopted") + XCTAssertEqual(third.map.meta(forKey: old.key)?.purpose, .superseded, "it owes a cancel callback") + third.complete(survivor, status: 200) + XCTAssertEqual(third.map.meta(forKey: old.key)?.purpose, .superseded, "kept past the settle") + let eventId = try XCTUnwrap(third.sink.settled.last?["eventId"] as? String) + third.ack([eventId]) + _ = try third.enqueue(third.dataRaw(id: "a", data: ["x": 2])).get() + let created = third.transport.created.count + third.deliverCancel(old) + XCTAssertEqual(third.entry("a")?.state, .running) + XCTAssertEqual(third.entry("a")?.attempts, 1) + XCTAssertEqual(third.transport.created.count, created, "no second send") + } +} diff --git a/ios/Tests/CoordinatorRelaunchTests.swift b/ios/Tests/CoordinatorRelaunchTests.swift index 6faba09a..761a14ea 100644 --- a/ios/Tests/CoordinatorRelaunchTests.swift +++ b/ios/Tests/CoordinatorRelaunchTests.swift @@ -73,7 +73,7 @@ final class CoordinatorRelaunchTests: XCTestCase { XCTAssertTrue(fresh.transport.created.isEmpty) fresh.advance(Double(QueueCoordinator.graceMs)) XCTAssertEqual(fresh.transport.created.count, 1) - XCTAssertEqual(fresh.map.meta(forKey: lost.key)?.purpose, .superseded, "kept, so a late result can win") + XCTAssertEqual(fresh.map.meta(forKey: lost.key)?.purpose, .lateResult, "kept, so a late result can win") let retry = try XCTUnwrap(fresh.transport.live.first) // The next launch does not wait the grace again for the same lost task: // the wait covers only attempt 2, so its replay ends the wait at once. @@ -133,7 +133,7 @@ final class CoordinatorRelaunchTests: XCTestCase { let task = try XCTUnwrap(fresh.transport.live.first) XCTAssertEqual(fresh.entry("a")?.attempts, 3) XCTAssertNotEqual(task.header("X-Request-Id"), waiting.header("X-Request-Id")) - XCTAssertEqual(fresh.map.meta(forKey: waiting.key)?.purpose, .superseded, "the old key is kept, marked") + XCTAssertEqual(fresh.map.meta(forKey: waiting.key)?.purpose, .lateResult, "the old key is kept, marked") // A late accepted replay of the old attempt wins over the new one. fresh.complete(FakeTask(key: waiting.key, description: waiting.taskDescription, request: waiting.request)) XCTAssertEqual(fresh.entry("a")?.state, .completed) @@ -141,138 +141,6 @@ final class CoordinatorRelaunchTests: XCTestCase { XCTAssertEqual(fresh.sink.settled.count, 1) } - // MARK: - A late result wins - - /// Attempt 1's task finished while the app was dead, and its callback is - /// late. The grace ends, attempt 2 (task B) is issued, and then attempt 1's - /// result arrives. - private func reissuedAfterTheGrace() throws -> (fresh: Harness, original: FakeTask, b: FakeTask) { - h.boot() - _ = try h.enqueue(h.dataRaw(id: "a")).get() - let original = h.transport.live[0] - let fresh = Harness(root: h.root) - fresh.boot() - fresh.advance(Double(QueueCoordinator.graceMs)) - let b = try XCTUnwrap(fresh.transport.live.first) - XCTAssertEqual(fresh.entry("a")?.attempts, 2) - return (fresh, original, b) - } - - private func replay(_ task: FakeTask) -> FakeTask { - FakeTask(key: task.key, description: task.taskDescription, request: task.request) - } - - func testALateReplayAfterAReissueCompletesTheEntryAndStopsTheNewTask() throws { - let (fresh, original, b) = try reissuedAfterTheGrace() - fresh.complete(replay(original), status: 201, body: #"{"id":7}"#) - XCTAssertEqual(fresh.entry("a")?.state, .completed) - XCTAssertTrue(b.cancelled, "the second send stops") - XCTAssertEqual(fresh.sink.settled.count, 1) - let response = fresh.sink.settled.first?["response"] as? [String: Any] - XCTAssertEqual(response?["status"] as? Int, 201, "the app gets the first attempt's response") - XCTAssertEqual(response?["body"] as? String, #"{"id":7}"#) - XCTAssertEqual(fresh.sink.settled.first?["attempts"] as? Int, 2, "the outcome counts both attempts") - XCTAssertEqual(fresh.sink.attempts.last?["attempt"] as? Int, 1) - XCTAssertEqual(fresh.sink.attempts.last?["requestId"] as? String, original.header("X-Request-Id")) - // B's cancel callback adds nothing. - fresh.deliverCancel(b) - XCTAssertEqual(fresh.sink.settled.count, 1) - XCTAssertEqual(fresh.transport.created.count, 1, "no third send") - } - - func testALateFailureAfterAReissueLeavesTheNewAttemptAlone() throws { - let (fresh, original, b) = try reissuedAfterTheGrace() - let attemptsBefore = fresh.sink.attempts.count - fresh.complete(replay(original), status: 500) - fresh.complete(replay(original), error: NSError(domain: NSURLErrorDomain, code: NSURLErrorTimedOut)) - fresh.deliverCancel(replay(original)) - XCTAssertEqual(fresh.entry("a")?.state, .running) - XCTAssertEqual(fresh.entry("a")?.attempts, 2) - XCTAssertFalse(b.cancelled) - XCTAssertEqual(fresh.transport.created.count, 1, "no retry is scheduled for the old attempt") - XCTAssertEqual(fresh.sink.attempts.count, attemptsBefore, "no event for the old attempt") - XCTAssertTrue(fresh.sink.settled.isEmpty) - fresh.complete(b, status: 201) - XCTAssertEqual(fresh.entry("a")?.state, .completed) - XCTAssertEqual(fresh.sink.settled.count, 1) - } - - func testALate2xxAfterTheNewAttemptCompletedIsDropped() throws { - let (fresh, original, b) = try reissuedAfterTheGrace() - fresh.complete(b, status: 200) - XCTAssertEqual(fresh.entry("a")?.state, .completed) - let attemptsBefore = fresh.sink.attempts.count - fresh.complete(replay(original), status: 201) - XCTAssertEqual(fresh.entry("a")?.state, .completed) - XCTAssertEqual(fresh.sink.settled.count, 1, "one outcome") - XCTAssertEqual(fresh.sink.attempts.count, attemptsBefore) - XCTAssertTrue(fresh.transport.live.isEmpty) - } - - func testTheSupersededKeysAreRemovedWhenTheEntrySettles() throws { - let (fresh, original, b) = try reissuedAfterTheGrace() - XCTAssertEqual(fresh.map.meta(forKey: original.key)?.purpose, .superseded, "kept for a late result") - fresh.complete(b, status: 400) - XCTAssertEqual(fresh.entry("a")?.state, .error) - XCTAssertNil(fresh.map.meta(forKey: original.key)) - XCTAssertTrue(fresh.map.keys(where: { $0.id == "a" }).isEmpty, "nothing left for the entry") - } - - /// The settle's journal write landed and its entry save did not, and the - /// process died before the kept key was removed. The relaunch repair of - /// the lost settle removes it. - func testTheRepairOfALostSettleRemovesTheSupersededKeys() throws { - let (fresh, original, b) = try reissuedAfterTheGrace() - let kept = try XCTUnwrap(fresh.map.meta(forKey: original.key)) - let dir = fresh.store.dir("a") - setReadOnly(dir, true) - fresh.complete(b, status: 201) - setReadOnly(dir, false) - XCTAssertEqual(fresh.store.load("a")?.state, .running, "the disk still says running") - fresh.map.set(kept, forKey: original.key) // the removal did not land - let third = Harness(root: h.root) - third.boot() - XCTAssertEqual(third.entry("a")?.state, .completed) - XCTAssertNil(third.map.meta(forKey: original.key)) - XCTAssertTrue(third.transport.created.isEmpty) - } - - /// A second relaunch between the re-issue and the late result. Attempt - /// 1's task is listed as finished (not live): reconcile neither adopts nor - /// cancels it, and its result still completes the entry. - func testALateReplayAfterASecondRelaunchStillCompletesTheEntry() throws { - let (_, original, b) = try reissuedAfterTheGrace() - let listed = replay(original) - listed.isLive = false - let survivor = replay(b) - let third = Harness(root: h.root) - third.relaunch(daemonTasks: [listed, survivor]) - third.boot() - XCTAssertFalse(listed.cancelled) - XCTAssertTrue(third.transport.created.isEmpty, "B is adopted") - XCTAssertEqual(third.map.meta(forKey: original.key)?.purpose, .superseded) - third.complete(listed, status: 201) - XCTAssertEqual(third.entry("a")?.state, .completed) - XCTAssertTrue(survivor.cancelled) - XCTAssertEqual(third.sink.settled.count, 1) - } - - /// Same, but B is lost too: the second grace issues attempt 3, and attempt - /// 1's late result still wins over it. - func testALateReplayAfterTwoReissuesCompletesTheEntry() throws { - let (_, original, _) = try reissuedAfterTheGrace() - let third = Harness(root: h.root) - third.boot() - third.advance(Double(QueueCoordinator.graceMs)) - let c = try XCTUnwrap(third.transport.live.first) - XCTAssertEqual(third.entry("a")?.attempts, 3) - third.complete(replay(original), status: 201) - XCTAssertEqual(third.entry("a")?.state, .completed) - XCTAssertTrue(c.cancelled) - XCTAssertEqual(third.sink.settled.count, 1) - XCTAssertTrue(third.map.keys(where: { $0.id == "a" && $0.purpose == .superseded && $0.attempt ?? 0 < 3 }).isEmpty) - } - // MARK: - Review fixes: lost saves, early completions, pending replays /// The journal write landed, the entry.json save did not. The relaunch From 098120c8901fcf560f0f5f2bf9a5a43d9b1d4ad3 Mon Sep 17 00:00:00 2001 From: Dylan Murphy Date: Tue, 17 Nov 2026 17:00:12 -0500 Subject: [PATCH 3/4] Docs: late-result keys and which request an outcome's requestId names Co-Authored-By: Claude Fable 5.2 --- CHANGELOG.md | 45 +++++++++++++++++++++++++++++---------------- README.md | 15 ++++++++++----- src/types.ts | 12 ++++++++---- 3 files changed, 47 insertions(+), 25 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9aa9b867..c9944e1b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,22 +2,35 @@ Behavior change (iOS): a late 2xx now completes an entry that earlier versions left running and sent again. After a relaunch, iOS waits 10 s for -the result of an upload that finished while the app was not running. When -no result came in that time, earlier versions sent the request again as a -new attempt and dropped the first attempt's result if it arrived later. So -a first attempt that the server accepted was always followed by a second -send. Now the library keeps the first attempt's task key, marked as -replaced. When that result arrives later and is accepted (a 2xx, or a -status the definition's `accept` rules allow), the entry completes with the first attempt's response and the second attempt's -task is cancelled. If the second task had not started yet, nothing is sent -twice. If it already sent its request, the server still gets two requests, -as before. A late failure or cancel of the first attempt is ignored, and -the second attempt goes on. A first attempt's result that arrives after -the entry already settled is ignored, so the app gets one outcome. The -kept keys are removed when the entry settles. The `attempt` event of the -late result names the first attempt and its `X-Request-Id`; the settled -outcome's `attempts` and `requestId` still name the last attempt. Android -and chunked parts are unchanged. +the result of an upload that finished while the app was not running. When no +result came in that time, earlier versions sent the request again as a new +attempt and dropped the first attempt's result if it arrived later. So a +first attempt that the server accepted was always followed by a second send. + +Now the library keeps the first attempt's task key, marked as a late-result +key. When that result arrives later and is accepted (a 2xx, or a status the +definition's `accept` rules allow), the entry completes with the first +attempt's response, and the second attempt's task is cancelled. If the +second task had not started yet, nothing is sent twice. If it already sent +its request, the server still gets two requests, as before. A late failure +or cancel of the first attempt is ignored, and the second attempt goes on. A +first attempt's result that arrives after the entry settled is ignored, so +the app gets one outcome. + +Every late-result key of a request is removed when the request settles, when +its row is forgotten (the ack of a `completed` or `cancelled` outcome, or +`cancel()` on a settled row), and when a same-id `mutate()` replaces its +body. Keys of tasks that still owe a cancel callback stay until that +callback, as before. + +On iOS, a simple request's outcome now carries the `requestId` of the +attempt whose result settled it, so its response and `requestId` belong to +the same request. Before, it carried the last attempt's `requestId`, which +after a late result could be a task that was cancelled before it began. +`attempts` still counts every attempt. iOS chunked requests already did +this. On Android a simple request's last attempt is always the one that +settled it; a chunked request's `requestId` is the last part attempt that +started. Android and iOS chunked parts are otherwise unchanged. ## 11.3.0 diff --git a/README.md b/README.md index 57e22803..d7f6b16f 100644 --- a/README.md +++ b/README.md @@ -284,9 +284,11 @@ completion may still be in flight from the daemon, the library waits up to 10 s for it before it sends again. This is what stops a request that finished while the app was dead from being sent twice. On iOS, when that result comes after the wait, it still counts: an accepted result completes the entry with -the first attempt's response and cancels the new attempt's task. A failure -that comes late is ignored, and the new attempt goes on. If the new task -already sent its request, the server gets the request twice. +the first attempt's response and cancels the new attempt's task. The +outcome's response and `requestId` then both belong to the first attempt; +`attempts` counts every attempt. A failure that comes late is ignored, and +the new attempt goes on. If the new task already sent its request, the server +gets the request twice. # Reliable delivery @@ -371,8 +373,11 @@ its place in the list: - A same-id `mutate()` that reopens a settled entry starts every count at 0. A same-id `mutate()` on a live entry keeps the counts. -Every attempt sends an `X-Request-Id` header, minted per attempt. `Meta.requestId` -carries the last one. +Every attempt sends an `X-Request-Id` header, minted per attempt. +`Meta.requestId` carries the one of the attempt whose result settled the +entry, or the last attempt's when no attempt settled it (a cancel, an +expiry). For a chunked request on Android it is the last part attempt that +started. # API diff --git a/src/types.ts b/src/types.ts index 4bd96fc9..1d4fb512 100644 --- a/src/types.ts +++ b/src/types.ts @@ -170,10 +170,14 @@ export type OutcomeError = { }; /** - * Handler context. `at` is the native outcome time. `requestId` is the last - * attempt's X-Request-Id. A handler that keeps throwing sees `deliveries` - * grow. The library never gives up on its own, so the app decides a poison - * policy from this number. + * Handler context. `at` is the native outcome time. `requestId` is an + * X-Request-Id. On iOS, and for a simple request on Android, it is the one + * of the attempt whose result settled the entry. An outcome that no attempt + * settled (a cancel, an expiry) has the last attempt's. For a chunked + * request on Android it is the last part attempt that started, which can be + * another part than the one whose response settled the entry. A handler + * that keeps throwing sees `deliveries` grow. The library never gives up on + * its own, so the app decides a poison policy from this number. */ export type Meta = { id: string; From bff8a1d114498b7bcc9e5db1222dfb300e9963a2 Mon Sep 17 00:00:00 2001 From: Dylan Murphy Date: Tue, 17 Nov 2026 17:09:48 -0500 Subject: [PATCH 4/4] iOS: change the entry's request id only for an accepted result The copy of the completing task's request id into the entry now runs only for an accepted result, so a result that does not settle the entry as completed never changes its request id. taskWillBegin, taskProgress and replaceBackgroundTasks now check isOutdated, so a lateResult key is treated like a superseded one there. No task holds a lateResult key in those paths today: reconcile changes a live task's lateResult key to superseded. The CHANGELOG now says that a superseded or pause key of the waited attempt stays until its cancel callback, where the end of the wait used to delete it. Co-Authored-By: Claude Fable 5.2 --- CHANGELOG.md | 6 ++++-- ios/QueueCoordinator+Simple.swift | 14 +++++++----- ios/QueueCoordinator.swift | 2 +- ios/Tests/CoordinatorLateResultTests.swift | 25 ++++++++++++++++++++++ 4 files changed, 39 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c9944e1b..cf2a3e15 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,8 +20,10 @@ the app gets one outcome. Every late-result key of a request is removed when the request settles, when its row is forgotten (the ack of a `completed` or `cancelled` outcome, or `cancel()` on a settled row), and when a same-id `mutate()` replaces its -body. Keys of tasks that still owe a cancel callback stay until that -callback, as before. +body. Before, the end of the 10 s wait deleted every key of the waited +attempt, including keys of tasks the library had cancelled (superseded or +paused). Now such a key stays until its task's cancel callback removes it, +or until the row is gone and the next launch prunes it. On iOS, a simple request's outcome now carries the `requestId` of the attempt whose result settled it, so its response and `requestId` belong to diff --git a/ios/QueueCoordinator+Simple.swift b/ios/QueueCoordinator+Simple.swift index 67ac7ccd..344826d7 100644 --- a/ios/QueueCoordinator+Simple.swift +++ b/ios/QueueCoordinator+Simple.swift @@ -187,8 +187,10 @@ extension QueueCoordinator { // must not disturb the newer attempt. if meta?.purpose?.isOutdated == true && (!accepted || !e.isLive) { return } // The outcome names the request whose response settles it. A late - // result belongs to an older attempt than the entry's last one. - if let requestId = meta?.requestId, requestId != e.lastRequestId { + // result belongs to an older attempt than the entry's last one. Only an + // accepted result settles the entry as completed, so only it may change + // the entry's request id. + if accepted, let requestId = meta?.requestId, requestId != e.lastRequestId { e.lastRequestId = requestId index.upsert(e) } @@ -269,8 +271,10 @@ extension QueueCoordinator { return chunked.partWillBegin(id: id, part: part, incarnation: incarnation, key: key, meta: meta) } // A task the library already replaced (a session move keeps the - // attempt ordinal, so the ordinal alone cannot tell them apart). - guard meta?.purpose != .superseded, + // attempt ordinal, so the ordinal alone cannot tell them apart), or + // one the relaunch grace gave up on. Cancel it: its key becomes + // superseded below, because it now owes a cancel callback. + guard meta?.purpose?.isOutdated != true, case .request(let id, let generation, let attempt) = owner, var e = index.entry(id), !e.isChunked, e.generation == generation, e.attempts == attempt, e.state == .queued || e.state == .running, !settings.isPaused(e.key), @@ -294,7 +298,7 @@ extension QueueCoordinator { /// a delayed retry that began while the app was dead to running. func taskProgress(key: String, description: String?, sent: Int64, expected: Int64) { let meta = taskMap.meta(forKey: key) - guard meta?.purpose != .superseded, let owner = TaskOwner.resolve(description: description, meta: meta), + guard meta?.purpose?.isOutdated != true, let owner = TaskOwner.resolve(description: description, meta: meta), throttle.shouldEmit(owner.id, now: now()) else { return } queue.async { switch owner { diff --git a/ios/QueueCoordinator.swift b/ios/QueueCoordinator.swift index 58b5e933..f7772abc 100644 --- a/ios/QueueCoordinator.swift +++ b/ios/QueueCoordinator.swift @@ -454,7 +454,7 @@ final class QueueCoordinator { var parts: [String: [ChunkedCoordinator.Replaced]] = [:] for (key, owner) in liveTasks { let meta = taskMap.meta(forKey: key) - guard meta?.foreground != true, meta?.purpose != .superseded, owner.task.isLive, + guard meta?.foreground != true, meta?.purpose?.isOutdated != true, owner.task.isLive, owner.task.bytesSent == 0, !owner.task.hasResponse, let resolved = TaskOwner.resolve(description: owner.task.taskDescription, meta: meta) else { continue } switch resolved { diff --git a/ios/Tests/CoordinatorLateResultTests.swift b/ios/Tests/CoordinatorLateResultTests.swift index 472e01fb..112e8c53 100644 --- a/ios/Tests/CoordinatorLateResultTests.swift +++ b/ios/Tests/CoordinatorLateResultTests.swift @@ -203,6 +203,31 @@ final class CoordinatorLateResultTests: XCTestCase { error: NSError(domain: NSURLErrorDomain, code: NSURLErrorCancelled)) } + func testALateNonAcceptedResultLeavesTheRequestIdAlone() throws { + let (fresh, original, b) = try reissuedAfterTheGrace() + XCTAssertEqual(fresh.entry("a")?.lastRequestId, b.header("X-Request-Id")) + fresh.complete(replay(original), status: 409) + XCTAssertEqual(fresh.entry("a")?.lastRequestId, b.header("X-Request-Id")) + XCTAssertEqual(fresh.store.load("a")?.lastRequestId, b.header("X-Request-Id")) + } + + /// The entry's request id changes only for an accepted result. Here the + /// task's TaskMap request id is set by hand to differ from the entry's, + /// and a 401 parks the entry: the parked row keeps its own request id. + func testOnlyAnAcceptedResultChangesTheEntrysRequestId() throws { + h.boot() + _ = try h.enqueue(h.dataRaw(id: "a")).get() + let task = h.transport.live[0] + var meta = try XCTUnwrap(h.map.meta(forKey: task.key)) + let own = h.entry("a")?.lastRequestId + meta.requestId = "another-request" + h.map.set(meta, forKey: task.key) + h.complete(task, status: 401) + XCTAssertEqual(h.entry("a")?.state, .awaitingAuth) + XCTAssertEqual(h.entry("a")?.lastRequestId, own) + XCTAssertEqual(h.store.load("a")?.lastRequestId, own) + } + func testALate2xxAfterTheNewAttemptCompletedIsDropped() throws { let (fresh, original, b) = try reissuedAfterTheGrace() fresh.complete(b, status: 200)