diff --git a/CHANGELOG.md b/CHANGELOG.md index 574795a4..cf2a3e15 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,39 @@ +## Unreleased + +Behavior change (iOS): a late 2xx now completes an entry that earlier +versions left running and sent again. After a relaunch, iOS waits 10 s for +the result of an upload that finished while the app was not running. When no +result came in that time, earlier versions sent the request again as a new +attempt and dropped the first attempt's result if it arrived later. So a +first attempt that the server accepted was always followed by a second send. + +Now the library keeps the first attempt's task key, marked as a late-result +key. When that result arrives later and is accepted (a 2xx, or a status the +definition's `accept` rules allow), the entry completes with the first +attempt's response, and the second attempt's task is cancelled. If the +second task had not started yet, nothing is sent twice. If it already sent +its request, the server still gets two requests, as before. A late failure +or cancel of the first attempt is ignored, and the second attempt goes on. A +first attempt's result that arrives after the entry settled is ignored, so +the app gets one outcome. + +Every late-result key of a request is removed when the request settles, when +its row is forgotten (the ack of a `completed` or `cancelled` outcome, or +`cancel()` on a settled row), and when a same-id `mutate()` replaces its +body. Before, the end of the 10 s wait deleted every key of the waited +attempt, including keys of tasks the library had cancelled (superseded or +paused). Now such a key stays until its task's cancel callback removes it, +or until the row is gone and the next launch prunes it. + +On iOS, a simple request's outcome now carries the `requestId` of the +attempt whose result settled it, so its response and `requestId` belong to +the same request. Before, it carried the last attempt's `requestId`, which +after a late result could be a task that was cancelled before it began. +`attempts` still counts every attempt. iOS chunked requests already did +this. On Android a simple request's last attempt is always the one that +settled it; a chunked request's `requestId` is the last part attempt that +started. Android and iOS chunked parts are otherwise unchanged. + ## 11.3.0 Added `useRequests(filter?)` and `useRequestsWithProgress(filter?)`, React diff --git a/README.md b/README.md index 525c964f..d7f6b16f 100644 --- a/README.md +++ b/README.md @@ -282,7 +282,13 @@ above lets the library journal their outcomes. **Relaunch.** When the app comes back and an entry's task is gone but a completion may still be in flight from the daemon, the library waits up to 10 s for it before it sends again. This is what stops a request that finished -while the app was dead from being sent twice. +while the app was dead from being sent twice. On iOS, when that result comes +after the wait, it still counts: an accepted result completes the entry with +the first attempt's response and cancels the new attempt's task. The +outcome's response and `requestId` then both belong to the first attempt; +`attempts` counts every attempt. A failure that comes late is ignored, and +the new attempt goes on. If the new task already sent its request, the server +gets the request twice. # Reliable delivery @@ -367,8 +373,11 @@ its place in the list: - A same-id `mutate()` that reopens a settled entry starts every count at 0. A same-id `mutate()` on a live entry keeps the counts. -Every attempt sends an `X-Request-Id` header, minted per attempt. `Meta.requestId` -carries the last one. +Every attempt sends an `X-Request-Id` header, minted per attempt. +`Meta.requestId` carries the one of the attempt whose result settled the +entry, or the last attempt's when no attempt settled it (a cancel, an +expiry). For a chunked request on Android it is the last part attempt that +started. # API diff --git a/ios/QueueCoordinator+Enqueue.swift b/ios/QueueCoordinator+Enqueue.swift index 94941cce..e3210aa9 100644 --- a/ios/QueueCoordinator+Enqueue.swift +++ b/ios/QueueCoordinator+Enqueue.swift @@ -167,6 +167,8 @@ extension QueueCoordinator { n.headerGeneration = settings.headerGeneration n.state = paused ? .paused : .queued try saveOrDiscard(n, staged: staged) + // A late result of the old body must not complete the new one. + removeLateResultKeys(p.id) store.removeV9Manifest(p.id) store.sweep(n) // deletes the old body publish(n) diff --git a/ios/QueueCoordinator+Outcomes.swift b/ios/QueueCoordinator+Outcomes.swift index 1e141a21..42e7d946 100644 --- a/ios/QueueCoordinator+Outcomes.swift +++ b/ios/QueueCoordinator+Outcomes.swift @@ -63,6 +63,7 @@ extension QueueCoordinator { journaled = false } cancelTasks(id, purpose: .superseded) + removeLateResultKeys(id) chunked.stop(id) emitProgress(id, sent: e.bytesSent, total: e.totalBytes) e.settledEventId = event.eventId @@ -108,12 +109,23 @@ extension QueueCoordinator { n.nextAttemptAt = nil n.authParked = false cancelTasks(e.id, purpose: .superseded) + removeLateResultKeys(e.id) chunked.stop(e.id) disarmExpiry(e.id) commit(n, emit: false) return n } + /// Removes every late-result key of `id` (see withoutTask), whatever its + /// attempt or generation. Called when the id settles, is forgotten, or + /// gets a different body: a late result can then no longer complete what + /// the id holds, and a kept key must not reach a later use of the id. No + /// task in this process holds such a key, so none owes a cancel callback + /// that would read it. Keys of other purposes stay until their callback. + func removeLateResultKeys(_ id: String) { + taskMap.removeAll { _, m in m.id == id && m.purpose == .lateResult } + } + /// Relaunch, before any task is matched: every live row whose own /// generation already has an unacked outcome takes that outcome, so it is /// never sent again. @@ -129,6 +141,7 @@ extension QueueCoordinator { /// unacked outcomes (cancel on a settled entry). func forget(_ id: String, dropEvents: Bool) { cancelTasks(id, purpose: .superseded) + removeLateResultKeys(id) chunked.stop(id) store.remove(id) index.remove(id) @@ -163,6 +176,7 @@ extension QueueCoordinator { } pendingJournal = pendingJournal.filter { $0.value.id != id } cancelTasks(id, purpose: .superseded) + removeLateResultKeys(id) chunked.stop(id) index.remove(id) disarmExpiry(id) diff --git a/ios/QueueCoordinator+Reconcile.swift b/ios/QueueCoordinator+Reconcile.swift index b9d3252b..be9e1669 100644 --- a/ios/QueueCoordinator+Reconcile.swift +++ b/ios/QueueCoordinator+Reconcile.swift @@ -111,8 +111,12 @@ extension QueueCoordinator { let meta = taskMap.meta(forKey: task.key) // A task the library replaced or dropped, still live because the // process died before the daemon applied the cancel. Never adopt it: - // its replacement, or the re-issue below, drives the entry. - if meta?.purpose == .superseded { + // its replacement, or the re-issue below, drives the entry. A kept + // late-result key on a live task becomes superseded: the task now owes + // a cancel callback, so its key must stay until that callback, and a + // settle removes only late-result keys. + if let purpose = meta?.purpose, purpose.isOutdated { + if purpose == .lateResult { taskMap.setPurpose(.superseded, forKey: task.key, id: meta?.id ?? "") } task.cancel() continue } @@ -182,13 +186,21 @@ extension QueueCoordinator { guard let self, let current = self.index.entry(e.id), current.state == e.state, current.generation == e.generation, current.attempts == e.attempts, !self.liveTasks.values.contains(where: { $0.id == e.id }) else { return } - // No replay moved the entry: the task is lost. Its key would send - // every later launch through this wait again. - self.taskMap.removeAll { _, m in - m.id == e.id && m.generation == e.generation && m.attempt == e.attempts - } - // It may have run, so the next attempt gets a new ordinal: a late - // replay of this one is then dropped as stale. + // No replay came in time. The task may be lost, or its result may + // still arrive later. Keep its keys as late-result keys instead of + // deleting them. A late accepted result then completes the entry and + // cancels the new attempt's task, and a late failure or cancel is + // dropped (taskCompletedLocked). The wait covers only the current + // attempt, so once the re-issue below moves the entry to a new + // attempt, a later launch does not wait for these keys again. A key + // already marked superseded or pause keeps its purpose: its task may + // still owe a cancel callback. + let stale = self.taskMap.keys(where: { + $0.id == e.id && $0.generation == e.generation && $0.attempt == e.attempts + && ($0.purpose ?? .attempt) == .attempt + }) + self.taskMap.setPurpose(.lateResult, for: stale.map { (key: $0, id: e.id) }) + // It may have run, so the next attempt gets a new ordinal and request id. self.reissue(current, advanceAttempt: true) } } diff --git a/ios/QueueCoordinator+Simple.swift b/ios/QueueCoordinator+Simple.swift index 032a119a..344826d7 100644 --- a/ios/QueueCoordinator+Simple.swift +++ b/ios/QueueCoordinator+Simple.swift @@ -172,17 +172,28 @@ extension QueueCoordinator { // entry, or an older one the library replaced (see below). guard case .request(let id, let generation, let attempt) = owner, var e = index.entry(id), !e.legacy, !e.isChunked, e.generation == generation, - e.attempts == attempt || (meta?.purpose == .superseded && attempt < e.attempts) else { return } + e.attempts == attempt || (meta?.purpose?.isOutdated == true && attempt < e.attempts) else { return } let cancelled = RetryClassifier.isCancellation(c.error) - if cancelled, meta?.purpose == .pause || meta?.purpose == .superseded { return } + if cancelled, meta?.purpose == .pause || meta?.purpose?.isOutdated == true { return } let accepted = c.error == nil && c.statusCode.map { UploadOutcome.isAccepted($0, body: c.body, accept: e.accept) } == true - // A replaced task that finished before its cancel took effect. Its - // replacement drives the entry, unless this one landed while the entry - // is live. The server has the body then, so the entry completes, even - // when the replacement failed and the entry moved to a later attempt. - if meta?.purpose == .superseded && (!accepted || !e.isLive) { return } + // A replaced task that finished before its cancel took effect, or an + // older attempt whose result came after the relaunch grace gave up on it + // (a late-result key, see withoutTask). The newer task drives the entry, + // unless this one landed while the entry is live. The server has the + // body then, so the entry completes and settle cancels the newer task, + // even when the entry moved to a later attempt. A failure is dropped: it + // must not disturb the newer attempt. + if meta?.purpose?.isOutdated == true && (!accepted || !e.isLive) { return } + // The outcome names the request whose response settles it. A late + // result belongs to an older attempt than the entry's last one. Only an + // accepted result settles the entry as completed, so only it may change + // the entry's request id. + if accepted, let requestId = meta?.requestId, requestId != e.lastRequestId { + e.lastRequestId = requestId + index.upsert(e) + } // A delayed task that ended before its begin date, with no response and // no body byte sent, never reached the daemon: nsurlsessiond refused the // connection (a simulator build with no entitlements does this). It is @@ -260,8 +271,10 @@ extension QueueCoordinator { return chunked.partWillBegin(id: id, part: part, incarnation: incarnation, key: key, meta: meta) } // A task the library already replaced (a session move keeps the - // attempt ordinal, so the ordinal alone cannot tell them apart). - guard meta?.purpose != .superseded, + // attempt ordinal, so the ordinal alone cannot tell them apart), or + // one the relaunch grace gave up on. Cancel it: its key becomes + // superseded below, because it now owes a cancel callback. + guard meta?.purpose?.isOutdated != true, case .request(let id, let generation, let attempt) = owner, var e = index.entry(id), !e.isChunked, e.generation == generation, e.attempts == attempt, e.state == .queued || e.state == .running, !settings.isPaused(e.key), @@ -285,7 +298,7 @@ extension QueueCoordinator { /// a delayed retry that began while the app was dead to running. func taskProgress(key: String, description: String?, sent: Int64, expected: Int64) { let meta = taskMap.meta(forKey: key) - guard meta?.purpose != .superseded, let owner = TaskOwner.resolve(description: description, meta: meta), + guard meta?.purpose?.isOutdated != true, let owner = TaskOwner.resolve(description: description, meta: meta), throttle.shouldEmit(owner.id, now: now()) else { return } queue.async { switch owner { diff --git a/ios/QueueCoordinator.swift b/ios/QueueCoordinator.swift index 58b5e933..f7772abc 100644 --- a/ios/QueueCoordinator.swift +++ b/ios/QueueCoordinator.swift @@ -454,7 +454,7 @@ final class QueueCoordinator { var parts: [String: [ChunkedCoordinator.Replaced]] = [:] for (key, owner) in liveTasks { let meta = taskMap.meta(forKey: key) - guard meta?.foreground != true, meta?.purpose != .superseded, owner.task.isLive, + guard meta?.foreground != true, meta?.purpose?.isOutdated != true, owner.task.isLive, owner.task.bytesSent == 0, !owner.task.hasResponse, let resolved = TaskOwner.resolve(description: owner.task.taskDescription, meta: meta) else { continue } switch resolved { diff --git a/ios/TaskMap.swift b/ios/TaskMap.swift index 32e8d6f4..c97c4631 100644 --- a/ios/TaskMap.swift +++ b/ios/TaskMap.swift @@ -18,6 +18,18 @@ final class TaskMap { case attempt case pause case superseded + /// A simple task the relaunch grace gave up on: it was not live, and + /// no result came in the wait. The entry moved to a new attempt, and the + /// key stays so a late accepted result can still complete the entry. No + /// task in this process holds it, so it owes no cancel callback, and + /// every key of this purpose for an id is removed when that id settles, + /// is forgotten, or gets a different body. An older build reads it as + /// nil and drops the late result. + case lateResult + + /// A task that no longer drives its entry: its failure and its cancel + /// produce nothing, and only an accepted result on a live entry counts. + var isOutdated: Bool { self == .superseded || self == .lateResult } } struct Meta: Codable, Equatable { diff --git a/ios/Tests/CoordinatorLateResultTests.swift b/ios/Tests/CoordinatorLateResultTests.swift new file mode 100644 index 00000000..112e8c53 --- /dev/null +++ b/ios/Tests/CoordinatorLateResultTests.swift @@ -0,0 +1,423 @@ +import XCTest +@testable import RNBGUCore + +/// After a relaunch, the grace for a simple entry with no live task ends +/// and the entry moves to a new attempt. The old attempt's key is kept as a +/// late-result key, so its late accepted result still completes the entry. +final class CoordinatorLateResultTests: XCTestCase { + private var h: Harness! + + override func setUp() { + h = Harness() + } + + override func tearDown() { + try? FileManager.default.removeItem(at: h.root) + } + + /// Attempt 1's task finished while the app was dead, and its callback is + /// late. The grace ends, attempt 2 (task B) is issued, and then attempt 1's + /// result arrives. + private func reissuedAfterTheGrace(extra: [String: Any] = [:]) throws + -> (fresh: Harness, original: FakeTask, b: FakeTask) { + h.boot() + _ = try h.enqueue(h.dataRaw(id: "a", extra: extra)).get() + let original = h.transport.live[0] + let fresh = Harness(root: h.root) + fresh.boot() + fresh.advance(Double(QueueCoordinator.graceMs)) + let b = try XCTUnwrap(fresh.transport.live.first) + XCTAssertEqual(fresh.entry("a")?.attempts, 2) + XCTAssertEqual(fresh.map.meta(forKey: original.key)?.purpose, .lateResult) + return (fresh, original, b) + } + + private func replay(_ task: FakeTask) -> FakeTask { + FakeTask(key: task.key, description: task.taskDescription, request: task.request) + } + + private func settledResponse(_ x: Harness) -> [String: Any]? { + x.sink.settled.last?["response"] as? [String: Any] + } + + // MARK: - A late accepted result wins + + func testALateReplayAfterAReissueCompletesTheEntryAndStopsTheNewTask() throws { + let (fresh, original, b) = try reissuedAfterTheGrace() + fresh.complete(replay(original), status: 201, body: #"{"id":7}"#) + XCTAssertEqual(fresh.entry("a")?.state, .completed) + XCTAssertTrue(b.cancelled, "the second send stops") + XCTAssertEqual(fresh.sink.settled.count, 1) + XCTAssertEqual(settledResponse(fresh)?["status"] as? Int, 201, "the app gets the first attempt's response") + XCTAssertEqual(settledResponse(fresh)?["body"] as? String, #"{"id":7}"#) + XCTAssertEqual(fresh.sink.settled.first?["attempts"] as? Int, 2, "the outcome counts both attempts") + XCTAssertEqual(fresh.sink.settled.first?["requestId"] as? String, original.header("X-Request-Id"), + "the request id of the response's own request") + XCTAssertEqual(fresh.sink.attempts.last?["attempt"] as? Int, 1) + XCTAssertEqual(fresh.sink.attempts.last?["requestId"] as? String, original.header("X-Request-Id")) + // B's cancel callback adds nothing. + fresh.deliverCancel(b) + XCTAssertEqual(fresh.sink.settled.count, 1) + XCTAssertEqual(fresh.transport.created.count, 1, "no third send") + XCTAssertTrue(fresh.map.keys(where: { $0.id == "a" }).isEmpty) + } + + /// A late 2xx while the new attempt waits out a backoff. The waiting task + /// never began, so the outcome must not name its request id. + func testALate2xxWhileTheNewAttemptWaitsInBackoffCompletesTheEntry() throws { + let (fresh, original, b) = try reissuedAfterTheGrace() + fresh.complete(b, status: 503) + let delayed = try XCTUnwrap(fresh.transport.live.first) + XCTAssertEqual(fresh.entry("a")?.state, .queued) + fresh.complete(replay(original), status: 201) + XCTAssertEqual(fresh.entry("a")?.state, .completed) + XCTAssertTrue(delayed.cancelled) + XCTAssertEqual(fresh.sink.settled.first?["attempts"] as? Int, 3) + XCTAssertEqual(fresh.sink.settled.first?["requestId"] as? String, original.header("X-Request-Id")) + XCTAssertNotEqual(fresh.sink.settled.first?["requestId"] as? String, delayed.header("X-Request-Id")) + fresh.deliverCancel(delayed) + fresh.advance(60_000) + XCTAssertEqual(fresh.transport.created.count, 2, "B and the delayed task only") + XCTAssertEqual(fresh.sink.settled.count, 1) + XCTAssertTrue(fresh.map.keys(where: { $0.id == "a" }).isEmpty) + } + + func testALate2xxWhilePausedCompletesTheEntry() throws { + let (fresh, original, b) = try reissuedAfterTheGrace() + fresh.pause() + XCTAssertEqual(fresh.entry("a")?.state, .paused) + fresh.complete(replay(original), status: 201) + XCTAssertEqual(fresh.entry("a")?.state, .completed) + XCTAssertEqual(fresh.sink.settled.count, 1) + fresh.deliverCancel(b) + fresh.resume() + XCTAssertEqual(fresh.entry("a")?.state, .completed) + XCTAssertEqual(fresh.transport.created.count, 1, "the resume sends nothing") + XCTAssertEqual(fresh.sink.settled.count, 1) + XCTAssertTrue(fresh.map.keys(where: { $0.id == "a" }).isEmpty) + } + + func testALate2xxWhileAwaitingAuthCompletesTheEntry() throws { + let (fresh, original, b) = try reissuedAfterTheGrace() + fresh.complete(b, status: 401) + XCTAssertEqual(fresh.entry("a")?.state, .awaitingAuth) + fresh.complete(replay(original), status: 201) + XCTAssertEqual(fresh.entry("a")?.state, .completed) + XCTAssertEqual(fresh.sink.settled.count, 1) + fresh.updateHeaders(["Authorization": "new"]) + XCTAssertEqual(fresh.transport.created.count, 1, "new headers send nothing") + } + + /// The late 2xx lands at the next launch, before its reconcile, while B is + /// still live in the daemon. Reconcile then cancels B. + func testALate2xxBeforeTheNextReconcileCompletesTheEntryAndReconcileStopsB() throws { + let (_, original, b) = try reissuedAfterTheGrace() + let survivor = replay(b) + let third = Harness(root: h.root) + third.transport.deferAllTasks = true + third.relaunch(daemonTasks: [survivor]) + var released = false + third.coordinator.reconcileAll { released = true } + third.drain() + third.complete(replay(original), status: 201) + XCTAssertEqual(third.entry("a")?.state, .completed) + third.transport.releaseAllTasks() + third.drain() + third.drain() + XCTAssertTrue(survivor.cancelled) + XCTAssertTrue(released) + XCTAssertTrue(third.transport.created.isEmpty) + third.deliverCancel(survivor) + XCTAssertEqual(third.sink.settled.count, 1) + XCTAssertTrue(third.map.keys(where: { $0.id == "a" }).isEmpty) + } + + /// A second relaunch between the re-issue and the late result. Attempt + /// 1's task is listed as finished (not live): reconcile neither adopts nor + /// cancels it, and its result still completes the entry. + func testALateReplayAfterASecondRelaunchStillCompletesTheEntry() throws { + let (_, original, b) = try reissuedAfterTheGrace() + let listed = replay(original) + listed.isLive = false + let survivor = replay(b) + let third = Harness(root: h.root) + third.relaunch(daemonTasks: [listed, survivor]) + third.boot() + XCTAssertFalse(listed.cancelled) + XCTAssertTrue(third.transport.created.isEmpty, "B is adopted") + XCTAssertEqual(third.map.meta(forKey: original.key)?.purpose, .lateResult) + third.complete(listed, status: 201) + XCTAssertEqual(third.entry("a")?.state, .completed) + XCTAssertTrue(survivor.cancelled) + XCTAssertEqual(third.sink.settled.count, 1) + } + + /// Same, but B is lost too: the second grace issues attempt 3, and attempt + /// 1's late result still wins over it. + func testALateReplayAfterTwoReissuesCompletesTheEntry() throws { + let (_, original, b) = try reissuedAfterTheGrace() + let third = Harness(root: h.root) + third.boot() + third.advance(Double(QueueCoordinator.graceMs)) + let c = try XCTUnwrap(third.transport.live.first) + XCTAssertEqual(third.entry("a")?.attempts, 3) + XCTAssertEqual(third.map.meta(forKey: b.key)?.purpose, .lateResult) + third.complete(replay(original), status: 201) + XCTAssertEqual(third.entry("a")?.state, .completed) + XCTAssertTrue(c.cancelled) + XCTAssertEqual(third.sink.settled.count, 1) + XCTAssertTrue(third.map.keys(where: { $0.id == "a" && $0.purpose == .lateResult }).isEmpty) + } + + // MARK: - A late failure or cancel is dropped + + /// Each late result gets its own kept key: the first delivery removes it. + private func assertALateResultLeavesTheNewAttemptAlone( + status: Int? = nil, error: NSError? = nil, file: StaticString = #filePath, line: UInt = #line) throws { + let (fresh, original, b) = try reissuedAfterTheGrace() + let attemptsBefore = fresh.sink.attempts.count + fresh.complete(replay(original), status: status, error: error) + XCTAssertEqual(fresh.entry("a")?.state, .running, file: file, line: line) + XCTAssertEqual(fresh.entry("a")?.attempts, 2, file: file, line: line) + XCTAssertFalse(b.cancelled, file: file, line: line) + XCTAssertEqual(fresh.transport.created.count, 1, "no retry for the old attempt", file: file, line: line) + XCTAssertEqual(fresh.sink.attempts.count, attemptsBefore, "no event for the old attempt", file: file, line: line) + XCTAssertTrue(fresh.sink.settled.isEmpty, file: file, line: line) + fresh.complete(b, status: 201) + XCTAssertEqual(fresh.entry("a")?.state, .completed, file: file, line: line) + XCTAssertEqual(fresh.sink.settled.count, 1, file: file, line: line) + XCTAssertEqual(fresh.sink.settled.first?["requestId"] as? String, b.header("X-Request-Id"), file: file, line: line) + } + + func testALate500AfterAReissueLeavesTheNewAttemptAlone() throws { + try assertALateResultLeavesTheNewAttemptAlone(status: 500) + } + + func testALateTransportErrorAfterAReissueLeavesTheNewAttemptAlone() throws { + try assertALateResultLeavesTheNewAttemptAlone( + error: NSError(domain: NSURLErrorDomain, code: NSURLErrorTimedOut)) + } + + func testALateCancelAfterAReissueLeavesTheNewAttemptAlone() throws { + try assertALateResultLeavesTheNewAttemptAlone( + error: NSError(domain: NSURLErrorDomain, code: NSURLErrorCancelled)) + } + + func testALateNonAcceptedResultLeavesTheRequestIdAlone() throws { + let (fresh, original, b) = try reissuedAfterTheGrace() + XCTAssertEqual(fresh.entry("a")?.lastRequestId, b.header("X-Request-Id")) + fresh.complete(replay(original), status: 409) + XCTAssertEqual(fresh.entry("a")?.lastRequestId, b.header("X-Request-Id")) + XCTAssertEqual(fresh.store.load("a")?.lastRequestId, b.header("X-Request-Id")) + } + + /// The entry's request id changes only for an accepted result. Here the + /// task's TaskMap request id is set by hand to differ from the entry's, + /// and a 401 parks the entry: the parked row keeps its own request id. + func testOnlyAnAcceptedResultChangesTheEntrysRequestId() throws { + h.boot() + _ = try h.enqueue(h.dataRaw(id: "a")).get() + let task = h.transport.live[0] + var meta = try XCTUnwrap(h.map.meta(forKey: task.key)) + let own = h.entry("a")?.lastRequestId + meta.requestId = "another-request" + h.map.set(meta, forKey: task.key) + h.complete(task, status: 401) + XCTAssertEqual(h.entry("a")?.state, .awaitingAuth) + XCTAssertEqual(h.entry("a")?.lastRequestId, own) + XCTAssertEqual(h.store.load("a")?.lastRequestId, own) + } + + func testALate2xxAfterTheNewAttemptCompletedIsDropped() throws { + let (fresh, original, b) = try reissuedAfterTheGrace() + fresh.complete(b, status: 200) + XCTAssertEqual(fresh.entry("a")?.state, .completed) + let attemptsBefore = fresh.sink.attempts.count + fresh.complete(replay(original), status: 201) + XCTAssertEqual(fresh.entry("a")?.state, .completed) + XCTAssertEqual(fresh.sink.settled.count, 1, "one outcome") + XCTAssertEqual(fresh.sink.attempts.count, attemptsBefore) + XCTAssertTrue(fresh.transport.live.isEmpty) + } + + func testALate2xxAfterAUserCancelIsDropped() throws { + let (fresh, original, b) = try reissuedAfterTheGrace() + fresh.cancel("a") + XCTAssertEqual(fresh.entry("a")?.state, .cancelled) + XCTAssertNil(fresh.map.meta(forKey: original.key)) + let attemptsBefore = fresh.sink.attempts.count + fresh.complete(replay(original), status: 201) + fresh.deliverCancel(b) + XCTAssertEqual(fresh.entry("a")?.state, .cancelled) + XCTAssertEqual(fresh.sink.settled.count, 1, "the cancel only") + XCTAssertEqual(fresh.sink.attempts.count, attemptsBefore) + } + + // MARK: - When the kept keys go + + func testTheLateResultKeysAreRemovedWhenTheEntrySettles() throws { + let (fresh, original, b) = try reissuedAfterTheGrace() + fresh.complete(b, status: 400) + XCTAssertEqual(fresh.entry("a")?.state, .error) + XCTAssertNil(fresh.map.meta(forKey: original.key)) + XCTAssertTrue(fresh.map.keys(where: { $0.id == "a" }).isEmpty, "nothing left for the entry") + } + + /// The settle's journal write landed and its entry save did not, and the + /// process died before the kept key was removed. The relaunch repair of + /// the lost settle removes it. + func testTheRepairOfALostSettleRemovesTheLateResultKeys() throws { + let (fresh, original, b) = try reissuedAfterTheGrace() + let kept = try XCTUnwrap(fresh.map.meta(forKey: original.key)) + let dir = fresh.store.dir("a") + setReadOnly(dir, true) + fresh.complete(b, status: 201) + setReadOnly(dir, false) + XCTAssertEqual(fresh.store.load("a")?.state, .running, "the disk still says running") + fresh.map.set(kept, forKey: original.key) // the removal did not land + let third = Harness(root: h.root) + third.boot() + XCTAssertEqual(third.entry("a")?.state, .completed) + XCTAssertNil(third.map.meta(forKey: original.key)) + XCTAssertTrue(third.transport.created.isEmpty) + } + + /// A settled row whose kept key is still there, as if the removal at the + /// settle did not land. Forgetting the row removes it. + private func settledWithAKeptKey(status: Int) throws -> (Harness, FakeTask) { + let (fresh, original, b) = try reissuedAfterTheGrace() + let kept = try XCTUnwrap(fresh.map.meta(forKey: original.key)) + fresh.complete(b, status: status) + fresh.map.set(kept, forKey: original.key) + return (fresh, original) + } + + func testTheAckThatForgetsTheRowRemovesTheLateResultKeys() throws { + let (fresh, original) = try settledWithAKeptKey(status: 200) + fresh.ack([try XCTUnwrap(fresh.sink.settled.last?["eventId"] as? String)]) + XCTAssertNil(fresh.row("a")) + XCTAssertNil(fresh.map.meta(forKey: original.key)) + } + + func testCancelOfASettledRowRemovesTheLateResultKeys() throws { + let (fresh, original) = try settledWithAKeptKey(status: 400) + XCTAssertNil(fresh.cancel("a")) + XCTAssertNil(fresh.row("a")) + XCTAssertNil(fresh.map.meta(forKey: original.key)) + } + + /// The entry expires inside the grace's own re-issue, so it settles before + /// a new attempt is added: the kept key has the entry's current attempt. + /// It still goes, so a later use of the id cannot take its late result. + /// The later use is on attempt 2 here. On attempt 1 the task description + /// alone matches it, with or without a key: that needs a per-use value, + /// which this change does not add. + func testAnEntryThatSettlesInsideTheGraceReissueDropsItsKeptKey() throws { + h.boot() + _ = try h.enqueue(h.dataRaw(id: "a", extra: ["expiresAt": h.clock + 60_000])).get() + let original = h.transport.live[0] + let fresh = Harness(root: h.root) + fresh.clock = h.clock + 120_000 + fresh.boot() + fresh.advance(Double(QueueCoordinator.graceMs)) + XCTAssertEqual(fresh.entry("a")?.state, .error, "expired") + XCTAssertEqual(fresh.entry("a")?.attempts, 1) + XCTAssertNil(fresh.map.meta(forKey: original.key)) + // The app cancels the error row and uses the id again. + fresh.cancel("a") + _ = try fresh.enqueue(fresh.dataRaw(id: "a", data: ["x": 2])).get() + fresh.complete(try XCTUnwrap(fresh.transport.live.first), status: 503) + XCTAssertEqual(fresh.entry("a")?.attempts, 2) + let delayed = try XCTUnwrap(fresh.transport.live.first) + let settledBefore = fresh.sink.settled.count + fresh.complete(replay(original), status: 201, body: "STALE") + XCTAssertEqual(fresh.entry("a")?.state, .queued, "the new body still goes out") + XCTAssertFalse(delayed.cancelled) + XCTAssertEqual(fresh.sink.settled.count, settledBefore) + } + + /// A different body replaces the entry: the old body's late result must + /// not complete the new one. + private func assertADifferentBodyDropsTheKeptKey( + file: StaticString = #filePath, line: UInt = #line, _ setUp: (Harness, FakeTask) throws -> Void) throws { + let (fresh, original, b) = try reissuedAfterTheGrace() + try setUp(fresh, b) + XCTAssertNotEqual(fresh.entry("a")?.state, .running, file: file, line: line) + _ = try fresh.enqueue(fresh.dataRaw(id: "a", data: ["x": 2])).get() + XCTAssertEqual(fresh.entry("a")?.generation, 2, file: file, line: line) + XCTAssertNil(fresh.map.meta(forKey: original.key), file: file, line: line) + let stateBefore = fresh.entry("a")?.state + let live = fresh.transport.live + fresh.complete(replay(original), status: 201, body: "STALE") + XCTAssertEqual(fresh.entry("a")?.state, stateBefore, file: file, line: line) + XCTAssertTrue(live.allSatisfy { !$0.cancelled }, file: file, line: line) + XCTAssertTrue(fresh.sink.settled.isEmpty, file: file, line: line) + } + + func testADifferentBodyInBackoffDropsTheKeptKey() throws { + try assertADifferentBodyDropsTheKeptKey { fresh, b in fresh.complete(b, status: 503) } + } + + func testADifferentBodyWhilePausedDropsTheKeptKey() throws { + try assertADifferentBodyDropsTheKeptKey { fresh, b in + fresh.pause() + fresh.deliverCancel(b) + } + } + + func testADifferentBodyWhileAwaitingAuthDropsTheKeptKey() throws { + try assertADifferentBodyDropsTheKeptKey { fresh, b in fresh.complete(b, status: 401) } + } + + /// Three uses of one id. Use 1's attempt 1 is kept; a different body + /// replaces it (use 1, generation 2), which completes and is acked; use 3 + /// starts again at generation 1 and its first attempt fails. Use 1's late + /// 2xx must not complete use 3 with an old response. + func testALateResultFromAnEarlierUseOfTheIdDoesNotCompleteALaterUse() throws { + let (fresh, original, b) = try reissuedAfterTheGrace() + fresh.complete(b, status: 503) + _ = try fresh.enqueue(fresh.dataRaw(id: "a", data: ["x": 2])).get() + let e = try XCTUnwrap(fresh.transport.live.last) + fresh.complete(e, status: 200) + let eventId = try XCTUnwrap(fresh.sink.settled.last?["eventId"] as? String) + fresh.ack([eventId]) + XCTAssertNil(fresh.row("a")) + _ = try fresh.enqueue(fresh.dataRaw(id: "a", data: ["x": 3])).get() + let c = try XCTUnwrap(fresh.transport.live.last) + fresh.complete(c, status: 503) + XCTAssertEqual(fresh.entry("a")?.generation, 1) + XCTAssertEqual(fresh.entry("a")?.attempts, 2) + let delayed = try XCTUnwrap(fresh.transport.live.last) + let settledBefore = fresh.sink.settled.count + fresh.complete(replay(original), status: 201, body: "STALE-FROM-USE-1") + XCTAssertEqual(fresh.entry("a")?.state, .queued, "use 3 still sends its own body") + XCTAssertFalse(delayed.cancelled) + XCTAssertEqual(fresh.sink.settled.count, settledBefore) + } + + /// A task that reconcile found live under a kept key is cancelled. It + /// owes a cancel callback, so its key must outlive the settle and the ack. + /// Without it, after the id is used again, the old cancel would count as + /// a system cancel of the new use and send it twice. + func testACancelCallbackOfAReconcileCancelledTaskDoesNothingAfterTheIdIsUsedAgain() throws { + let (_, original, b) = try reissuedAfterTheGrace() + let old = replay(original) + let survivor = replay(b) + let third = Harness(root: h.root) + third.relaunch(daemonTasks: [old, survivor]) + third.boot() + XCTAssertTrue(old.cancelled) + XCTAssertFalse(survivor.cancelled, "B is adopted") + XCTAssertEqual(third.map.meta(forKey: old.key)?.purpose, .superseded, "it owes a cancel callback") + third.complete(survivor, status: 200) + XCTAssertEqual(third.map.meta(forKey: old.key)?.purpose, .superseded, "kept past the settle") + let eventId = try XCTUnwrap(third.sink.settled.last?["eventId"] as? String) + third.ack([eventId]) + _ = try third.enqueue(third.dataRaw(id: "a", data: ["x": 2])).get() + let created = third.transport.created.count + third.deliverCancel(old) + XCTAssertEqual(third.entry("a")?.state, .running) + XCTAssertEqual(third.entry("a")?.attempts, 1) + XCTAssertEqual(third.transport.created.count, created, "no second send") + } +} diff --git a/ios/Tests/CoordinatorRelaunchTests.swift b/ios/Tests/CoordinatorRelaunchTests.swift index e0e7fecf..761a14ea 100644 --- a/ios/Tests/CoordinatorRelaunchTests.swift +++ b/ios/Tests/CoordinatorRelaunchTests.swift @@ -64,7 +64,7 @@ final class CoordinatorRelaunchTests: XCTestCase { XCTAssertTrue(fresh.transport.created.isEmpty, "no duplicate request") } - func testRunningWithALostTaskReissuesAfterTheGraceAndPrunesItsKey() throws { + func testRunningWithALostTaskReissuesAfterTheGraceAndKeepsItsKeyForALateResult() throws { h.boot() _ = try h.enqueue(h.dataRaw(id: "a")).get() let lost = h.transport.live[0] @@ -73,12 +73,20 @@ final class CoordinatorRelaunchTests: XCTestCase { XCTAssertTrue(fresh.transport.created.isEmpty) fresh.advance(Double(QueueCoordinator.graceMs)) XCTAssertEqual(fresh.transport.created.count, 1) - XCTAssertNil(fresh.map.meta(forKey: lost.key)) - XCTAssertTrue(fresh.map.keys(where: { $0.id == "a" && $0.attempt == 1 }).isEmpty, "\(lost.key) pruned") - // The next launch does not wait the grace again for the same lost task. + XCTAssertEqual(fresh.map.meta(forKey: lost.key)?.purpose, .lateResult, "kept, so a late result can win") + let retry = try XCTUnwrap(fresh.transport.live.first) + // The next launch does not wait the grace again for the same lost task: + // the wait covers only attempt 2, so its replay ends the wait at once. let third = Harness(root: h.root) - third.boot() - XCTAssertTrue(third.map.keys(where: { $0.attempt == 1 }).isEmpty) + var released = false + third.coordinator.reconcileAll { released = true } + third.drain() + third.drain() + XCTAssertFalse(released) + third.complete(FakeTask(key: retry.key, description: retry.taskDescription, request: retry.request)) + XCTAssertTrue(released, "attempt 1's key does not hold the wait") + XCTAssertEqual(third.entry("a")?.state, .completed) + XCTAssertTrue(third.map.keys(where: { $0.id == "a" }).isEmpty, "the settle removes the kept key") } func testReconcileDropsKeysWhoseIdHasNoEntry() throws { @@ -111,7 +119,8 @@ final class CoordinatorRelaunchTests: XCTestCase { } // The key says the delayed task may have run while the app was dead: wait - // for its replay, then mint a new attempt so a late replay is stale. + // for its replay, then mint a new attempt. A late accepted replay of the + // old attempt still completes the entry. func testQueuedEntryWithAPendingKeyWaitsTheGraceThenMintsANewAttempt() throws { h.boot() _ = try h.enqueue(h.dataRaw(id: "a")).get() @@ -124,10 +133,12 @@ final class CoordinatorRelaunchTests: XCTestCase { let task = try XCTUnwrap(fresh.transport.live.first) XCTAssertEqual(fresh.entry("a")?.attempts, 3) XCTAssertNotEqual(task.header("X-Request-Id"), waiting.header("X-Request-Id")) - XCTAssertTrue(fresh.map.keys(where: { $0.id == "a" && $0.attempt == 2 }).isEmpty, "the lost task's key is pruned") - // A late replay of the lost attempt is dropped. + XCTAssertEqual(fresh.map.meta(forKey: waiting.key)?.purpose, .lateResult, "the old key is kept, marked") + // A late accepted replay of the old attempt wins over the new one. fresh.complete(FakeTask(key: waiting.key, description: waiting.taskDescription, request: waiting.request)) - XCTAssertEqual(fresh.entry("a")?.state, .running) + XCTAssertEqual(fresh.entry("a")?.state, .completed) + XCTAssertTrue(task.cancelled) + XCTAssertEqual(fresh.sink.settled.count, 1) } // MARK: - Review fixes: lost saves, early completions, pending replays diff --git a/src/types.ts b/src/types.ts index 4bd96fc9..1d4fb512 100644 --- a/src/types.ts +++ b/src/types.ts @@ -170,10 +170,14 @@ export type OutcomeError = { }; /** - * Handler context. `at` is the native outcome time. `requestId` is the last - * attempt's X-Request-Id. A handler that keeps throwing sees `deliveries` - * grow. The library never gives up on its own, so the app decides a poison - * policy from this number. + * Handler context. `at` is the native outcome time. `requestId` is an + * X-Request-Id. On iOS, and for a simple request on Android, it is the one + * of the attempt whose result settled the entry. An outcome that no attempt + * settled (a cancel, an expiry) has the last attempt's. For a chunked + * request on Android it is the last part attempt that started, which can be + * another part than the one whose response settled the entry. A handler + * that keeps throwing sees `deliveries` grow. The library never gives up on + * its own, so the app decides a poison policy from this number. */ export type Meta = { id: string;