diff --git a/CHANGELOG.md b/CHANGELOG.md index cf2a3e15..1270deed 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,7 +23,9 @@ its row is forgotten (the ack of a `completed` or `cancelled` outcome, or body. Before, the end of the 10 s wait deleted every key of the waited attempt, including keys of tasks the library had cancelled (superseded or paused). Now such a key stays until its task's cancel callback removes it, -or until the row is gone and the next launch prunes it. +or until a later launch prunes it. A launch prunes a key whose task is not +running when the row is gone, or when the id now belongs to a newer request +(see the next fix). On iOS, a simple request's outcome now carries the `requestId` of the attempt whose result settled it, so its response and `requestId` belong to @@ -34,6 +36,32 @@ this. On Android a simple request's last attempt is always the one that settled it; a chunked request's `requestId` is the last part attempt that started. Android and iOS chunked parts are otherwise unchanged. +Fixed (iOS): a late result from an earlier use of a request id could change +a later request with the same id. An app may `mutate()` an id again after +its row is gone (the outcome was acknowledged, or the row was cancelled). The +new request starts again at attempt 1, so a result that arrived late from +the earlier request could match it. A late 2xx then completed the new +request with the old response and cancelled its task, so the new body was +not sent. A late 4xx failed it, and a late 5xx counted as one of its failed +attempts. Now each use of an id has its own value (the entry's +`incarnation`, a random value made when the entry is created or its body is +replaced). Every simple upload task carries it, and a result, progress +event or relaunch match from another use is ignored. iOS chunked parts +already carried it. + +A task started by 11.3.0 or earlier carries no value. A request created +before the upgrade keeps the old behavior until it settles: it accepts any +task with no value that has its id, generation and attempt. So its own task +in flight during the upgrade is not sent again. A late result from an even +earlier use of the id, also started by the old version, can still change +it, as before. A request created after the upgrade ignores every task with +no value: such a task can only come from an earlier use. + +Also on iOS: when the 10 s relaunch wait gave up on a task and the next +attempt could not be saved, the next launch cancelled that task if it was +still running and sent the request again. It now keeps the task, which may +already have sent the body. + ## 11.3.0 Added `useRequests(filter?)` and `useRequestsWithProgress(filter?)`, React diff --git a/README.md b/README.md index d7f6b16f..bb91f2a5 100644 --- a/README.md +++ b/README.md @@ -203,6 +203,12 @@ id that exists: - A completed entry whose outcome is not yet acknowledged: the journaled outcome is emitted again. The request does not run again. +When the row is gone, the same id starts a new request. The ack of a +`completed` or `cancelled` outcome removes the row, and so does `cancel()` +on a settled row. The ack of an `error` outcome keeps it. On iOS a result, progress event or +cancel that arrives late from the earlier request is ignored: it never +completes, fails or retries the new one. + ### Silent uploads (Android) `android: { noNotification: true }` runs the request without a progress diff --git a/ios/ChunkedEngine.swift b/ios/ChunkedEngine.swift index 195aa31e..b8bee6dc 100644 --- a/ios/ChunkedEngine.swift +++ b/ios/ChunkedEngine.swift @@ -46,10 +46,14 @@ enum ChunkedEngine { var inc: String? } + /// `inc` is the entry's incarnation: one value per use of the id (see + /// QueueEntry.incarnation). nil in a task from a build before it. An + /// older build decodes a description with `inc` and ignores the field. private struct RequestRef: Codable { let id: String let gen: Int let att: Int + var inc: String? } /// A chunked part task: (id, part index, incarnation). @@ -62,14 +66,16 @@ enum ChunkedEngine { return (ref.id, ref.part, ref.inc) } - /// A simple attempt: (id, entry generation, attempt ordinal). - static func taskDescription(id: String, attempt: Int, generation: Int) -> String { - requestPrefix + encode(RequestRef(id: id, gen: generation, att: attempt)) + /// A simple attempt: (id, entry generation, attempt ordinal, the + /// entry's incarnation). + static func taskDescription(id: String, attempt: Int, generation: Int, incarnation: String) -> String { + requestPrefix + encode(RequestRef(id: id, gen: generation, att: attempt, inc: incarnation)) } - static func parseRequestDescription(_ description: String?) -> (id: String, generation: Int, attempt: Int)? { + static func parseRequestDescription(_ description: String?) + -> (id: String, generation: Int, attempt: Int, incarnation: String?)? { guard let ref: RequestRef = decode(description, requestPrefix) else { return nil } - return (ref.id, ref.gen, ref.att) + return (ref.id, ref.gen, ref.att, ref.inc) } private static func encode(_ value: T) -> String { diff --git a/ios/QueueCoordinator+Reconcile.swift b/ios/QueueCoordinator+Reconcile.swift index be9e1669..0b137537 100644 --- a/ios/QueueCoordinator+Reconcile.swift +++ b/ios/QueueCoordinator+Reconcile.swift @@ -108,7 +108,18 @@ extension QueueCoordinator { let liveKeys = Set(tasks.filter(\.isLive).map(\.key)) for task in tasks where task.isLive { - let meta = taskMap.meta(forKey: task.key) + var meta = taskMap.meta(forKey: task.key) + let owner = TaskOwner.resolve(description: task.taskDescription, meta: meta) + let entry = owner.flatMap { index.entry($0.id) } + // A late-result key whose task is live and is the entry's current + // attempt. The wait gave up on this attempt, but the re-issue after it + // was never saved (a failed write-ahead), so the entry is still on this + // attempt. Adopt the task: it may have sent the body already. + if meta?.purpose == .lateResult, case .request(let id, let generation, let attempt, let incarnation)? = owner, + let e = entry, isCurrentSimpleTask(e, generation: generation, attempt: attempt, incarnation: incarnation) { + taskMap.setPurpose(.attempt, forKey: task.key, id: id) + meta?.purpose = .attempt + } // A task the library replaced or dropped, still live because the // process died before the daemon applied the cancel. Never adopt it: // its replacement, or the re-issue below, drives the entry. A kept @@ -120,27 +131,33 @@ extension QueueCoordinator { task.cancel() continue } - let owner = TaskOwner.resolve(description: task.taskDescription, meta: meta) - let entry = owner.flatMap { index.entry($0.id) } let runnable = entry.map { !$0.legacy && ($0.state == .queued || $0.state == .running) } ?? false switch owner { case .part(let id, let part, let incarnation)? where runnable && entry?.isChunked == true: partTasks[id, default: []].append(.init(task: task, part: part, incarnation: incarnation)) - case .request(let id, let generation, let attempt)? - where runnable && entry?.isChunked == false && entry?.generation == generation - && entry?.attempts == attempt && !simpleLive.contains(id): + case .request(let id, let generation, let attempt, let incarnation)? + where entry.map { isCurrentSimpleTask($0, generation: generation, attempt: attempt, incarnation: incarnation) } + == true && !simpleLive.contains(id): simpleLive.insert(id) liveTasks[task.key] = (id, task) default: - // No v10 owner (a v9 task), an older generation or attempt, a - // duplicate, or an entry that must have no task. Drop its callback. + // No v10 owner (a v9 task), an older generation or attempt, an + // earlier use of the id, a duplicate, or an entry that must have no + // task. Drop its callback. taskMap.setPurpose(.superseded, forKey: task.key, id: owner?.id ?? "") task.cancel() } } - // A key whose id has no entry belongs to nothing. A live task keeps its - // key until its cancel callback, which reads the purpose. - taskMap.removeAll { key, meta in !liveKeys.contains(key) && index.entry(meta.id) == nil } + // A key whose id has no entry belongs to nothing, and a simple key of an + // earlier use of the id belongs to nothing now. A live task keeps its key + // until its cancel callback, which reads the purpose. A task listed as + // finished may still report: its description names its use, so the + // report is dropped without the key. + taskMap.removeAll { key, meta in + guard !liveKeys.contains(key) else { return false } + guard let e = index.entry(meta.id) else { return true } + return meta.partIndex == nil && meta.generation != nil && !e.ownsSimpleTask(incarnation: meta.incarnation) + } for e in index.entries() where e.isLive && !e.legacy { armExpiry(e) @@ -174,7 +191,8 @@ extension QueueCoordinator { /// completion is handled. private func withoutTask(_ e: QueueEntry) { let pending = taskMap.keys(where: { - $0.id == e.id && $0.generation == e.generation && $0.attempt == e.attempts + $0.id == e.id && e.ownsSimpleTask(incarnation: $0.incarnation) + && $0.generation == e.generation && $0.attempt == e.attempts }) guard !pending.isEmpty else { // No task was ever made for a waiting attempt: it never ran, so it @@ -196,7 +214,8 @@ extension QueueCoordinator { // already marked superseded or pause keeps its purpose: its task may // still owe a cancel callback. let stale = self.taskMap.keys(where: { - $0.id == e.id && $0.generation == e.generation && $0.attempt == e.attempts + $0.id == e.id && e.ownsSimpleTask(incarnation: $0.incarnation) + && $0.generation == e.generation && $0.attempt == e.attempts && ($0.purpose ?? .attempt) == .attempt }) self.taskMap.setPurpose(.lateResult, for: stale.map { (key: $0, id: e.id) }) @@ -205,6 +224,13 @@ extension QueueCoordinator { } } + /// Whether a simple task with this identity is the current attempt of `e`, + /// a simple entry that may have a task. + private func isCurrentSimpleTask(_ e: QueueEntry, generation: Int, attempt: Int, incarnation: String?) -> Bool { + !e.legacy && !e.isChunked && (e.state == .queued || e.state == .running) + && e.ownsSimpleTask(incarnation: incarnation) && e.generation == generation && e.attempts == attempt + } + /// Issues again, keeping what is left of a wait. private func reissue(_ e: QueueEntry, advanceAttempt: Bool) { var n = e diff --git a/ios/QueueCoordinator+Simple.swift b/ios/QueueCoordinator+Simple.swift index 344826d7..5df0f5c5 100644 --- a/ios/QueueCoordinator+Simple.swift +++ b/ios/QueueCoordinator+Simple.swift @@ -85,7 +85,7 @@ extension QueueCoordinator { } let meta = TaskMap.Meta( - id: id, attempt: e.attempts, requestId: requestId, + id: id, incarnation: e.incarnation, attempt: e.attempts, requestId: requestId, headerGeneration: settings.headerGeneration, generation: e.generation, purpose: .attempt, foreground: activity.isActive) let task: UploadTask @@ -93,7 +93,8 @@ extension QueueCoordinator { task = try transport.upload( buildRequest(e, url: url, requestId: requestId), fromFile: body, wifiOnly: settings.wifiOnly(e.wifiOnly), - description: ChunkedEngine.taskDescription(id: id, attempt: e.attempts, generation: e.generation), + description: ChunkedEngine.taskDescription( + id: id, attempt: e.attempts, generation: e.generation, incarnation: e.incarnation), beginAt: beginAt.map { Date(timeIntervalSince1970: $0 / 1000) }) } catch { // The session could not open the body. Gone since the check above: it @@ -169,9 +170,12 @@ extension QueueCoordinator { return } // Only the current attempt of the current generation may drive the - // entry, or an older one the library replaced (see below). - guard case .request(let id, let generation, let attempt) = owner, - var e = index.entry(id), !e.legacy, !e.isChunked, e.generation == generation, + // entry, or an older one the library replaced (see below). The task must + // belong to this use of the id: an earlier use (acked, cancelled, or + // replaced, then used again) also starts at generation 1, attempt 1. + guard case .request(let id, let generation, let attempt, let incarnation) = owner, + var e = index.entry(id), !e.legacy, !e.isChunked, e.ownsSimpleTask(incarnation: incarnation), + e.generation == generation, e.attempts == attempt || (meta?.purpose?.isOutdated == true && attempt < e.attempts) else { return } let cancelled = RetryClassifier.isCancellation(c.error) @@ -275,8 +279,9 @@ extension QueueCoordinator { // one the relaunch grace gave up on. Cancel it: its key becomes // superseded below, because it now owes a cancel callback. guard meta?.purpose?.isOutdated != true, - case .request(let id, let generation, let attempt) = owner, - var e = index.entry(id), !e.isChunked, e.generation == generation, + case .request(let id, let generation, let attempt, let incarnation) = owner, + var e = index.entry(id), !e.isChunked, e.ownsSimpleTask(incarnation: incarnation), + e.generation == generation, e.attempts == attempt, e.state == .queued || e.state == .running, !settings.isPaused(e.key), let url = e.url.flatMap(URL.init(string:)) else { taskMap.setPurpose(.superseded, forKey: key, id: owner.id) @@ -304,12 +309,14 @@ extension QueueCoordinator { switch owner { case .part(let id, let part, let incarnation): self.chunked.partProgress(id: id, part: part, incarnation: incarnation, key: key, sent: sent) - case .request(let id, let generation, let attempt): + case .request(let id, let generation, let attempt, let incarnation): // A replacement shares its attempt with the task it replaced, so // only the task this process holds may report. Before reconcile no - // task is held yet, and none was replaced. + // task is held yet, and none was replaced: then the task's own + // identity, use of the id included, must match. guard !self.ready || self.liveTasks[key] != nil, - var e = self.index.entry(id), e.generation == generation, e.attempts == attempt, + var e = self.index.entry(id), e.ownsSimpleTask(incarnation: incarnation), + e.generation == generation, e.attempts == attempt, e.state == .queued || e.state == .running else { return } if e.state == .queued { e.state = .running diff --git a/ios/QueueCoordinator.swift b/ios/QueueCoordinator.swift index f7772abc..b051795a 100644 --- a/ios/QueueCoordinator.swift +++ b/ios/QueueCoordinator.swift @@ -460,8 +460,8 @@ final class QueueCoordinator { switch resolved { case .part(let id, let part, let incarnation): parts[id, default: []].append(.init(part: part, incarnation: incarnation, key: key)) - case .request(let id, let generation, let attempt): - replaceRequestTask(id, generation: generation, attempt: attempt, key: key) + case .request(let id, let generation, let attempt, let incarnation): + replaceRequestTask(id, generation: generation, attempt: attempt, incarnation: incarnation, key: key) } } for (id, replaced) in parts { @@ -472,8 +472,12 @@ final class QueueCoordinator { /// A simple entry's current attempt, sent again from a new task. The /// attempt never sent its body, so it keeps its ordinal and request id. /// An empty body is left alone: 0 bytes sent cannot tell whether it went. - private func replaceRequestTask(_ id: String, generation: Int, attempt: Int, key: String) { - guard var e = index.entry(id), !e.legacy, !e.isChunked, e.generation == generation, + /// The task must belong to this use of the id, as in every other reader + /// of a task's owner. A forget already cancels and drops the held tasks, + /// so this check is a guard, not the only one. + private func replaceRequestTask(_ id: String, generation: Int, attempt: Int, incarnation: String?, key: String) { + guard var e = index.entry(id), !e.legacy, !e.isChunked, e.ownsSimpleTask(incarnation: incarnation), + e.generation == generation, e.attempts == attempt, e.state == .queued || e.state == .running, e.totalBytes > 0, !settings.isPaused(e.key) else { return } let remaining = e.nextAttemptAt.map { Int($0 - now()) }.flatMap { $0 > 0 ? $0 : nil } diff --git a/ios/QueueEntry.swift b/ios/QueueEntry.swift index 176fdef8..48e42739 100644 --- a/ios/QueueEntry.swift +++ b/ios/QueueEntry.swift @@ -55,8 +55,13 @@ struct QueueEntry: Codable, Equatable { /// The body identity for the same-id rules. See EnqueueParser.fingerprint. var bodyFingerprint: String var parts: [Part] - /// The parts-plan identity. Rotates when the body is replaced. Part tasks - /// carry it, so a late callback from a replaced plan is dropped. + /// The parts-plan identity, and the identity of this use of the id. A new + /// random value at create and when the body is replaced, so no two uses of + /// an id share one. Part tasks and simple tasks carry it, so a late + /// callback from a replaced plan or from an earlier use of the id is + /// dropped. A same-body reopen keeps it: the generation tells those apart. + /// A v9 chunked upload that create() resumes keeps its v9 value; its + /// manifest is removed then, so a later use gets a new one. var incarnation: String /// The merged request headers. updateHeaders() patches them. var headers: [String: String] @@ -90,6 +95,12 @@ struct QueueEntry: Codable, Equatable { var legacy: Bool let createdAt: Double var updatedAt: Double + /// true when this use of the id started in a build that writes + /// `incarnation` into every simple task. Then a simple task with no + /// incarnation comes from an earlier use. nil in an entry that an older + /// build created or saved: its tasks may carry no incarnation. An older + /// build ignores the field and drops it at its next save of the entry. + var tasksCarryIncarnation: Bool? = nil } extension QueueEntry { @@ -101,6 +112,17 @@ extension QueueEntry { } var isSettled: Bool { !isLive } + + /// Whether a simple task that carries `incarnation` (nil: none) belongs + /// to this use of the id. A task with a value must carry this entry's. A + /// task with none is from a build before the value. It matches only an + /// entry such a build created or saved, as every task did before, so a + /// task in flight across the upgrade still drives its entry and is not + /// sent again. + func ownsSimpleTask(incarnation task: String?) -> Bool { + guard let task else { return tasksCarryIncarnation != true } + return task == incarnation + } var isChunked: Bool { bodyKind == .parts } var acceptedBytes: Int64 { parts.filter(\.accepted).reduce(0) { $0 + $1.size } } @@ -158,7 +180,7 @@ extension QueueEntry { /// Rule 2: a new entry. static func created(from p: ParsedEnqueue, staged: StagedBody, headerGeneration: Int, paused: Bool, now: Double, createdAt: Double? = nil) -> QueueEntry { - QueueEntry( + var e = QueueEntry( id: p.id, key: p.key, varsJSON: p.varsJSON, url: p.url, method: p.method, accept: p.accept, retry: p.retry, wifiOnly: p.wifiOnly, bodyKind: staged.kind, bodyPath: staged.relativePath, bodyContentType: staged.contentType, forceContentType: staged.forceContentType, @@ -168,6 +190,8 @@ extension QueueEntry { bytesSent: 0, totalBytes: staged.totalBytes, expiresAt: p.expiresAt, nextAttemptAt: nil, settledEventId: nil, lastRequestId: nil, lastUrl: nil, lastPartIndex: nil, legacy: false, createdAt: createdAt ?? now, updatedAt: now) + e.tasksCarryIncarnation = true + return e } /// Rule 3, same body: the new vars, headers, expiresAt and descriptor diff --git a/ios/Tests/CoordinatorLateResultTests.swift b/ios/Tests/CoordinatorLateResultTests.swift index 112e8c53..77a4e7b2 100644 --- a/ios/Tests/CoordinatorLateResultTests.swift +++ b/ios/Tests/CoordinatorLateResultTests.swift @@ -310,8 +310,8 @@ final class CoordinatorLateResultTests: XCTestCase { /// a new attempt is added: the kept key has the entry's current attempt. /// It still goes, so a later use of the id cannot take its late result. /// The later use is on attempt 2 here. On attempt 1 the task description - /// alone matches it, with or without a key: that needs a per-use value, - /// which this change does not add. + /// alone would match it by id, generation and attempt; the incarnation in + /// the description drops it (CoordinatorPerUseTests). func testAnEntryThatSettlesInsideTheGraceReissueDropsItsKeptKey() throws { h.boot() _ = try h.enqueue(h.dataRaw(id: "a", extra: ["expiresAt": h.clock + 60_000])).get() diff --git a/ios/Tests/CoordinatorPerUseTests.swift b/ios/Tests/CoordinatorPerUseTests.swift new file mode 100644 index 00000000..51f94802 --- /dev/null +++ b/ios/Tests/CoordinatorPerUseTests.swift @@ -0,0 +1,471 @@ +import XCTest +@testable import RNBGUCore + +/// An app can use one request id again for the same object. Each use is a +/// new entry: it starts again at generation 1 and attempt 1. A task result, +/// cancel callback, progress event or relaunch match that comes from an +/// earlier use of the id must never change a later use. +final class CoordinatorPerUseTests: XCTestCase { + private var h: Harness! + + override func setUp() { + h = Harness() + } + + override func tearDown() { + try? FileManager.default.removeItem(at: h.root) + } + + private func replay(_ task: FakeTask) -> FakeTask { + FakeTask(key: task.key, description: task.taskDescription, request: task.request) + } + + /// Use 1's attempt 1 (`original`) finished while the app was dead and its + /// callback is late. The relaunch wait ends, attempt 2 (B) completes use 1, + /// the outcome is acked, and the id is used again with another body. Use + /// 2 runs its attempt 1 (`c`). + private func secondUseAfterALateWait() throws -> (fresh: Harness, original: FakeTask, c: FakeTask) { + h.boot() + _ = try h.enqueue(h.dataRaw(id: "a")).get() + let original = h.transport.live[0] + let fresh = Harness(root: h.root) + fresh.boot() + fresh.advance(Double(QueueCoordinator.graceMs)) + let b = try XCTUnwrap(fresh.transport.live.first) + fresh.complete(b, status: 200) + fresh.ack([try XCTUnwrap(fresh.sink.settled.first?["eventId"] as? String)]) + XCTAssertNil(fresh.row("a")) + _ = try fresh.enqueue(fresh.dataRaw(id: "a", data: ["x": 2])).get() + let c = try XCTUnwrap(fresh.transport.live.first) + XCTAssertEqual(fresh.entry("a")?.generation, 1) + XCTAssertEqual(fresh.entry("a")?.attempts, 1) + return (fresh, original, c) + } + + /// Use 2 is untouched: still running its own attempt 1, with no event. + private func assertTheSecondUseIsUntouched( + _ fresh: Harness, c: FakeTask, settledBefore: Int, attemptsBefore: Int, createdBefore: Int, + file: StaticString = #filePath, line: UInt = #line) { + XCTAssertEqual(fresh.entry("a")?.state, .running, file: file, line: line) + XCTAssertEqual(fresh.entry("a")?.attempts, 1, file: file, line: line) + XCTAssertEqual(fresh.entry("a")?.lastRequestId, c.header("X-Request-Id"), file: file, line: line) + XCTAssertFalse(c.cancelled, "use 2's own task goes on", file: file, line: line) + XCTAssertEqual(fresh.transport.created.count, createdBefore, "no retry", file: file, line: line) + XCTAssertEqual(fresh.sink.attempts.count, attemptsBefore, "no attempt event", file: file, line: line) + XCTAssertEqual(fresh.sink.settled.count, settledBefore, "no outcome", file: file, line: line) + // Use 2's own result still settles it. + fresh.complete(c, status: 201, body: "USE-2") + XCTAssertEqual(fresh.entry("a")?.state, .completed, file: file, line: line) + XCTAssertEqual((fresh.sink.settled.last?["response"] as? [String: Any])?["body"] as? String, "USE-2", + file: file, line: line) + } + + private func assertALateResultOfTheFirstUseLeavesTheSecondAlone( + status: Int, file: StaticString = #filePath, line: UInt = #line) throws { + let (fresh, original, c) = try secondUseAfterALateWait() + let settled = fresh.sink.settled.count + let attempts = fresh.sink.attempts.count + let created = fresh.transport.created.count + fresh.complete(replay(original), status: status, body: "STALE-FROM-USE-1") + assertTheSecondUseIsUntouched(fresh, c: c, settledBefore: settled, attemptsBefore: attempts, + createdBefore: created, file: file, line: line) + } + + // MARK: - A late result of an earlier use + + func testALate500FromAnEarlierUseIsNotAFailedAttemptOfTheNextUse() throws { + try assertALateResultOfTheFirstUseLeavesTheSecondAlone(status: 500) + } + + func testALate2xxFromAnEarlierUseDoesNotCompleteTheNextUse() throws { + try assertALateResultOfTheFirstUseLeavesTheSecondAlone(status: 201) + } + + func testALate4xxFromAnEarlierUseDoesNotFailTheNextUse() throws { + try assertALateResultOfTheFirstUseLeavesTheSecondAlone(status: 400) + } + + /// The task-map key of use 1's attempt 1 is still there: the app cancelled + /// use 1 during the relaunch wait, so the key kept its purpose. Use 2 is + /// on attempt 1, so the key and the description both match it by id, + /// generation and attempt. + func testALate2xxFromAnEarlierUseWithItsKeyStillThereDoesNotCompleteTheNextUse() throws { + h.boot() + _ = try h.enqueue(h.dataRaw(id: "a")).get() + let original = h.transport.live[0] + let fresh = Harness(root: h.root) + fresh.boot() + XCTAssertNil(fresh.cancel("a")) + fresh.ack([try XCTUnwrap(fresh.unacknowledged().first?["eventId"] as? String)]) + XCTAssertNil(fresh.row("a")) + XCTAssertNotNil(fresh.map.meta(forKey: original.key), "the waited key is still there") + _ = try fresh.enqueue(fresh.dataRaw(id: "a", data: ["x": 2])).get() + let c = try XCTUnwrap(fresh.transport.live.first) + let settled = fresh.sink.settled.count + let attempts = fresh.sink.attempts.count + fresh.complete(replay(original), status: 201, body: "STALE-FROM-USE-1") + assertTheSecondUseIsUntouched(fresh, c: c, settledBefore: settled, attemptsBefore: attempts, createdBefore: 1) + } + + /// Use 1's task B was cancelled by the late win of attempt 1, and its 2xx + /// lands anyway, after use 2 moved to attempt 2 (B's own ordinal). + func testATaskCancelledByALateWinThatLandsInALaterUseIsDropped() throws { + h.boot() + _ = try h.enqueue(h.dataRaw(id: "a")).get() + let original = h.transport.live[0] + let fresh = Harness(root: h.root) + fresh.boot() + fresh.advance(Double(QueueCoordinator.graceMs)) + let b = try XCTUnwrap(fresh.transport.live.first) + fresh.complete(replay(original), status: 201) + XCTAssertTrue(b.cancelled) + fresh.ack([try XCTUnwrap(fresh.sink.settled.first?["eventId"] as? String)]) + _ = try fresh.enqueue(fresh.dataRaw(id: "a", data: ["x": 2])).get() + fresh.complete(try XCTUnwrap(fresh.transport.live.first), status: 503) + XCTAssertEqual(fresh.entry("a")?.attempts, 2) + let delayed = try XCTUnwrap(fresh.transport.live.first) + let settled = fresh.sink.settled.count + fresh.complete(b, status: 200, body: "STALE-B") + XCTAssertEqual(fresh.entry("a")?.state, .queued, "use 2 still sends its own body") + XCTAssertFalse(delayed.cancelled) + XCTAssertEqual(fresh.sink.settled.count, settled) + } + + /// cancel(id) cancels use 1's running task, but the task had already + /// landed. Its 2xx comes after the ack and after use 2 started. + func testATaskStoppedByCancelThatLandsInALaterUseIsDropped() throws { + h.boot() + _ = try h.enqueue(h.dataRaw(id: "a")).get() + let first = h.transport.live[0] + XCTAssertNil(h.cancel("a")) + XCTAssertTrue(first.cancelled) + h.ack([try XCTUnwrap(h.sink.settled.first?["eventId"] as? String)]) + XCTAssertNil(h.row("a")) + _ = try h.enqueue(h.dataRaw(id: "a", data: ["x": 2])).get() + let c = try XCTUnwrap(h.transport.live.first) + let settled = h.sink.settled.count + let attempts = h.sink.attempts.count + h.complete(first, status: 200, body: "STALE-FROM-USE-1") + assertTheSecondUseIsUntouched(h, c: c, settledBefore: settled, attemptsBefore: attempts, createdBefore: 2) + } + + // MARK: - Begin, progress and relaunch + + /// Use 1 was cancelled and acked, use 2 started. Use 1's task-map key is + /// gone (as after a lost write), so only the description names the task. + private func secondUseWithAnUnkeyedFirstTask() throws -> (first: FakeTask, c: FakeTask) { + h.boot() + _ = try h.enqueue(h.dataRaw(id: "a")).get() + let first = h.transport.live[0] + XCTAssertNil(h.cancel("a")) + h.ack([try XCTUnwrap(h.sink.settled.first?["eventId"] as? String)]) + _ = try h.enqueue(h.dataRaw(id: "a", data: ["x": 2])).get() + let c = try XCTUnwrap(h.transport.live.first) + h.map.removeKey(first.key) + return (replay(first), c) + } + + func testADelayedTaskOfAnEarlierUseThatBeginsIsCancelled() throws { + let (first, _) = try secondUseWithAnUnkeyedFirstTask() + XCTAssertNil(h.coordinator.taskWillBegin(key: first.key, description: first.taskDescription), + "the old task must not send") + XCTAssertEqual(h.map.meta(forKey: first.key)?.purpose, .superseded, "its cancel callback is dropped") + XCTAssertEqual(h.entry("a")?.state, .running) + } + + /// Before the first reconcile no task is held, so the progress guard can + /// only use the task's own identity. + func testProgressOfAnEarlierUseBeforeReconcileIsDropped() throws { + let (first, _) = try secondUseWithAnUnkeyedFirstTask() + h.relaunch() + let progressBefore = h.sink.progress.count + h.coordinator.taskProgress(key: first.key, description: first.taskDescription, sent: 5, expected: 10) + h.drain() + XCTAssertEqual(h.sink.progress.count, progressBefore) + XCTAssertEqual(h.entry("a")?.bytesSent, 0) + } + + /// Both tasks are live at the next launch, the old one listed first. The + /// old one must not be adopted in place of use 2's own task. + func testReconcileAdoptsTheTaskOfTheCurrentUseNotAnEarlierOne() throws { + let (first, c) = try secondUseWithAnUnkeyedFirstTask() + let current = replay(c) + let next = Harness(root: h.root) + next.relaunch(daemonTasks: [first, current]) + next.boot() + XCTAssertTrue(first.cancelled, "the old task stops") + XCTAssertFalse(current.cancelled, "use 2's task is adopted") + XCTAssertTrue(next.transport.created.isEmpty) + next.complete(current, status: 201, body: "USE-2") + XCTAssertEqual(next.entry("a")?.state, .completed) + } + + /// A superseded key of use 1 whose task never reported (the relaunch wait + /// gave up on it). Use 2 is on generation 1, attempt 1 too. At the next + /// launch use 2's own task finished while the app was dead, and its + /// replay must end the wait at once: the old key holds nothing. + func testAKeyOfAnEarlierUseDoesNotHoldTheNextUsesRelaunchWait() throws { + h.boot() + _ = try h.enqueue(h.dataRaw(id: "a")).get() + let original = h.transport.live[0] + h.map.setPurpose(.superseded, forKey: original.key, id: "a") + let fresh = Harness(root: h.root) + fresh.boot() + fresh.advance(Double(QueueCoordinator.graceMs)) + let b = try XCTUnwrap(fresh.transport.live.first) + fresh.complete(b, status: 200) + fresh.ack([try XCTUnwrap(fresh.sink.settled.first?["eventId"] as? String)]) + XCTAssertEqual(fresh.map.meta(forKey: original.key)?.purpose, .superseded, "it outlives the use") + _ = try fresh.enqueue(fresh.dataRaw(id: "a", data: ["x": 2])).get() + let c = try XCTUnwrap(fresh.transport.live.first) + let next = Harness(root: h.root) + var released = false + next.coordinator.reconcileAll { released = true } + next.drain() + next.drain() + XCTAssertNil(next.map.meta(forKey: original.key), "the next launch prunes it") + XCTAssertFalse(released, "it waits for use 2's own replay") + next.complete(replay(c), status: 201, body: "USE-2") + XCTAssertTrue(released, "the replay of use 2's task ends the wait") + XCTAssertEqual(next.entry("a")?.state, .completed) + XCTAssertTrue(next.transport.created.isEmpty) + } + + /// cancel(id) marked use 1's task superseded, but the process died before + /// the daemon applied the cancel, so the task is still live at the next + /// launch. Use 2's task finished while the app was dead. Use 1's task owes + /// a cancel callback; the wait for use 2's replay must not wait for it. + func testALiveTaskOfAnEarlierUseDoesNotHoldTheNextUsesRelaunchWait() throws { + h.boot() + _ = try h.enqueue(h.dataRaw(id: "a")).get() + let first = h.transport.live[0] + XCTAssertNil(h.cancel("a")) + h.ack([try XCTUnwrap(h.sink.settled.first?["eventId"] as? String)]) + _ = try h.enqueue(h.dataRaw(id: "a", data: ["x": 2])).get() + let c = try XCTUnwrap(h.transport.live.first) + let old = replay(first) + let next = Harness(root: h.root) + next.relaunch(daemonTasks: [old]) + var released = false + next.coordinator.reconcileAll { released = true } + next.drain() + next.drain() + XCTAssertTrue(old.cancelled) + XCTAssertFalse(released, "it waits for use 2's own replay") + next.complete(replay(c), status: 201, body: "USE-2") + XCTAssertTrue(released, "the replay of use 2's task ends the wait") + XCTAssertEqual(next.entry("a")?.state, .completed) + next.deliverCancel(old) + XCTAssertEqual(next.sink.settled.count, 1) + XCTAssertTrue(next.transport.created.isEmpty) + } + + // MARK: - Tasks and files from 11.3.0 and earlier + + /// What 11.3.0 wrote for a simple attempt: a description and a key with no + /// incarnation, and an entry.json with no tasksCarryIncarnation. Literal, + /// so a change to the encoder cannot change the fixture with it. + private static func description113(id: String, generation: Int, attempt: Int) -> String { + #"rnbgu-req:{"id":"\#(id)","gen":\#(generation),"att":\#(attempt)}"# + } + + /// Enqueues "a" and rewrites its entry, its task and its key as 11.3.0 + /// wrote them. Returns the task as the daemon holds it after the upgrade. + private func entryAndTaskFrom113() throws -> FakeTask { + h.boot() + _ = try h.enqueue(h.dataRaw(id: "a")).get() + let task = h.transport.live[0] + let url = h.store.fileURL("a", QueueStore.entryName) + var json = try XCTUnwrap(JSONSerialization.jsonObject(with: Data(contentsOf: url)) as? [String: Any]) + XCTAssertEqual(json["tasksCarryIncarnation"] as? Bool, true) + json["tasksCarryIncarnation"] = nil + try JSONSerialization.data(withJSONObject: json).write(to: url) + var meta = try XCTUnwrap(h.map.meta(forKey: task.key)) + XCTAssertEqual(meta.incarnation, h.entry("a")?.incarnation) + meta.incarnation = nil + h.map.set(meta, forKey: task.key) + return FakeTask(key: task.key, description: Self.description113(id: "a", generation: 1, attempt: 1), + request: task.request) + } + + func testATaskFrom113StillLiveAfterTheUpgradeIsAdoptedAndCompletesItsEntry() throws { + let old = try entryAndTaskFrom113() + let next = Harness(root: h.root) + next.relaunch(daemonTasks: [old]) + next.boot() + XCTAssertNil(next.entry("a")?.tasksCarryIncarnation) + XCTAssertFalse(old.cancelled, "the 11.3.0 task goes on") + XCTAssertTrue(next.transport.created.isEmpty, "nothing is sent again") + next.complete(old, status: 201, body: "FROM-113") + XCTAssertEqual(next.entry("a")?.state, .completed) + XCTAssertEqual((next.sink.settled.last?["response"] as? [String: Any])?["body"] as? String, "FROM-113") + } + + /// The 11.3.0 task finished while the app was dead, and its callback + /// replays inside the relaunch wait after the upgrade. + func testATaskFrom113ThatFinishesAfterTheUpgradeCompletesItsEntry() throws { + let old = try entryAndTaskFrom113() + let next = Harness(root: h.root) + var released = false + next.coordinator.reconcileAll { released = true } + next.drain() + next.drain() + XCTAssertFalse(released, "the 11.3.0 key is waited for") + next.complete(old, status: 201) + XCTAssertTrue(released) + XCTAssertEqual(next.entry("a")?.state, .completed) + XCTAssertTrue(next.transport.created.isEmpty) + } + + /// A failure of an 11.3.0 task still counts for its entry. The retry is a + /// tagged task, and it drives the entry that 11.3.0 created: it begins, + /// and its 2xx completes the entry. + func testAFailureOfATaskFrom113StillRetriesItsEntry() throws { + let old = try entryAndTaskFrom113() + let next = Harness(root: h.root) + next.relaunch(daemonTasks: [old]) + next.boot() + next.complete(old, status: 503) + XCTAssertEqual(next.entry("a")?.attempts, 2) + let retry = try XCTUnwrap(next.transport.live.first) + XCTAssertEqual(ChunkedEngine.parseRequestDescription(retry.taskDescription)?.incarnation, + next.entry("a")?.incarnation) + XCTAssertEqual(next.map.meta(forKey: retry.key)?.incarnation, next.entry("a")?.incarnation) + XCTAssertNil(next.entry("a")?.tasksCarryIncarnation, "the entry is still the one 11.3.0 created") + XCTAssertNotNil(next.coordinator.taskWillBegin(key: retry.key, description: retry.taskDescription), + "the retry begins") + XCTAssertEqual(next.entry("a")?.state, .running) + next.complete(retry, status: 201, body: "RETRY") + XCTAssertEqual(next.entry("a")?.state, .completed) + XCTAssertEqual((next.sink.settled.last?["response"] as? [String: Any])?["body"] as? String, "RETRY") + XCTAssertEqual(next.sink.settled.count, 1) + } + + /// After the upgrade the 11.3.0 entry completes and is acked, and the id + /// is used again. Another task from 11.3.0 then reports a 2xx. It has no + /// incarnation, and the new use was created by this build, so it is from + /// an earlier use. + func testATaskFrom113ThatReportsAfterTheIdIsUsedAgainIsDropped() throws { + let old = try entryAndTaskFrom113() + let next = Harness(root: h.root) + next.relaunch(daemonTasks: [old]) + next.boot() + next.complete(old, status: 201) + next.ack([try XCTUnwrap(next.sink.settled.last?["eventId"] as? String)]) + _ = try next.enqueue(next.dataRaw(id: "a", data: ["x": 2])).get() + XCTAssertEqual(next.entry("a")?.tasksCarryIncarnation, true) + let c = try XCTUnwrap(next.transport.live.first) + let stray = FakeTask(key: "any:113113", description: Self.description113(id: "a", generation: 1, attempt: 1)) + let settled = next.sink.settled.count + next.complete(stray, status: 201, body: "STALE-FROM-113") + XCTAssertEqual(next.entry("a")?.state, .running) + XCTAssertFalse(c.cancelled) + XCTAssertEqual(next.sink.settled.count, settled) + XCTAssertNil(next.coordinator.taskWillBegin(key: "any:113114", + description: Self.description113(id: "a", generation: 1, attempt: 1))) + } + + /// The 11.3.0 RequestRef, exactly as released. + private struct RequestRef113: Decodable { + let id: String + let gen: Int + let att: Int + } + + /// The stored fields of the 11.3.0 QueueEntry, exactly as released, with + /// its synthesized decoder. + private struct Entry113: Decodable { + let id: String + var key: String + var varsJSON: String + var url: String? + var method: String + var accept: [UploadOutcome.AcceptRule] + var retry: RetryOverride? + var wifiOnly: Bool? + var bodyKind: QueueEntry.BodyKind + var bodyPath: String? + var bodyContentType: String? + var forceContentType: Bool + var bodyFingerprint: String + var parts: [QueueEntry.Part] + var incarnation: String + var headers: [String: String] + var headerGeneration: Int + var state: QueueEntry.State + var authParked: Bool + var generation: Int + var attempts: Int + var transientHitsByRule: [String: Int]? = nil + var bytesSent: Int64 + var totalBytes: Int64 + var expiresAt: Double + var nextAttemptAt: Double? + var settledEventId: String? + var lastRequestId: String? + var lastUrl: String? + var lastPartIndex: Int? + var legacy: Bool + let createdAt: Double + var updatedAt: Double + } + + /// A downgrade to 11.3.0 reads what this build wrote. + func testAnOlderBuildReadsTheNewDescriptionEntryAndKey() throws { + h.boot() + _ = try h.enqueue(h.dataRaw(id: "a")).get() + let task = h.transport.live[0] + let entry = try XCTUnwrap(h.entry("a")) + let description = try XCTUnwrap(task.taskDescription) + XCTAssertTrue(description.contains(entry.incarnation)) + let prefix = "rnbgu-req:" + XCTAssertTrue(description.hasPrefix(prefix)) + let ref = try JSONDecoder().decode(RequestRef113.self, from: Data(description.dropFirst(prefix.count).utf8)) + XCTAssertEqual(ref.id, "a") + XCTAssertEqual(ref.gen, 1) + XCTAssertEqual(ref.att, 1) + let saved = try Data(contentsOf: h.store.fileURL("a", QueueStore.entryName)) + let old = try JSONDecoder().decode(Entry113.self, from: saved) + XCTAssertEqual(old.incarnation, entry.incarnation) + XCTAssertEqual(old.attempts, 1) + // The key adds no field: an 11.3.0 key of a part already had + // `incarnation`, and 11.3.0 resolves a key with no partIndex as a + // simple attempt whatever its incarnation. + let reread = TaskMap(fileURL: h.map.fileURL) + XCTAssertEqual(reread.meta(forKey: task.key)?.incarnation, entry.incarnation) + XCTAssertNil(reread.meta(forKey: task.key)?.partIndex) + } + + // MARK: - A failed re-issue save + + /// The relaunch wait ends and marks attempt 1's key as a late-result key, + /// but the re-issue cannot save, so the entry stays on attempt 1. At the + /// next launch attempt 1's task is live after all. It is the entry's + /// current attempt: adopt it, do not cancel it and send again. + func testALiveTaskOfTheCurrentAttemptUnderALateResultKeyIsAdopted() throws { + h.boot() + _ = try h.enqueue(h.dataRaw(id: "a")).get() + let original = h.transport.live[0] + let fresh = Harness(root: h.root) + fresh.boot() + let dir = fresh.store.dir("a") + setReadOnly(dir, true) + fresh.advance(Double(QueueCoordinator.graceMs)) + setReadOnly(dir, false) + XCTAssertEqual(fresh.map.meta(forKey: original.key)?.purpose, .lateResult) + XCTAssertEqual(fresh.store.load("a")?.attempts, 1, "the disk still has attempt 1") + let live = replay(original) + let next = Harness(root: h.root) + next.relaunch(daemonTasks: [live]) + next.boot() + XCTAssertFalse(live.cancelled, "the task that may have sent the body goes on") + XCTAssertEqual(next.map.meta(forKey: original.key)?.purpose, .attempt) + XCTAssertTrue(next.transport.created.isEmpty, "nothing is sent again") + next.advance(Double(QueueCoordinator.graceMs)) + XCTAssertTrue(next.transport.created.isEmpty) + next.complete(live, status: 201) + XCTAssertEqual(next.entry("a")?.state, .completed) + XCTAssertEqual(next.entry("a")?.attempts, 1) + XCTAssertEqual(next.sink.settled.count, 1) + } +} diff --git a/ios/Tests/CoordinatorRelaunchTests.swift b/ios/Tests/CoordinatorRelaunchTests.swift index 761a14ea..d2d7dcf9 100644 --- a/ios/Tests/CoordinatorRelaunchTests.swift +++ b/ios/Tests/CoordinatorRelaunchTests.swift @@ -92,7 +92,7 @@ final class CoordinatorRelaunchTests: XCTestCase { func testReconcileDropsKeysWhoseIdHasNoEntry() throws { h.boot() h.map.set(.init(id: "ghost", attempt: 1, generation: 1, purpose: .attempt), forKey: "any:77") - let live = FakeTask(key: "any:78", description: ChunkedEngine.taskDescription(id: "ghost2", attempt: 1, generation: 1)) + let live = FakeTask(key: "any:78", description: ChunkedEngine.taskDescription(id: "ghost2", attempt: 1, generation: 1, incarnation: "i")) h.map.set(.init(id: "ghost2", attempt: 1, generation: 1, purpose: .attempt), forKey: live.key) let fresh = Harness(root: h.root) fresh.relaunch(daemonTasks: [live]) @@ -373,7 +373,7 @@ final class CoordinatorRelaunchTests: XCTestCase { func testUnownedAndStaleTasksAreCancelled() throws { let v9 = FakeTask(key: "any:90", description: "bare-v9-id") - let orphan = FakeTask(key: "any:91", description: ChunkedEngine.taskDescription(id: "gone", attempt: 1, generation: 1)) + let orphan = FakeTask(key: "any:91", description: ChunkedEngine.taskDescription(id: "gone", attempt: 1, generation: 1, incarnation: "i")) h.relaunch(daemonTasks: [v9, orphan]) h.boot() XCTAssertTrue(v9.cancelled) diff --git a/ios/Tests/SupportComponentTests.swift b/ios/Tests/SupportComponentTests.swift index 3d3762e4..8a596dd8 100644 --- a/ios/Tests/SupportComponentTests.swift +++ b/ios/Tests/SupportComponentTests.swift @@ -227,12 +227,15 @@ final class TaskMapTests: XCTestCase { } func testOwnerResolution() { - let desc = ChunkedEngine.taskDescription(id: "a:b/c", attempt: 2, generation: 3) - XCTAssertEqual(TaskOwner.resolve(description: desc, meta: nil), .request(id: "a:b/c", generation: 3, attempt: 2)) + let desc = ChunkedEngine.taskDescription(id: "a:b/c", attempt: 2, generation: 3, incarnation: "use") + XCTAssertEqual(TaskOwner.resolve(description: desc, meta: nil), + .request(id: "a:b/c", generation: 3, attempt: 2, incarnation: "use")) XCTAssertEqual(TaskOwner.resolve(description: nil, meta: .init(id: "m", generation: 1, purpose: nil)), nil, "a meta without attempt has no v10 owner") XCTAssertEqual(TaskOwner.resolve(description: nil, meta: .init(id: "m", attempt: 1, generation: 1)), - .request(id: "m", generation: 1, attempt: 1)) + .request(id: "m", generation: 1, attempt: 1, incarnation: nil)) + XCTAssertEqual(TaskOwner.resolve(description: nil, meta: .init(id: "m", incarnation: "use", attempt: 1, generation: 1)), + .request(id: "m", generation: 1, attempt: 1, incarnation: "use"), "the key carries the use too") XCTAssertEqual(TaskOwner.resolve(description: "legacy-bare-id", meta: .init(id: "legacy-bare-id")), nil) XCTAssertEqual(TaskOwner.resolve(description: nil, meta: .init(id: "p", partIndex: 2, incarnation: "i")), .part(id: "p", part: 2, incarnation: "i")) @@ -265,9 +268,10 @@ final class ChunkedEngineTests: XCTestCase { XCTAssertEqual(ChunkedEngine.parsePartDescription(part)?.id, id) XCTAssertEqual(ChunkedEngine.parsePartDescription(part)?.part, 7) XCTAssertNil(ChunkedEngine.parseRequestDescription(part)) - let req = ChunkedEngine.taskDescription(id: id, attempt: 4, generation: 2) + let req = ChunkedEngine.taskDescription(id: id, attempt: 4, generation: 2, incarnation: "inc") XCTAssertEqual(ChunkedEngine.parseRequestDescription(req)?.id, id) XCTAssertEqual(ChunkedEngine.parseRequestDescription(req)?.attempt, 4) + XCTAssertEqual(ChunkedEngine.parseRequestDescription(req)?.incarnation, "inc") XCTAssertNil(ChunkedEngine.parsePartDescription(req)) XCTAssertNil(ChunkedEngine.parsePartDescription("bare-v9-id")) } diff --git a/ios/Transport.swift b/ios/Transport.swift index 2c2e9a41..54de36ed 100644 --- a/ios/Transport.swift +++ b/ios/Transport.swift @@ -69,13 +69,15 @@ struct TaskCompletion { } /// Who a task belongs to. From taskDescription first, TaskMap second. +/// `incarnation` names the use of the id the task belongs to (see +/// QueueEntry.incarnation). nil in a task from a build before it. enum TaskOwner: Equatable { - case request(id: String, generation: Int, attempt: Int) + case request(id: String, generation: Int, attempt: Int, incarnation: String?) case part(id: String, part: Int, incarnation: String?) var id: String { switch self { - case .request(let id, _, _), .part(let id, _, _): return id + case .request(let id, _, _, _), .part(let id, _, _): return id } } @@ -84,12 +86,12 @@ enum TaskOwner: Equatable { return .part(id: p.id, part: p.part, incarnation: p.incarnation) } if let r = ChunkedEngine.parseRequestDescription(description) { - return .request(id: r.id, generation: r.generation, attempt: r.attempt) + return .request(id: r.id, generation: r.generation, attempt: r.attempt, incarnation: r.incarnation) } guard let meta else { return nil } if let part = meta.partIndex { return .part(id: meta.id, part: part, incarnation: meta.incarnation) } if let generation = meta.generation, let attempt = meta.attempt { - return .request(id: meta.id, generation: generation, attempt: attempt) + return .request(id: meta.id, generation: generation, attempt: attempt, incarnation: meta.incarnation) } // A v9 simple task: a bare id with no generation. No v10 owner. return nil