From 42e1a3e0e2c1320684af1d994723c40db34f128e Mon Sep 17 00:00:00 2001 From: Vu Anh Phung Date: Wed, 11 Nov 2026 23:36:59 +0000 Subject: [PATCH 1/3] fix: bound Thrift CloudFetch downloads by retriesTimeout end to end A stalled CloudFetch GET (headers or body never arrive) blocked the fetch forever: the request had no timeout, and HttpRetryPolicy only checks retriesTimeout between attempts. Enforce retriesTimeout as a deadline over the whole chunk download, including retry backoff, abort the in-flight request when it expires, and fail with RetryError(TIMEOUT_EXCEEDED). Addresses #520. Co-authored-by: Isaac Signed-off-by: Vu Anh Phung --- CHANGELOG.md | 1 + lib/contracts/IDBSQLClient.ts | 5 +- lib/result/CloudFetchResultHandler.ts | 62 ++++++- tests/e2e/cloudfetch.test.ts | 78 +++++++++ .../result/CloudFetchResultHandler.test.ts | 160 ++++++++++++++++++ 5 files changed, 298 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2858c667..17dfba54 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ## Unreleased +- Thrift CloudFetch downloads are now bounded end to end by `retriesTimeout` (default 15 minutes), including every attempt and retry backoff. Previously a stalled download, where the response headers or body never arrived, blocked the fetch forever. A download still running when the deadline expires is aborted and fails with `RetryError` (`TIMEOUT_EXCEEDED`, message `CloudFetch result download failed: Retry timeout exceeded`). (#520) - Thrift connections made through `proxy` now apply `checkServerCertificate`, `customCaCert` and `clientCert`/`clientKey`. Previously the proxied path ignored them and verified against Node's process-wide trust store, so a certificate outside the configured trust set could be accepted. **Compatibility:** proxied connections now verify unless `checkServerCertificate: false` is set; as on direct connections, `NODE_TLS_REJECT_UNAUTHORIZED=0` no longer turns verification off. For an `https` proxy these options also apply to the proxy's own certificate. (#485) - Honor the client `useCloudFetch: false` default on the kernel backend so complete results can be fetched without cloud-storage downloads. (#574) - Thrift HTTP errors (`THTTPException`) now append server detail from the error headers (at most 500 characters). Kernel-backend errors now carry a non-enumerable `cause` summarizing the kernel error code and message (also at most 500 characters). Reyden fallback preserves KP001 after any existing kernel cause. (#467) diff --git a/lib/contracts/IDBSQLClient.ts b/lib/contracts/IDBSQLClient.ts index 2220ca12..162a6de1 100644 --- a/lib/contracts/IDBSQLClient.ts +++ b/lib/contracts/IDBSQLClient.ts @@ -141,7 +141,10 @@ export type ConnectionOptions = { * attempt with no retry. Both backends honour the same total-attempt * semantics (the kernel converts it to its after-initial retry count). * • `retriesTimeout` — maximum total wallclock spent retrying, in - * milliseconds. Default 900000 (15 minutes). + * milliseconds. Default 900000 (15 minutes). On the Thrift backend it is + * also the end-to-end deadline of each CloudFetch result download (all + * attempts plus backoff): a download still running when it expires is + * aborted and fails with a `RetryError` (`TIMEOUT_EXCEEDED`). * • `retryDelayMin` — minimum backoff between attempts, in milliseconds. * Default 1000. * • `retryDelayMax` — maximum backoff between attempts, in milliseconds. diff --git a/lib/result/CloudFetchResultHandler.ts b/lib/result/CloudFetchResultHandler.ts index a64bb4bb..e8251402 100644 --- a/lib/result/CloudFetchResultHandler.ts +++ b/lib/result/CloudFetchResultHandler.ts @@ -1,7 +1,7 @@ import fetch, { RequestInfo, RequestInit, Request, Response } from 'node-fetch'; import { TGetResultSetMetadataResp, TRowSet, TSparkArrowResultLink } from '../../thrift/TCLIService_types'; import HiveDriverError from '../errors/HiveDriverError'; -import RetryError from '../errors/RetryError'; +import RetryError, { RetryErrorCode } from '../errors/RetryError'; import IClientContext from '../contracts/IClientContext'; import IResultsProvider, { ResultsProviderFetchNextOptions } from './IResultsProvider'; import { ArrowBatch } from './utils'; @@ -17,6 +17,9 @@ import { safeEmit } from '../telemetry/telemetryUtils'; // unchanged while the unhandled-rejection leak is eliminated. type SettledDownload = { ok: true; batch: ArrowBatch } | { ok: false; error: unknown }; +// Largest delay `setTimeout` accepts; anything longer would fire immediately. +const MAX_TIMER_DELAY_MS = 2 ** 31 - 1; + export default class CloudFetchResultHandler implements IResultsProvider { private readonly context: IClientContext; @@ -150,8 +153,19 @@ export default class CloudFetchResultHandler implements IResultsProvider 0 && retriesTimeout <= MAX_TIMER_DELAY_MS ? retriesTimeout : undefined; + let controller: AbortController | undefined; + let timeoutError: RetryError | undefined; + let lastFailure: unknown; + let timer: NodeJS.Timeout | undefined; + // Read the body inside the retried block. CloudFetch downloads are large // GETs against pre-signed cloud-storage URLs that frequently surface // `socket hang up` / "Premature close" once the stream is mid-transfer. @@ -160,17 +174,51 @@ export default class CloudFetchResultHandler implements IResultsProvider { + const download = retryPolicy.invokeWithRetry(async () => { + // The timeout error is not retryable, so this also ends the retry loop when the + // deadline expired during a backoff sleep or aborted the attempt. + if (timeoutError) { + throw timeoutError; + } downloaded = undefined; - const request = new Request(url, requestConfig); - const response = await fetch(request); - if (response.ok) { - downloaded = await response.arrayBuffer(); + controller = deadlineMs === undefined ? undefined : new AbortController(); + const request = new Request(url, { + ...requestConfig, + signal: controller?.signal as unknown as RequestInit['signal'], + }); + try { + const response = await fetch(request); + if (response.ok) { + downloaded = await response.arrayBuffer(); + } else { + lastFailure = { request, response }; + } + return { request, response }; + } catch (error) { + if (timeoutError) { + throw timeoutError; + } + lastFailure = error; + throw error; } - return { request, response }; }); + + if (deadlineMs === undefined) { + result = await download; + } else { + const deadline = new Promise((_, reject) => { + timer = setTimeout(() => { + timeoutError = new RetryError(RetryErrorCode.TimeoutExceeded, lastFailure); + controller?.abort(); + reject(timeoutError); + }, deadlineMs); + }); + result = await Promise.race([download, deadline]); + } } catch (error) { throw CloudFetchResultHandler.describeDownloadError(error); + } finally { + clearTimeout(timer); } // Fall back to reading the body here if the retry policy returned a // response without consuming it via our operation (e.g. unit-test stubs diff --git a/tests/e2e/cloudfetch.test.ts b/tests/e2e/cloudfetch.test.ts index 4022fc0a..68a7a58c 100644 --- a/tests/e2e/cloudfetch.test.ts +++ b/tests/e2e/cloudfetch.test.ts @@ -234,4 +234,82 @@ describe('CloudFetch', () => { await session.close(); } }); + + it('should fail a stalled CloudFetch download with a timeout error within retriesTimeout', async () => { + const retriesTimeout = 3000; + const session = await openSession({ + cloudFetchConcurrentDownloads: 1, + useLZ4Compression: false, + retriesTimeout, + }); + + // Accepts every download and never responds + let serverHits = 0; + const server = http.createServer(() => { + serverHits += 1; + }); + await new Promise((resolve) => { + server.listen(0, '127.0.0.1', () => resolve()); + }); + const { port } = server.address() as AddressInfo; + + try { + const queriedRowsCount = 10000000; // result has to be quite big to enable CloudFetch + const operation = await session.executeStatement( + ` + SELECT * + FROM range(0, ${queriedRowsCount}) AS t1 + LEFT JOIN (SELECT 1) AS t2 + `, + { + maxRows: null, // disable DirectResults + useCloudFetch: true, + }, + ); + + await operation.finished(); + + // @ts-expect-error TS2339: Property getResultHandler does not exist on type IOperation + const resultHandler = await operation.getResultHandler(); + expect(resultHandler.source.source).to.be.instanceOf(CloudFetchResultHandler); + const cfResultHandler = resultHandler.source.source; + + const connectionProvider = await cfResultHandler.context.getConnectionProvider(); + const getAgentStub = sinon.stub(connectionProvider, 'getAgent').resolves(undefined); + + const rowSetProvider = cfResultHandler.source; + const originalFetchNext = rowSetProvider.fetchNext.bind(rowSetProvider); + sinon.stub(rowSetProvider, 'fetchNext').callsFake(async (...args: Array) => { + const rowSet = await originalFetchNext(...args); + rowSet?.resultLinks?.forEach((link: any) => { + link.fileLink = `http://127.0.0.1:${port}/result`; + link.httpHeaders = {}; + }); + return rowSet; + }); + + const startedAt = Date.now(); + let thrown: unknown; + try { + await operation.fetchChunk({ maxRows: 100000, disableBuffering: true }); + } catch (error) { + thrown = error; + } finally { + getAgentStub.restore(); + } + const elapsedMs = Date.now() - startedAt; + + expect(thrown).to.be.instanceOf(RetryError); + expect((thrown as RetryError).errorCode).to.be.equal(RetryErrorCode.TimeoutExceeded); + expect((thrown as RetryError).message).to.contain('CloudFetch result download failed: Retry timeout exceeded'); + expect(serverHits).to.be.equal(1); + expect(elapsedMs).to.be.lt(retriesTimeout + 5000); + } finally { + server.closeAllConnections(); + await new Promise((resolve) => { + server.close(() => resolve()); + }); + await session.close(); + } + }); }); diff --git a/tests/unit/result/CloudFetchResultHandler.test.ts b/tests/unit/result/CloudFetchResultHandler.test.ts index 030e4775..6b34f7d0 100644 --- a/tests/unit/result/CloudFetchResultHandler.test.ts +++ b/tests/unit/result/CloudFetchResultHandler.test.ts @@ -1,5 +1,7 @@ import { expect, AssertionError } from 'chai'; import sinon, { SinonStub } from 'sinon'; +import http from 'http'; +import { AddressInfo } from 'net'; import Int64 from 'node-int64'; import { compressFrameSync } from 'lz4-napi'; import { Request, Response } from 'node-fetch'; @@ -15,6 +17,7 @@ import RetryError, { RetryErrorCode } from '../../../lib/errors/RetryError'; import BaseClientContextStub from '../.stubs/ClientContextStub'; import { ClientConfig } from '../../../lib/contracts/IClientContext'; import ConnectionProviderStub from '../.stubs/ConnectionProviderStub'; +import HttpRetryPolicy from '../../../lib/connection/connections/HttpRetryPolicy'; const sampleArrowSchema = Buffer.from([ 255, 255, 255, 255, 208, 0, 0, 0, 16, 0, 0, 0, 0, 0, 10, 0, 14, 0, 6, 0, 13, 0, 8, 0, 10, 0, 0, 0, 0, 0, 4, 0, 16, 0, @@ -721,4 +724,161 @@ describe('CloudFetchResultHandler', () => { // Ensure the batches array matches the expected structure expect(batches).to.deep.eq([sampleArrowBatch]); }); + + describe('download deadline', () => { + let server: http.Server | undefined; + + afterEach(async () => { + if (server) { + server.closeAllConnections(); + await new Promise((resolve) => { + server!.close(() => resolve()); + }); + server = undefined; + } + }); + + async function startServer(handler: http.RequestListener): Promise { + server = http.createServer(handler); + await new Promise((resolve) => { + server!.listen(0, '127.0.0.1', () => resolve()); + }); + return `http://127.0.0.1:${(server.address() as AddressInfo).port}/chunk`; + } + + // Real `HttpRetryPolicy` and node-fetch against a local server, so the deadline is exercised + // across actual attempts, backoff sleeps and sockets. + function createHandler(fileLink: string, configOverrides: Partial) { + const context = new BaseClientContextStub({ cloudFetchConcurrentDownloads: 1, ...configOverrides }); + sinon.stub(context.connectionProvider, 'getRetryPolicy').callsFake(async () => new HttpRetryPolicy(context)); + const rowSet: TRowSet = { + startRowOffset: new Int64(0), + rows: [], + resultLinks: [ + { + fileLink, + expiryTime: new Int64(defaultLinkExpiryTime), + rowCount: new Int64(1), + startRowOffset: new Int64(0), + bytesNum: new Int64(0), + }, + ], + }; + return new CloudFetchResultHandler(context, new ResultsProviderStub([rowSet], undefined), { + lz4Compressed: false, + status: { statusCode: TStatusCode.SUCCESS_STATUS }, + }); + } + + async function fetchWithin(handler: CloudFetchResultHandler, limitMs: number) { + const startedAt = Date.now(); + let guard: NodeJS.Timeout | undefined; + const outcome = await Promise.race([ + handler.fetchNext({ limit: 10000 }).then( + (value) => ({ value, error: undefined as any }), + (error) => ({ value: undefined, error }), + ), + new Promise<'pending'>((resolve) => { + guard = setTimeout(() => resolve('pending'), limitMs); + }), + ]); + clearTimeout(guard); + if (outcome === 'pending') { + throw new AssertionError(`fetchNext did not settle within ${limitMs}ms`); + } + return { ...outcome, elapsedMs: Date.now() - startedAt }; + } + + it('should fail a download whose response headers never arrive with a timeout error', async () => { + const closedRequests: Array> = []; + const url = await startServer((req) => { + // Accept the GET and never respond + closedRequests.push(new Promise((resolve) => req.socket.once('close', () => resolve()))); + }); + const handler = createHandler(url, { retriesTimeout: 300 }); + + const { error, elapsedMs } = await fetchWithin(handler, 1500); + + expect(error).to.be.instanceOf(RetryError); + expect(error.errorCode).to.be.equal(RetryErrorCode.TimeoutExceeded); + expect(error.message).to.be.equal('CloudFetch result download failed: Retry timeout exceeded'); + expect(elapsedMs).to.be.gte(250); + expect(elapsedMs).to.be.lt(1000); + // The stalled request is aborted, not left hanging + expect(closedRequests.length).to.be.equal(1); + await closedRequests[0]; + }); + + it('should fail a download whose response body stalls with a timeout error', async () => { + const url = await startServer((req, res) => { + res.writeHead(200, { 'Content-Length': '1000' }); + res.write(Buffer.alloc(10)); + // ...and never send the rest + }); + const handler = createHandler(url, { retriesTimeout: 300 }); + + const { error, elapsedMs } = await fetchWithin(handler, 1500); + + expect(error).to.be.instanceOf(RetryError); + expect(error.errorCode).to.be.equal(RetryErrorCode.TimeoutExceeded); + expect(elapsedMs).to.be.lt(1000); + }); + + it('should include retry backoff in the deadline and report the last error', async function () { + this.timeout(5000); + let hits = 0; + const url = await startServer((req, res) => { + hits += 1; + res.writeHead(503, 'Service Unavailable'); + res.end(); + }); + // The first backoff (2 * retryDelayMin = 2s) alone outlasts the 300ms budget + const handler = createHandler(url, { retriesTimeout: 300, retryDelayMin: 1000, retryMaxAttempts: 100 }); + + const { error, elapsedMs } = await fetchWithin(handler, 1500); + + expect(error).to.be.instanceOf(RetryError); + expect(error.errorCode).to.be.equal(RetryErrorCode.TimeoutExceeded); + expect(error.message).to.be.equal( + 'CloudFetch result download failed: Retry timeout exceeded (last error: HTTP 503: Service Unavailable)', + ); + expect(elapsedMs).to.be.lt(1000); + expect(hits).to.be.equal(1); + + // Once the backoff sleep ends, the retry loop must stop instead of sending another request + await new Promise((resolve) => { + setTimeout(resolve, 2200); + }); + expect(hits).to.be.equal(1); + }); + + it('should not fail a download that completes within the deadline', async () => { + const body = Buffer.concat([sampleArrowSchema, sampleArrowBatch]); + const url = await startServer((req, res) => { + // Slow but progressing: send the body in two parts + res.writeHead(200, { 'Content-Length': `${body.length}` }); + res.write(body.subarray(0, 10)); + setTimeout(() => res.end(body.subarray(10)), 100); + }); + const handler = createHandler(url, { retriesTimeout: 1000 }); + + const { value, error } = await fetchWithin(handler, 1500); + + expect(error).to.be.undefined; + expect(value?.batches).to.deep.eq([body]); + }); + + it('should not enforce a deadline when retriesTimeout is not positive', async () => { + const body = Buffer.concat([sampleArrowSchema, sampleArrowBatch]); + const url = await startServer((req, res) => { + setTimeout(() => res.end(body), 100); + }); + const handler = createHandler(url, { retriesTimeout: 0 }); + + const { value, error } = await fetchWithin(handler, 1500); + + expect(error).to.be.undefined; + expect(value?.batches).to.deep.eq([body]); + }); + }); }); From ec6c18103e9f4d672a9216411bc731b79380b67a Mon Sep 17 00:00:00 2001 From: Vu Anh Phung Date: Wed, 11 Nov 2026 23:46:06 +0000 Subject: [PATCH 2/3] fix: clamp an oversized retriesTimeout instead of disabling the CloudFetch deadline A retriesTimeout above the setTimeout limit (2^31-1 ms) silently turned the download deadline off. Clamp it to the limit instead. Co-authored-by: Isaac Signed-off-by: Vu Anh Phung --- lib/result/CloudFetchResultHandler.ts | 4 ++-- .../unit/result/CloudFetchResultHandler.test.ts | 16 ++++++++++++++++ 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/lib/result/CloudFetchResultHandler.ts b/lib/result/CloudFetchResultHandler.ts index e8251402..6d0d2d2a 100644 --- a/lib/result/CloudFetchResultHandler.ts +++ b/lib/result/CloudFetchResultHandler.ts @@ -17,7 +17,7 @@ import { safeEmit } from '../telemetry/telemetryUtils'; // unchanged while the unhandled-rejection leak is eliminated. type SettledDownload = { ok: true; batch: ArrowBatch } | { ok: false; error: unknown }; -// Largest delay `setTimeout` accepts; anything longer would fire immediately. +// Largest delay `setTimeout` accepts; longer values fire immediately, so they are clamped. const MAX_TIMER_DELAY_MS = 2 ** 31 - 1; export default class CloudFetchResultHandler implements IResultsProvider { @@ -160,7 +160,7 @@ export default class CloudFetchResultHandler implements IResultsProvider 0 && retriesTimeout <= MAX_TIMER_DELAY_MS ? retriesTimeout : undefined; + const deadlineMs = retriesTimeout > 0 ? Math.min(retriesTimeout, MAX_TIMER_DELAY_MS) : undefined; let controller: AbortController | undefined; let timeoutError: RetryError | undefined; let lastFailure: unknown; diff --git a/tests/unit/result/CloudFetchResultHandler.test.ts b/tests/unit/result/CloudFetchResultHandler.test.ts index 6b34f7d0..9a87637f 100644 --- a/tests/unit/result/CloudFetchResultHandler.test.ts +++ b/tests/unit/result/CloudFetchResultHandler.test.ts @@ -868,6 +868,22 @@ describe('CloudFetchResultHandler', () => { expect(value?.batches).to.deep.eq([body]); }); + it('should clamp a retriesTimeout that is too large for a timer instead of disabling the deadline', async () => { + const body = Buffer.concat([sampleArrowSchema, sampleArrowBatch]); + const url = await startServer((req, res) => { + res.end(body); + }); + const handler = createHandler(url, { retriesTimeout: 30 * 24 * 60 * 60 * 1000 }); + const setTimeoutSpy = sinon.spy(global, 'setTimeout'); + try { + const { error } = await fetchWithin(handler, 1500); + expect(error).to.be.undefined; + expect(setTimeoutSpy.calledWith(sinon.match.func, 2 ** 31 - 1)).to.be.true; + } finally { + setTimeoutSpy.restore(); + } + }); + it('should not enforce a deadline when retriesTimeout is not positive', async () => { const body = Buffer.concat([sampleArrowSchema, sampleArrowBatch]); const url = await startServer((req, res) => { From f211ce10d01b18fc23eb14e3964195437e520f17 Mon Sep 17 00:00:00 2001 From: "peco-engineer-bot[bot]" Date: Fri, 13 Nov 2026 06:44:13 +0000 Subject: [PATCH 3/3] engineer-bot: address @mention on PR #600 Signed-off-by: peco-engineer-bot[bot] Signed-off-by: Vu Anh Phung --- lib/connection/connections/HttpRetryPolicy.ts | 30 ++++++-- lib/connection/contracts/IRetryPolicy.ts | 4 +- lib/result/CloudFetchResultHandler.ts | 12 +-- .../connections/HttpRetryPolicy.test.ts | 73 +++++++++++++++++++ .../result/CloudFetchResultHandler.test.ts | 24 +++++- 5 files changed, 128 insertions(+), 15 deletions(-) diff --git a/lib/connection/connections/HttpRetryPolicy.ts b/lib/connection/connections/HttpRetryPolicy.ts index 0d2ecde8..3e6a2e53 100644 --- a/lib/connection/connections/HttpRetryPolicy.ts +++ b/lib/connection/connections/HttpRetryPolicy.ts @@ -3,9 +3,24 @@ import { HttpTransactionDetails } from '../contracts/IConnectionProvider'; import IClientContext from '../../contracts/IClientContext'; import RetryError, { RetryErrorCode } from '../../errors/RetryError'; -function delay(milliseconds: number): Promise { - return new Promise((resolve) => { - setTimeout(() => resolve(), milliseconds); +// Sleeps for `milliseconds`; aborting `signal` clears the timer and rejects with +// `signal.reason`, so a cancelled backoff does not keep the process alive. +function delay(milliseconds: number, signal?: AbortSignal): Promise { + return new Promise((resolve, reject) => { + if (signal?.aborted) { + reject(signal.reason); + return; + } + let timer: NodeJS.Timeout | undefined; + const onAbort = () => { + clearTimeout(timer); + reject(signal?.reason); + }; + timer = setTimeout(() => { + signal?.removeEventListener('abort', onAbort); + resolve(); + }, milliseconds); + signal?.addEventListener('abort', onAbort, { once: true }); }); } @@ -52,7 +67,10 @@ export default class HttpRetryPolicy implements IRetryPolicy): Promise { + public async invokeWithRetry( + operation: RetryableOperation, + signal?: AbortSignal, + ): Promise { for (;;) { // Capture either the resolved response or the thrown error so the // retry-decision logic below can flow without an early `continue` and @@ -73,7 +91,7 @@ export default class HttpRetryPolicy implements IRetryPolicy = () => Promise; export default interface IRetryPolicy { shouldRetry(details: R): Promise; - invokeWithRetry(operation: RetryableOperation): Promise; + // `signal` cancels a pending backoff sleep: aborting it rejects with `signal.reason` + // instead of waiting for the sleep to finish (the operation itself must honour the signal). + invokeWithRetry(operation: RetryableOperation, signal?: AbortSignal): Promise; } diff --git a/lib/result/CloudFetchResultHandler.ts b/lib/result/CloudFetchResultHandler.ts index 6d0d2d2a..578272b8 100644 --- a/lib/result/CloudFetchResultHandler.ts +++ b/lib/result/CloudFetchResultHandler.ts @@ -159,9 +159,11 @@ export default class CloudFetchResultHandler implements IResultsProvider 0 ? Math.min(retriesTimeout, MAX_TIMER_DELAY_MS) : undefined; - let controller: AbortController | undefined; + // Aborted only once, by the deadline, so a single controller serves every attempt and backoff. + const controller = deadlineMs === undefined ? undefined : new AbortController(); let timeoutError: RetryError | undefined; let lastFailure: unknown; let timer: NodeJS.Timeout | undefined; @@ -181,7 +183,6 @@ export default class CloudFetchResultHandler implements IResultsProvider((_, reject) => { timer = setTimeout(() => { timeoutError = new RetryError(RetryErrorCode.TimeoutExceeded, lastFailure); - controller?.abort(); + // Aborts the in-flight request, and rejects a pending backoff sleep with `timeoutError` + controller?.abort(timeoutError); reject(timeoutError); }, deadlineMs); }); diff --git a/tests/unit/connection/connections/HttpRetryPolicy.test.ts b/tests/unit/connection/connections/HttpRetryPolicy.test.ts index fe65adcb..35e0c0fc 100644 --- a/tests/unit/connection/connections/HttpRetryPolicy.test.ts +++ b/tests/unit/connection/connections/HttpRetryPolicy.test.ts @@ -400,5 +400,78 @@ describe('HttpRetryPolicy', () => { expect(operation.callCount).to.equal(clientConfig.retryMaxAttempts); } }); + + it('should cancel a pending backoff sleep when the signal is aborted', async () => { + const context = new ClientContextStub({ + retryDelayMin: 10000, + retryDelayMax: 60000, + retryMaxAttempts: 20, + }); + const policy = new HttpRetryPolicy(context); + const operation = sinon.stub().resolves({ + request: new Request('http://localhost'), + response: new Response(undefined, { status: 503 }), + }); + const controller = new AbortController(); + const reason = new Error('deadline'); + const clearTimeoutSpy = sinon.spy(global, 'clearTimeout'); + const setTimeoutSpy = sinon.spy(global, 'setTimeout'); + + try { + const invocation = policy.invokeWithRetry(operation, controller.signal); + // Let the first attempt complete and the 20s backoff sleep start + await new Promise((resolve) => { + setImmediate(resolve); + }); + const backoffCall = setTimeoutSpy.getCalls().find((call) => call.args[1] === 20000); + expect(backoffCall).to.not.be.undefined; + + controller.abort(reason); + + try { + await invocation; + expect.fail('It should reject with the abort reason'); + } catch (error) { + if (error instanceof AssertionError) { + throw error; + } + expect(error).to.equal(reason); + } + expect(clearTimeoutSpy.calledWith(backoffCall!.returnValue)).to.be.true; + expect(operation.callCount).to.equal(1); + } finally { + setTimeoutSpy.restore(); + clearTimeoutSpy.restore(); + } + }); + + it('should not start another attempt when the signal is already aborted before the backoff', async () => { + const context = new ClientContextStub({ + retryDelayMin: 10000, + retryDelayMax: 60000, + retryMaxAttempts: 20, + }); + const policy = new HttpRetryPolicy(context); + const controller = new AbortController(); + const reason = new Error('deadline'); + const operation = sinon.stub().callsFake(async () => { + controller.abort(reason); + return { + request: new Request('http://localhost'), + response: new Response(undefined, { status: 503 }), + }; + }); + + try { + await policy.invokeWithRetry(operation, controller.signal); + expect.fail('It should reject with the abort reason'); + } catch (error) { + if (error instanceof AssertionError) { + throw error; + } + expect(error).to.equal(reason); + } + expect(operation.callCount).to.equal(1); + }); }); }); diff --git a/tests/unit/result/CloudFetchResultHandler.test.ts b/tests/unit/result/CloudFetchResultHandler.test.ts index 9a87637f..4498d177 100644 --- a/tests/unit/result/CloudFetchResultHandler.test.ts +++ b/tests/unit/result/CloudFetchResultHandler.test.ts @@ -834,8 +834,26 @@ describe('CloudFetchResultHandler', () => { }); // The first backoff (2 * retryDelayMin = 2s) alone outlasts the 300ms budget const handler = createHandler(url, { retriesTimeout: 300, retryDelayMin: 1000, retryMaxAttempts: 100 }); + const setTimeoutSpy = sinon.spy(global, 'setTimeout'); + const clearTimeoutSpy = sinon.spy(global, 'clearTimeout'); - const { error, elapsedMs } = await fetchWithin(handler, 1500); + let outcome: Awaited>; + let backoffTimers: Array; + try { + outcome = await fetchWithin(handler, 1500); + backoffTimers = setTimeoutSpy + .getCalls() + .filter((call) => call.args[1] === 2000) + .map((call) => call.returnValue); + } finally { + setTimeoutSpy.restore(); + clearTimeoutSpy.restore(); + } + const { error, elapsedMs } = outcome; + + // The deadline cancels the pending backoff sleep instead of leaving its timer to keep Node alive + expect(backoffTimers).to.have.length(1); + expect(clearTimeoutSpy.calledWith(backoffTimers[0])).to.be.true; expect(error).to.be.instanceOf(RetryError); expect(error.errorCode).to.be.equal(RetryErrorCode.TimeoutExceeded); @@ -845,9 +863,9 @@ describe('CloudFetchResultHandler', () => { expect(elapsedMs).to.be.lt(1000); expect(hits).to.be.equal(1); - // Once the backoff sleep ends, the retry loop must stop instead of sending another request + // The cancelled backoff ends the retry loop instead of sending another request await new Promise((resolve) => { - setTimeout(resolve, 2200); + setTimeout(resolve, 100); }); expect(hits).to.be.equal(1); });