diff --git a/CHANGELOG.md b/CHANGELOG.md index 5eff56e6..e6c4b382 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ ## Unreleased - Reduce result chunking and `fetchAll()` overhead for large query results. +- Thrift CloudFetch downloads now retry a 401, 403 or 404 from cloud storage with the same URL, like 429 and 5xx, within the `retryMaxAttempts`/`retriesTimeout` budget; other 4xx responses still fail with `CloudFetch HTTP error `. `socketTimeout` is now also the inactivity timeout of each download attempt, so a stalled attempt is retried when `socketTimeout` is shorter than `retriesTimeout`. Previously a one-shot 401/403/404 failed the query and a stalled attempt waited out `retriesTimeout`. (#610) - Thrift CloudFetch downloads are now bounded end to end by `retriesTimeout` (default 15 minutes), including every attempt and retry backoff. Previously a stalled download, where the response headers or body never arrived, blocked the fetch forever. A download still running when the deadline expires is aborted and fails with `RetryError` (`TIMEOUT_EXCEEDED`, message `CloudFetch result download failed: Retry timeout exceeded`). (#520) - Thrift connections made through `proxy` now apply `checkServerCertificate`, `customCaCert` and `clientCert`/`clientKey`. Previously the proxied path ignored them and verified against Node's process-wide trust store, so a certificate outside the configured trust set could be accepted. **Compatibility:** proxied connections now verify unless `checkServerCertificate: false` is set; as on direct connections, `NODE_TLS_REJECT_UNAUTHORIZED=0` no longer turns verification off. For an `https` proxy these options also apply to the proxy's own certificate. (#485) - Honor the client `useCloudFetch: false` default on the kernel backend so complete results can be fetched without cloud-storage downloads. (#574) diff --git a/lib/DBSQLClient.ts b/lib/DBSQLClient.ts index 4256873a..e1bda5e0 100644 --- a/lib/DBSQLClient.ts +++ b/lib/DBSQLClient.ts @@ -742,6 +742,10 @@ export default class DBSQLClient extends EventEmitter implements IDBSQLClient, I if (options.retryDelayMax !== undefined) { this.config.retryDelayMax = options.retryDelayMax; } + // Also bounds each CloudFetch download attempt on the Thrift backend + if (options.socketTimeout !== undefined) { + this.config.socketTimeout = options.socketTimeout; + } // Override telemetry config if provided in options. Per-key narrowed copy // preserves the structural type system: `ConnectionOptions` and diff --git a/lib/connection/connections/HttpRetryPolicy.ts b/lib/connection/connections/HttpRetryPolicy.ts index 3e6a2e53..4c5d996d 100644 --- a/lib/connection/connections/HttpRetryPolicy.ts +++ b/lib/connection/connections/HttpRetryPolicy.ts @@ -191,10 +191,11 @@ export default class HttpRetryPolicy implements IRetryPolicy = new Set([401, 403, 404]); + export default class CloudFetchResultHandler implements IResultsProvider { private readonly context: IClientContext; @@ -157,7 +161,10 @@ export default class CloudFetchResultHandler implements IResultsProvider 0 ? Math.min(socketTimeout, MAX_TIMER_DELAY_MS) : undefined; const requestConfig: RequestInit = { agent, ...init }; // The retry policy checks `retriesTimeout` only between attempts, so a stalled attempt @@ -187,24 +194,48 @@ export default class CloudFetchResultHandler implements IResultsProvider attempt.abort(); + controller?.signal.addEventListener('abort', abortAttempt, { once: true }); + let attemptTimedOut = false; + let idleTimer: NodeJS.Timeout | undefined; + const restartIdleTimer = () => { + if (idleTimeoutMs !== undefined) { + clearTimeout(idleTimer); + idleTimer = setTimeout(() => { + attemptTimedOut = true; + attempt.abort(); + }, idleTimeoutMs); + } + }; + restartIdleTimer(); const request = new Request(url, { ...requestConfig, - signal: controller?.signal as unknown as RequestInit['signal'], + signal: attempt.signal as unknown as RequestInit['signal'], }); try { const response = await fetch(request); + restartIdleTimer(); if (response.ok) { - downloaded = await response.arrayBuffer(); + const bodyRead = response.arrayBuffer(); + response.body.on('data', restartIdleTimer); + downloaded = await bodyRead; } else { lastFailure = { request, response }; } - return { request, response }; + return { request, response, retryable: SAME_URL_RETRY_STATUSES.has(response.status) }; } catch (error) { if (timeoutError) { throw timeoutError; } - lastFailure = error; - throw error; + lastFailure = attemptTimedOut + ? Object.assign(new Error('CloudFetch download attempt timed out'), { code: 'ETIMEDOUT' }) + : error; + throw lastFailure; + } finally { + clearTimeout(idleTimer); + controller?.signal.removeEventListener('abort', abortAttempt); } }, controller?.signal); diff --git a/tests/unit/DBSQLClient.test.ts b/tests/unit/DBSQLClient.test.ts index 386f268f..c051920b 100644 --- a/tests/unit/DBSQLClient.test.ts +++ b/tests/unit/DBSQLClient.test.ts @@ -879,6 +879,7 @@ describe('DBSQLClient retry-policy ConnectionOptions', () => { expect(client.getConfig().retriesTimeout).to.equal(15 * 60 * 1000); expect(client.getConfig().retryDelayMin).to.equal(1000); expect(client.getConfig().retryDelayMax).to.equal(60 * 1000); + expect(client.getConfig().socketTimeout).to.equal(15 * 60 * 1000); await client.connect({ ...connectOptions, @@ -886,12 +887,23 @@ describe('DBSQLClient retry-policy ConnectionOptions', () => { retriesTimeout: 5000, retryDelayMin: 100, retryDelayMax: 500, + socketTimeout: 30000, }); expect(client.getConfig().retryMaxAttempts).to.equal(2); expect(client.getConfig().retriesTimeout).to.equal(5000); expect(client.getConfig().retryDelayMin).to.equal(100); expect(client.getConfig().retryDelayMax).to.equal(500); + // Read by the Thrift CloudFetch download idle timeout + expect(client.getConfig().socketTimeout).to.equal(30000); + }); + + it('ingests socketTimeout: 0 from connect() into ClientConfig', async () => { + const client = new DBSQLClient(); + + await client.connect({ ...connectOptions, socketTimeout: 0 }); + + expect(client.getConfig().socketTimeout).to.equal(0); }); it('keeps defaults when retry-policy options are omitted', async () => { @@ -903,6 +915,7 @@ describe('DBSQLClient retry-policy ConnectionOptions', () => { expect(client.getConfig().retriesTimeout).to.equal(15 * 60 * 1000); expect(client.getConfig().retryDelayMin).to.equal(1000); expect(client.getConfig().retryDelayMax).to.equal(60 * 1000); + expect(client.getConfig().socketTimeout).to.equal(15 * 60 * 1000); }); }); diff --git a/tests/unit/connection/connections/HttpRetryPolicy.test.ts b/tests/unit/connection/connections/HttpRetryPolicy.test.ts index 35e0c0fc..0577c9c9 100644 --- a/tests/unit/connection/connections/HttpRetryPolicy.test.ts +++ b/tests/unit/connection/connections/HttpRetryPolicy.test.ts @@ -103,6 +103,29 @@ describe('HttpRetryPolicy', () => { expect(result.shouldRetry).to.be.false; }); + it('should retry a response the caller marks retryable, within the attempts budget', async () => { + const context = new ClientContextStub({ retryMaxAttempts: 3 }); + const policy = new HttpRetryPolicy(context); + const forbidden = () => ({ + request: new Request('http://localhost'), + response: new Response(undefined, { status: 403 }), + }); + + expect((await policy.shouldRetry(forbidden())).shouldRetry).to.be.false; + expect((await policy.shouldRetry({ ...forbidden(), retryable: true })).shouldRetry).to.be.true; + expect((await policy.shouldRetry({ ...forbidden(), retryable: true })).shouldRetry).to.be.true; + try { + await policy.shouldRetry({ ...forbidden(), retryable: true }); + expect.fail('It should throw an error'); + } catch (error) { + if (error instanceof AssertionError) { + throw error; + } + expect(error).to.be.instanceOf(RetryError); + expect((error as RetryError).errorCode).to.equal(RetryErrorCode.AttemptsExceeded); + } + }); + it('should use `Retry-After` header as a base for backoff', async () => { const context = new ClientContextStub({ retryDelayMin: 3, retryDelayMax: 100, retryMaxAttempts: 10 }); const policy = new HttpRetryPolicy(context); diff --git a/tests/unit/result/CloudFetchResultHandler.test.ts b/tests/unit/result/CloudFetchResultHandler.test.ts index 4498d177..0539e13c 100644 --- a/tests/unit/result/CloudFetchResultHandler.test.ts +++ b/tests/unit/result/CloudFetchResultHandler.test.ts @@ -902,6 +902,140 @@ describe('CloudFetchResultHandler', () => { } }); + describe('recovery', () => { + const body = Buffer.concat([sampleArrowSchema, sampleArrowBatch]); + const fastRetries = { retryDelayMin: 10, retryDelayMax: 20 }; + + [401, 403, 404].forEach((status) => { + it(`should retry a one-shot HTTP ${status} with the same URL`, async () => { + const hits: Array = []; + const url = await startServer((req, res) => { + hits.push(req.url ?? ''); + if (hits.length === 1) { + res.writeHead(status); + res.end(); + } else { + res.end(body); + } + }); + const handler = createHandler(`${url}?sig=a`, fastRetries); + + const { value, error } = await fetchWithin(handler, 1500); + + expect(error).to.be.undefined; + expect(value?.batches).to.deep.eq([body]); + expect(hits).to.deep.eq(['/chunk?sig=a', '/chunk?sig=a']); + }); + }); + + it('should fail with the HTTP status once a persistent 403 exhausts the retries', async () => { + let hits = 0; + const url = await startServer((req, res) => { + hits += 1; + res.writeHead(403); + res.end(); + }); + const handler = createHandler(url, { ...fastRetries, retryMaxAttempts: 3 }); + + const { error } = await fetchWithin(handler, 1500); + + expect(error).to.be.instanceOf(RetryError); + expect(error.errorCode).to.be.equal(RetryErrorCode.AttemptsExceeded); + expect(error.message).to.be.equal( + 'CloudFetch result download failed: Max retry count exceeded (last error: HTTP 403: Forbidden)', + ); + expect(hits).to.be.equal(3); + }); + + [ + [400, 'Bad Request'], + [405, 'Method Not Allowed'], + [412, 'Precondition Failed'], + ].forEach(([status, statusText]) => { + it(`should fail at once with the HTTP error on ${status}`, async () => { + let hits = 0; + const url = await startServer((req, res) => { + hits += 1; + res.writeHead(status as number); + res.end(); + }); + const handler = createHandler(url, fastRetries); + + const { error } = await fetchWithin(handler, 1500); + + expect(error.message).to.be.equal(`CloudFetch HTTP error ${status} ${statusText}`); + expect(hits).to.be.equal(1); + }); + }); + + it('should time out a stalled attempt after socketTimeout and retry it', async () => { + let hits = 0; + const url = await startServer((req, res) => { + hits += 1; + if (hits > 1) { + res.end(body); + } + // ...the first request never gets a response + }); + const handler = createHandler(url, { ...fastRetries, socketTimeout: 200 }); + + const { value, error, elapsedMs } = await fetchWithin(handler, 1500); + + expect(error).to.be.undefined; + expect(value?.batches).to.deep.eq([body]); + expect(hits).to.be.equal(2); + expect(elapsedMs).to.be.gte(150); + }); + + it('should report a timed-out last attempt without the presigned URL', async () => { + const url = await startServer(() => { + // never respond + }); + const handler = createHandler(`${url}?sig=SECRET_SIGNATURE`, { + ...fastRetries, + socketTimeout: 100, + retryMaxAttempts: 2, + }); + + const { error } = await fetchWithin(handler, 1500); + + expect(error).to.be.instanceOf(RetryError); + expect(error.errorCode).to.be.equal(RetryErrorCode.AttemptsExceeded); + expect(error.message).to.be.equal( + 'CloudFetch result download failed: Max retry count exceeded (last error: network error ETIMEDOUT)', + ); + expect(error.message).to.not.contain('SECRET_SIGNATURE'); + }); + + it('should not time out an attempt that keeps receiving data', async function () { + this.timeout(5000); + let hits = 0; + const url = await startServer((req, res) => { + hits += 1; + res.writeHead(200, { 'Content-Length': `${body.length}` }); + // Slower than socketTimeout in total, but never idle for that long + let sent = 0; + const interval = setInterval(() => { + const next = Math.min(sent + 20, body.length); + res.write(body.subarray(sent, next)); + sent = next; + if (sent === body.length) { + clearInterval(interval); + res.end(); + } + }, 50); + }); + const handler = createHandler(url, { socketTimeout: 300 }); + + const { value, error, elapsedMs } = await fetchWithin(handler, 4000); + + expect(error).to.be.undefined; + expect(value?.batches).to.deep.eq([body]); + expect(hits).to.be.equal(1); + expect(elapsedMs).to.be.gte(300); + }); + }); + it('should not enforce a deadline when retriesTimeout is not positive', async () => { const body = Buffer.concat([sampleArrowSchema, sampleArrowBatch]); const url = await startServer((req, res) => {