From 2e7bf21d31bc442759c14f1e33da20472d60a464 Mon Sep 17 00:00:00 2001 From: Vu Anh Phung Date: Fri, 13 Nov 2026 08:37:47 +0000 Subject: [PATCH 01/10] fix: recover Thrift CloudFetch downloads from rejected URLs, 4xx and stalls A one-shot 401, 403 or 404 from cloud storage failed the query: CloudFetchResultHandler threw on any non-OK response and HttpRetryPolicy retries only 429 and 5xx. Other 4xx responses failed the same way, and a stalled attempt was never timed out, so it waited until retriesTimeout instead of being retried. On 401/403/404 the presigned URL itself was rejected, so fetch a fresh link for the same rows with FetchResults(startRowOffset) and download it, counting each refresh against retryMaxAttempts. RowSetProvider serializes its FetchResults calls and pages by explicit offset after a refresh, since the refresh may move the server cursor. Retry other 4xx with the same URL, and apply socketTimeout to each download attempt so a stalled one is retried. Signed-off-by: Vu Anh Phung Co-authored-by: Isaac --- CHANGELOG.md | 1 + lib/DBSQLClient.ts | 4 + lib/connection/connections/HttpConnection.ts | 6 +- lib/connection/connections/HttpRetryPolicy.ts | 15 +- .../contracts/IConnectionProvider.ts | 10 +- lib/contracts/IDBSQLClient.ts | 4 + lib/result/CloudFetchResultHandler.ts | 72 +++++++- lib/result/RowSetProvider.ts | 60 ++++++- tests/unit/.stubs/ConnectionProviderStub.ts | 8 +- .../connections/HttpRetryPolicy.test.ts | 23 +++ .../result/CloudFetchResultHandler.test.ts | 156 ++++++++++++++++++ tests/unit/result/RowSetProvider.test.ts | 81 ++++++++- 12 files changed, 418 insertions(+), 22 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5eff56e6..92f6695b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ ## Unreleased - Reduce result chunking and `fetchAll()` overhead for large query results. +- Thrift CloudFetch downloads now recover from more failures. When cloud storage rejects a presigned URL with 401, 403 or 404 (an expired or invalid signature), the driver fetches a fresh link for the same rows from the server and retries with it; each refresh counts as an attempt against `retryMaxAttempts`. Other 4xx responses are retried with the same URL. `socketTimeout` now also bounds each download attempt, so a stalled attempt is retried instead of waiting until `retriesTimeout`. Previously these failures failed the query, and a stalled attempt waited out the whole deadline. - Thrift CloudFetch downloads are now bounded end to end by `retriesTimeout` (default 15 minutes), including every attempt and retry backoff. Previously a stalled download, where the response headers or body never arrived, blocked the fetch forever. A download still running when the deadline expires is aborted and fails with `RetryError` (`TIMEOUT_EXCEEDED`, message `CloudFetch result download failed: Retry timeout exceeded`). (#520) - Thrift connections made through `proxy` now apply `checkServerCertificate`, `customCaCert` and `clientCert`/`clientKey`. Previously the proxied path ignored them and verified against Node's process-wide trust store, so a certificate outside the configured trust set could be accepted. **Compatibility:** proxied connections now verify unless `checkServerCertificate: false` is set; as on direct connections, `NODE_TLS_REJECT_UNAUTHORIZED=0` no longer turns verification off. For an `https` proxy these options also apply to the proxy's own certificate. (#485) - Honor the client `useCloudFetch: false` default on the kernel backend so complete results can be fetched without cloud-storage downloads. (#574) diff --git a/lib/DBSQLClient.ts b/lib/DBSQLClient.ts index 4256873a..e1bda5e0 100644 --- a/lib/DBSQLClient.ts +++ b/lib/DBSQLClient.ts @@ -742,6 +742,10 @@ export default class DBSQLClient extends EventEmitter implements IDBSQLClient, I if (options.retryDelayMax !== undefined) { this.config.retryDelayMax = options.retryDelayMax; } + // Also bounds each CloudFetch download attempt on the Thrift backend + if (options.socketTimeout !== undefined) { + this.config.socketTimeout = options.socketTimeout; + } // Override telemetry config if provided in options. Per-key narrowed copy // preserves the structural type system: `ConnectionOptions` and diff --git a/lib/connection/connections/HttpConnection.ts b/lib/connection/connections/HttpConnection.ts index 6472265a..5dc1a3ce 100644 --- a/lib/connection/connections/HttpConnection.ts +++ b/lib/connection/connections/HttpConnection.ts @@ -4,7 +4,7 @@ import http from 'http'; import { HeadersInit } from 'node-fetch'; import { ProxyAgent } from 'proxy-agent'; -import IConnectionProvider, { HttpTransactionDetails } from '../contracts/IConnectionProvider'; +import IConnectionProvider, { HttpRetryPolicyOptions, HttpTransactionDetails } from '../contracts/IConnectionProvider'; import IConnectionOptions, { ProxyOptions } from '../contracts/IConnectionOptions'; import IClientContext from '../../contracts/IClientContext'; import { LogLevel } from '../../contracts/IDBSQLLogger'; @@ -227,7 +227,7 @@ export default class HttpConnection implements IConnectionProvider { return this.connection; } - public async getRetryPolicy(): Promise> { - return new HttpRetryPolicy(this.context); + public async getRetryPolicy(options?: HttpRetryPolicyOptions): Promise> { + return new HttpRetryPolicy(this.context, options); } } diff --git a/lib/connection/connections/HttpRetryPolicy.ts b/lib/connection/connections/HttpRetryPolicy.ts index 3e6a2e53..41cb4633 100644 --- a/lib/connection/connections/HttpRetryPolicy.ts +++ b/lib/connection/connections/HttpRetryPolicy.ts @@ -1,5 +1,5 @@ import IRetryPolicy, { ShouldRetryResult, RetryableOperation } from '../contracts/IRetryPolicy'; -import { HttpTransactionDetails } from '../contracts/IConnectionProvider'; +import { HttpRetryPolicyOptions, HttpTransactionDetails } from '../contracts/IConnectionProvider'; import IClientContext from '../../contracts/IClientContext'; import RetryError, { RetryErrorCode } from '../../errors/RetryError'; @@ -53,10 +53,13 @@ export default class HttpRetryPolicy implements IRetryPolicy { @@ -193,6 +196,7 @@ export default class HttpRetryPolicy implements IRetryPolicy= 500 && statusCode !== 501); + (statusCode >= 500 && statusCode !== 501) || + // ...and, when enabled, on the other `4xx` codes + (retryClientErrorsExcept !== undefined && + statusCode >= 400 && + statusCode < 500 && + !retryClientErrorsExcept.has(statusCode)); return result; } diff --git a/lib/connection/contracts/IConnectionProvider.ts b/lib/connection/contracts/IConnectionProvider.ts index a81bbe82..935ead37 100644 --- a/lib/connection/contracts/IConnectionProvider.ts +++ b/lib/connection/contracts/IConnectionProvider.ts @@ -7,6 +7,14 @@ export interface HttpTransactionDetails { response: Response; } +export interface HttpRetryPolicyOptions { + /** + * Also retry 4xx responses, except these statuses. For idempotent GETs of presigned + * URLs (CloudFetch), where a 4xx from storage does not mean the request was wrong. + */ + retryClientErrorsExcept?: ReadonlySet; +} + export default interface IConnectionProvider { getThriftConnection(): Promise; @@ -18,5 +26,5 @@ export default interface IConnectionProvider { setHeaders(headers: HeadersInit): void; - getRetryPolicy(): Promise>; + getRetryPolicy(options?: HttpRetryPolicyOptions): Promise>; } diff --git a/lib/contracts/IDBSQLClient.ts b/lib/contracts/IDBSQLClient.ts index 5c302bb0..2d6c3929 100644 --- a/lib/contracts/IDBSQLClient.ts +++ b/lib/contracts/IDBSQLClient.ts @@ -70,6 +70,10 @@ export type ConnectionOptions = { port?: number; path: string; userAgentEntry?: string; + /** + * Request timeout in milliseconds. On the Thrift backend it also bounds each CloudFetch + * download attempt; an attempt that runs out is retried. Default 900000 (15 minutes). + */ socketTimeout?: number; proxy?: ProxyOptions; enableMetricViewMetadata?: boolean; diff --git a/lib/result/CloudFetchResultHandler.ts b/lib/result/CloudFetchResultHandler.ts index fbfe5c7c..36baf436 100644 --- a/lib/result/CloudFetchResultHandler.ts +++ b/lib/result/CloudFetchResultHandler.ts @@ -1,4 +1,5 @@ import fetch, { RequestInfo, RequestInit, Request, Response } from 'node-fetch'; +import Int64 from 'node-int64'; import { TGetResultSetMetadataResp, TRowSet, TSparkArrowResultLink } from '../../thrift/TCLIService_types'; import HiveDriverError from '../errors/HiveDriverError'; import RetryError, { RetryErrorCode } from '../errors/RetryError'; @@ -20,6 +21,19 @@ type SettledDownload = { ok: true; batch: ArrowBatch } | { ok: false; error: unk // Largest delay `setTimeout` accepts; longer values fire immediately, so they are clamped. const MAX_TIMER_DELAY_MS = 2 ** 31 - 1; +// Statuses that mean the presigned URL itself was rejected (expired or invalid signature; GCS +// answers 404 for an expired one). Retrying the same URL can't succeed, so the link is refreshed. +// Other 4xx responses are retried with the same URL. +const LINK_REFRESH_STATUSES: ReadonlySet = new Set([401, 403, 404]); + +interface LinkRefresher { + refreshLinks(startRowOffset: Int64, maxRows: Int64): Promise>; +} + +function canRefreshLinks(source: unknown): source is LinkRefresher { + return typeof (source as Partial | undefined)?.refreshLinks === 'function'; +} + export default class CloudFetchResultHandler implements IResultsProvider { private readonly context: IClientContext; @@ -128,7 +142,20 @@ export default class CloudFetchResultHandler implements IResultsProvider { + if (!canRefreshLinks(this.source)) { + return undefined; + } + const offset = link.startRowOffset.toNumber(); + this.context + .getLogger() + .log(LogLevel.debug, `CloudFetch download got HTTP ${status}; refreshing the link at row offset ${offset}`); + const links = await this.source.refreshLinks(link.startRowOffset, link.rowCount); + return links.find((candidate) => candidate.startRowOffset.toNumber() === offset); + } + private async fetch(url: RequestInfo, init?: RequestInit): Promise<{ response: Response; body?: ArrayBuffer }> { const connectionProvider = await this.context.getConnectionProvider(); let requestUrl = url; @@ -156,8 +196,11 @@ export default class CloudFetchResultHandler implements IResultsProvider 0 ? Math.min(socketTimeout, MAX_TIMER_DELAY_MS) : undefined; const requestConfig: RequestInit = { agent, ...init }; // The retry policy checks `retriesTimeout` only between attempts, so a stalled attempt @@ -187,9 +230,21 @@ export default class CloudFetchResultHandler implements IResultsProvider attempt.abort(); + controller?.signal.addEventListener('abort', abortAttempt, { once: true }); + let attemptTimedOut = false; + const attemptTimer = + attemptTimeoutMs === undefined + ? undefined + : setTimeout(() => { + attemptTimedOut = true; + attempt.abort(); + }, attemptTimeoutMs); const request = new Request(url, { ...requestConfig, - signal: controller?.signal as unknown as RequestInit['signal'], + signal: attempt.signal as unknown as RequestInit['signal'], }); try { const response = await fetch(request); @@ -203,8 +258,13 @@ export default class CloudFetchResultHandler implements IResultsProvider = Promise.resolve(); + + private serializeFetch(fetchResults: () => Promise): Promise { + const result = this.lastFetch.then(fetchResults, fetchResults); + this.lastFetch = result.catch(() => undefined); + return result; + } + private get hasMoreRows(): boolean { // `hasMoreRowsFlag` is populated only after fetching the first row set. // Prior to that, we use a `operationHandle.hasResultSet` flag which @@ -129,9 +145,38 @@ export default class RowSetProvider implements IResultsProvider { + this.nextLinkRowOffset = Math.max( + this.nextLinkRowOffset, + link.startRowOffset.toNumber() + link.rowCount.toNumber(), + ); + }); return response.results; } + /** + * Fetches fresh result links starting at `startRowOffset`, e.g. to replace a presigned + * URL that cloud storage rejected. Returns no links when the operation is already closed. + */ + public async refreshLinks(startRowOffset: Int64, maxRows: Int64): Promise> { + if (this.returnOnlyPrefetchedResults) { + return []; + } + const driver = await this.context.getDriver(); + const response = await this.serializeFetch(async () => { + this.fetchByOffset = true; + return driver.fetchResults({ + operationHandle: this.operationHandle, + orientation: TFetchOrientation.FETCH_NEXT, + maxRows, + fetchType: FetchType.Data, + startRowOffset, + }); + }); + Status.assert(response.status); + return response.results?.resultLinks ?? []; + } + public async fetchNext({ limit }: ResultsProviderFetchNextOptions) { const prefetchedResponse = this.prefetchedResults.shift(); if (prefetchedResponse) { @@ -150,12 +195,15 @@ export default class RowSetProvider implements IResultsProvider + driver.fetchResults({ + operationHandle: this.operationHandle, + orientation: this.fetchOrientation, + maxRows: new Int64(limit), + fetchType: FetchType.Data, + ...(this.fetchByOffset ? { startRowOffset: new Int64(this.nextLinkRowOffset) } : {}), + }), + ); const latencyMs = Date.now() - startTime; this.emitChunkEvent(latencyMs, response); diff --git a/tests/unit/.stubs/ConnectionProviderStub.ts b/tests/unit/.stubs/ConnectionProviderStub.ts index 20b54dfd..579d441b 100644 --- a/tests/unit/.stubs/ConnectionProviderStub.ts +++ b/tests/unit/.stubs/ConnectionProviderStub.ts @@ -1,6 +1,9 @@ import http from 'http'; import { HeadersInit } from 'node-fetch'; -import IConnectionProvider, { HttpTransactionDetails } from '../../../lib/connection/contracts/IConnectionProvider'; +import IConnectionProvider, { + HttpRetryPolicyOptions, + HttpTransactionDetails, +} from '../../../lib/connection/contracts/IConnectionProvider'; import IRetryPolicy from '../../../lib/connection/contracts/IRetryPolicy'; import NullRetryPolicy from '../../../lib/connection/connections/NullRetryPolicy'; @@ -19,7 +22,8 @@ export default class ConnectionProviderStub implements IConnectionProvider { this.headers = headers; } - public async getRetryPolicy(): Promise> { + // eslint-disable-next-line @typescript-eslint/no-unused-vars + public async getRetryPolicy(options?: HttpRetryPolicyOptions): Promise> { return new NullRetryPolicy(); } } diff --git a/tests/unit/connection/connections/HttpRetryPolicy.test.ts b/tests/unit/connection/connections/HttpRetryPolicy.test.ts index 35e0c0fc..96ccece0 100644 --- a/tests/unit/connection/connections/HttpRetryPolicy.test.ts +++ b/tests/unit/connection/connections/HttpRetryPolicy.test.ts @@ -103,6 +103,29 @@ describe('HttpRetryPolicy', () => { expect(result.shouldRetry).to.be.false; }); + it('should retry 4xx responses only when enabled, except the listed statuses', async () => { + function createStub(status: number) { + return { + request: new Request('http://localhost'), + response: new Response(undefined, { status }), + }; + } + + const context = new ClientContextStub({ retryMaxAttempts: 100 }); + const defaultPolicy = new HttpRetryPolicy(context); + const cloudFetchPolicy = new HttpRetryPolicy(context, { retryClientErrorsExcept: new Set([401, 403, 404]) }); + + for (const status of [400, 405, 408, 412]) { + expect((await defaultPolicy.shouldRetry(createStub(status))).shouldRetry, `${status}`).to.be.false; + expect((await cloudFetchPolicy.shouldRetry(createStub(status))).shouldRetry, `${status}`).to.be.true; + } + for (const status of [401, 403, 404]) { + expect((await cloudFetchPolicy.shouldRetry(createStub(status))).shouldRetry, `${status}`).to.be.false; + } + expect((await cloudFetchPolicy.shouldRetry(createStub(501))).shouldRetry).to.be.false; + expect((await cloudFetchPolicy.shouldRetry(createStub(503))).shouldRetry).to.be.true; + }); + it('should use `Retry-After` header as a base for backoff', async () => { const context = new ClientContextStub({ retryDelayMin: 3, retryDelayMax: 100, retryMaxAttempts: 10 }); const policy = new HttpRetryPolicy(context); diff --git a/tests/unit/result/CloudFetchResultHandler.test.ts b/tests/unit/result/CloudFetchResultHandler.test.ts index 4498d177..7e50ad36 100644 --- a/tests/unit/result/CloudFetchResultHandler.test.ts +++ b/tests/unit/result/CloudFetchResultHandler.test.ts @@ -902,6 +902,162 @@ describe('CloudFetchResultHandler', () => { } }); + describe('recovery', () => { + const body = Buffer.concat([sampleArrowSchema, sampleArrowBatch]); + + function makeLink(fileLink: string): TSparkArrowResultLink { + return { + fileLink, + expiryTime: new Int64(defaultLinkExpiryTime), + rowCount: new Int64(1), + startRowOffset: new Int64(7), + bytesNum: new Int64(0), + }; + } + + // Like `createHandler`, with a source that can refresh links + function createRefreshingHandler( + link: TSparkArrowResultLink, + refreshedLinks: () => Array, + configOverrides: Partial = {}, + ) { + const context = new BaseClientContextStub({ cloudFetchConcurrentDownloads: 1, ...configOverrides }); + sinon + .stub(context.connectionProvider, 'getRetryPolicy') + .callsFake(async (options) => new HttpRetryPolicy(context, options)); + const source = new ResultsProviderStub( + [{ startRowOffset: new Int64(0), rows: [], resultLinks: [link] }], + undefined, + ); + const refreshLinks = sinon.stub().callsFake(async () => refreshedLinks()); + const handler = new CloudFetchResultHandler(context, Object.assign(source, { refreshLinks }), { + lz4Compressed: false, + status: { statusCode: TStatusCode.SUCCESS_STATUS }, + }); + return { handler, refreshLinks }; + } + + [401, 403, 404].forEach((status) => { + it(`should refresh the link and download the fresh URL after HTTP ${status}`, async () => { + const hits: Array = []; + const url = await startServer((req, res) => { + hits.push(req.url ?? ''); + if (req.url?.includes('sig=old')) { + res.writeHead(status); + res.end(); + } else { + res.end(body); + } + }); + const { handler, refreshLinks } = createRefreshingHandler(makeLink(`${url}?sig=old`), () => [ + makeLink(`${url}?sig=new`), + ]); + + const { value, error } = await fetchWithin(handler, 1500); + + expect(error).to.be.undefined; + expect(value?.batches).to.deep.eq([body]); + expect(hits).to.deep.eq(['/chunk?sig=old', '/chunk?sig=new']); + expect(refreshLinks.callCount).to.be.equal(1); + expect(refreshLinks.firstCall.args[0].toNumber()).to.be.equal(7); + expect(refreshLinks.firstCall.args[1].toNumber()).to.be.equal(1); + }); + }); + + it('should fail with the HTTP error when the source cannot refresh links', async () => { + const url = await startServer((req, res) => { + res.writeHead(403, 'Forbidden'); + res.end(); + }); + const handler = createHandler(url, {}); + + const { error } = await fetchWithin(handler, 1500); + + expect(error.message).to.be.equal('CloudFetch HTTP error 403 Forbidden'); + }); + + it('should count link refreshes against retryMaxAttempts', async () => { + const url = await startServer((req, res) => { + res.writeHead(403, 'Forbidden'); + res.end(); + }); + const { handler, refreshLinks } = createRefreshingHandler( + makeLink(`${url}?sig=0`), + () => [makeLink(`${url}?sig=${Math.random()}`)], + { retryMaxAttempts: 3 }, + ); + + const { error } = await fetchWithin(handler, 1500); + + expect(error.message).to.be.equal('CloudFetch HTTP error 403 Forbidden'); + expect(refreshLinks.callCount).to.be.equal(2); + }); + + it('should retry other 4xx responses with the same URL', async () => { + const hits: Array = []; + const url = await startServer((req, res) => { + hits.push(req.url ?? ''); + if (hits.length === 1) { + res.writeHead(412, 'Precondition Failed'); + res.end(); + } else { + res.end(body); + } + }); + const { handler, refreshLinks } = createRefreshingHandler(makeLink(`${url}?sig=a`), () => [], { + retryDelayMin: 10, + retryDelayMax: 20, + }); + + const { value, error } = await fetchWithin(handler, 1500); + + expect(error).to.be.undefined; + expect(value?.batches).to.deep.eq([body]); + expect(hits).to.deep.eq(['/chunk?sig=a', '/chunk?sig=a']); + expect(refreshLinks.called).to.be.false; + }); + + it('should time out a stalled attempt after socketTimeout and retry it', async () => { + let hits = 0; + const url = await startServer((req, res) => { + hits += 1; + if (hits > 1) { + res.end(body); + } + // ...the first request never gets a response + }); + const handler = createHandler(url, { socketTimeout: 200, retryDelayMin: 10, retryDelayMax: 20 }); + + const { value, error, elapsedMs } = await fetchWithin(handler, 1500); + + expect(error).to.be.undefined; + expect(value?.batches).to.deep.eq([body]); + expect(hits).to.be.equal(2); + expect(elapsedMs).to.be.gte(150); + }); + + it('should report a timed-out last attempt without the presigned URL', async () => { + const url = await startServer(() => { + // never respond + }); + const handler = createHandler(`${url}?sig=SECRET_SIGNATURE`, { + socketTimeout: 100, + retryMaxAttempts: 2, + retryDelayMin: 10, + retryDelayMax: 20, + }); + + const { error } = await fetchWithin(handler, 1500); + + expect(error).to.be.instanceOf(RetryError); + expect(error.errorCode).to.be.equal(RetryErrorCode.AttemptsExceeded); + expect(error.message).to.be.equal( + 'CloudFetch result download failed: Max retry count exceeded (last error: network error ETIMEDOUT)', + ); + expect(error.message).to.not.contain('SECRET_SIGNATURE'); + }); + }); + it('should not enforce a deadline when retriesTimeout is not positive', async () => { const body = Buffer.concat([sampleArrowSchema, sampleArrowBatch]); const url = await startServer((req, res) => { diff --git a/tests/unit/result/RowSetProvider.test.ts b/tests/unit/result/RowSetProvider.test.ts index f2aebe44..ab29bff1 100644 --- a/tests/unit/result/RowSetProvider.test.ts +++ b/tests/unit/result/RowSetProvider.test.ts @@ -11,7 +11,13 @@ import { expect } from 'chai'; import sinon from 'sinon'; import Int64 from 'node-int64'; -import { TOperationHandle, TOperationType, THandleIdentifier } from '../../../thrift/TCLIService_types'; +import { + TFetchResultsResp, + TOperationHandle, + TOperationType, + THandleIdentifier, + TStatusCode, +} from '../../../thrift/TCLIService_types'; import RowSetProvider from '../../../lib/result/RowSetProvider'; import ClientContextStub from '../.stubs/ClientContextStub'; @@ -159,4 +165,77 @@ describe('RowSetProvider', () => { expect(emitCloudFetchChunk.firstCall.args[0].bytes).to.equal(64); }); }); + + describe('link refresh', () => { + function linksResponse(fileLink: string, startRowOffset: number, rowCount: number): TFetchResultsResp { + return { + status: { statusCode: TStatusCode.SUCCESS_STATUS }, + hasMoreRows: true, + results: { + startRowOffset: new Int64(startRowOffset), + rows: [], + resultLinks: [ + { + fileLink, + expiryTime: new Int64(Date.now() + 60_000), + startRowOffset: new Int64(startRowOffset), + rowCount: new Int64(rowCount), + bytesNum: new Int64(0), + }, + ], + }, + }; + } + + it('fetches fresh links from the offset, then pages by offset', async () => { + const context = new ClientContextStub(); + const fetchResults = sinon.spy(context.driver, 'fetchResults'); + const provider = new RowSetProvider(context, makeOperationHandle(), [], false); + + context.driver.fetchResultsResp = linksResponse('https://storage/0?sig=old', 0, 100); + await provider.fetchNext({ limit: 1000 }); + expect(fetchResults.firstCall.args[0].startRowOffset).to.be.undefined; + + context.driver.fetchResultsResp = linksResponse('https://storage/0?sig=new', 0, 100); + const links = await provider.refreshLinks(new Int64(0), new Int64(100)); + expect(links.map((link) => link.fileLink)).to.deep.equal(['https://storage/0?sig=new']); + expect(fetchResults.secondCall.args[0].startRowOffset?.toNumber()).to.equal(0); + expect(fetchResults.secondCall.args[0].maxRows.toNumber()).to.equal(100); + + // The refresh may have moved the server cursor, so the next page is requested by offset + context.driver.fetchResultsResp = linksResponse('https://storage/100?sig=a', 100, 50); + await provider.fetchNext({ limit: 1000 }); + expect(fetchResults.thirdCall.args[0].startRowOffset?.toNumber()).to.equal(100); + }); + + it('serializes a refresh with a page fetch', async () => { + const context = new ClientContextStub(); + const provider = new RowSetProvider(context, makeOperationHandle(), [], false); + context.driver.fetchResultsResp = linksResponse('https://storage/0?sig=old', 0, 100); + await provider.fetchNext({ limit: 1000 }); + + let inFlight = 0; + let maxInFlight = 0; + sinon.stub(context.driver, 'fetchResults').callsFake(async () => { + inFlight += 1; + maxInFlight = Math.max(maxInFlight, inFlight); + await new Promise((resolve) => { + setTimeout(resolve, 10); + }); + inFlight -= 1; + return linksResponse('https://storage/0?sig=new', 0, 100); + }); + await Promise.all([provider.refreshLinks(new Int64(0), new Int64(100)), provider.fetchNext({ limit: 1000 })]); + expect(maxInFlight).to.equal(1); + }); + + it('returns no links for an operation closed by direct results', async () => { + const context = new ClientContextStub(); + const fetchResults = sinon.spy(context.driver, 'fetchResults'); + const provider = new RowSetProvider(context, makeOperationHandle(), [], true); + + expect(await provider.refreshLinks(new Int64(0), new Int64(100))).to.deep.equal([]); + expect(fetchResults.called).to.be.false; + }); + }); }); From b4b9ab95235393442f59ebfd60426e9f6474dca7 Mon Sep 17 00:00:00 2001 From: Vu Anh Phung Date: Fri, 13 Nov 2026 09:04:58 +0000 Subject: [PATCH 02/10] fix: make socketTimeout an idle timeout and share the deadline across refreshes Review follow-up. The per-attempt download timer now restarts on every received chunk, so socketTimeout aborts only an attempt that stops receiving data, not a large download still in progress. Link refreshes reuse the download's retriesTimeout deadline instead of starting a new one each. Signed-off-by: Vu Anh Phung Co-authored-by: Isaac --- CHANGELOG.md | 2 +- lib/contracts/IDBSQLClient.ts | 5 +- lib/result/CloudFetchResultHandler.ts | 59 +++++++++++++------ .../result/CloudFetchResultHandler.test.ts | 53 +++++++++++++++++ 4 files changed, 97 insertions(+), 22 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 92f6695b..16c8eacc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,7 +3,7 @@ ## Unreleased - Reduce result chunking and `fetchAll()` overhead for large query results. -- Thrift CloudFetch downloads now recover from more failures. When cloud storage rejects a presigned URL with 401, 403 or 404 (an expired or invalid signature), the driver fetches a fresh link for the same rows from the server and retries with it; each refresh counts as an attempt against `retryMaxAttempts`. Other 4xx responses are retried with the same URL. `socketTimeout` now also bounds each download attempt, so a stalled attempt is retried instead of waiting until `retriesTimeout`. Previously these failures failed the query, and a stalled attempt waited out the whole deadline. +- Thrift CloudFetch downloads now recover from more failures. When cloud storage rejects a presigned URL with 401, 403 or 404 (an expired or invalid signature), the driver fetches a fresh link for the same rows from the server and retries with it; each refresh counts as an attempt against `retryMaxAttempts`. Other 4xx responses are retried with the same URL. `socketTimeout` is now also the inactivity timeout of each download attempt, so an attempt that stops receiving data is retried instead of waiting until `retriesTimeout`; one `retriesTimeout` deadline still covers the whole download, including link refreshes. Previously these failures failed the query, and a stalled attempt waited out the whole deadline. - Thrift CloudFetch downloads are now bounded end to end by `retriesTimeout` (default 15 minutes), including every attempt and retry backoff. Previously a stalled download, where the response headers or body never arrived, blocked the fetch forever. A download still running when the deadline expires is aborted and fails with `RetryError` (`TIMEOUT_EXCEEDED`, message `CloudFetch result download failed: Retry timeout exceeded`). (#520) - Thrift connections made through `proxy` now apply `checkServerCertificate`, `customCaCert` and `clientCert`/`clientKey`. Previously the proxied path ignored them and verified against Node's process-wide trust store, so a certificate outside the configured trust set could be accepted. **Compatibility:** proxied connections now verify unless `checkServerCertificate: false` is set; as on direct connections, `NODE_TLS_REJECT_UNAUTHORIZED=0` no longer turns verification off. For an `https` proxy these options also apply to the proxy's own certificate. (#485) - Honor the client `useCloudFetch: false` default on the kernel backend so complete results can be fetched without cloud-storage downloads. (#574) diff --git a/lib/contracts/IDBSQLClient.ts b/lib/contracts/IDBSQLClient.ts index 2d6c3929..fb036cb1 100644 --- a/lib/contracts/IDBSQLClient.ts +++ b/lib/contracts/IDBSQLClient.ts @@ -71,8 +71,9 @@ export type ConnectionOptions = { path: string; userAgentEntry?: string; /** - * Request timeout in milliseconds. On the Thrift backend it also bounds each CloudFetch - * download attempt; an attempt that runs out is retried. Default 900000 (15 minutes). + * Request timeout in milliseconds. On the Thrift backend it is also the inactivity timeout + * of each CloudFetch download attempt: an attempt that receives nothing for this long is + * aborted and retried. Default 900000 (15 minutes). */ socketTimeout?: number; proxy?: ProxyOptions; diff --git a/lib/result/CloudFetchResultHandler.ts b/lib/result/CloudFetchResultHandler.ts index 36baf436..2ec2d3f7 100644 --- a/lib/result/CloudFetchResultHandler.ts +++ b/lib/result/CloudFetchResultHandler.ts @@ -142,10 +142,12 @@ export default class CloudFetchResultHandler implements IResultsProvider candidate.startRowOffset.toNumber() === offset); } - private async fetch(url: RequestInfo, init?: RequestInit): Promise<{ response: Response; body?: ArrayBuffer }> { + // End (epoch ms) of a download's `retriesTimeout` deadline; `undefined` when it is off + private downloadDeadline(): number | undefined { + const { retriesTimeout } = this.context.getConfig(); + return retriesTimeout > 0 ? Date.now() + retriesTimeout : undefined; + } + + // `deadlineAt` is shared by a download and its link refreshes + private async fetch( + url: RequestInfo, + init?: RequestInit, + deadlineAt: number | undefined = this.downloadDeadline(), + ): Promise<{ response: Response; body?: ArrayBuffer }> { const connectionProvider = await this.context.getConnectionProvider(); let requestUrl = url; if (typeof requestUrl !== 'string') { @@ -197,10 +210,10 @@ export default class CloudFetchResultHandler implements IResultsProvider 0 ? Math.min(socketTimeout, MAX_TIMER_DELAY_MS) : undefined; + const { socketTimeout } = this.context.getConfig(); + // `socketTimeout` is an inactivity timeout for each attempt: an attempt that receives + // nothing for that long is aborted and retried like a network timeout. + const idleTimeoutMs = socketTimeout > 0 ? Math.min(socketTimeout, MAX_TIMER_DELAY_MS) : undefined; const requestConfig: RequestInit = { agent, ...init }; // The retry policy checks `retriesTimeout` only between attempts, so a stalled attempt @@ -208,7 +221,8 @@ export default class CloudFetchResultHandler implements IResultsProvider 0 ? Math.min(retriesTimeout, MAX_TIMER_DELAY_MS) : undefined; + const deadlineMs = + deadlineAt === undefined ? undefined : Math.min(Math.max(deadlineAt - Date.now(), 0), MAX_TIMER_DELAY_MS); // Aborted only once, by the deadline, so a single controller serves every attempt and backoff. const controller = deadlineMs === undefined ? undefined : new AbortController(); let timeoutError: RetryError | undefined; @@ -230,26 +244,33 @@ export default class CloudFetchResultHandler implements IResultsProvider attempt.abort(); controller?.signal.addEventListener('abort', abortAttempt, { once: true }); let attemptTimedOut = false; - const attemptTimer = - attemptTimeoutMs === undefined - ? undefined - : setTimeout(() => { - attemptTimedOut = true; - attempt.abort(); - }, attemptTimeoutMs); + let idleTimer: NodeJS.Timeout | undefined; + const restartIdleTimer = () => { + if (idleTimeoutMs !== undefined) { + clearTimeout(idleTimer); + idleTimer = setTimeout(() => { + attemptTimedOut = true; + attempt.abort(); + }, idleTimeoutMs); + } + }; + restartIdleTimer(); const request = new Request(url, { ...requestConfig, signal: attempt.signal as unknown as RequestInit['signal'], }); try { const response = await fetch(request); + restartIdleTimer(); if (response.ok) { - downloaded = await response.arrayBuffer(); + const bodyRead = response.arrayBuffer(); + response.body.on('data', restartIdleTimer); + downloaded = await bodyRead; } else { lastFailure = { request, response }; } @@ -263,7 +284,7 @@ export default class CloudFetchResultHandler implements IResultsProvider { ); expect(error.message).to.not.contain('SECRET_SIGNATURE'); }); + + it('should not time out an attempt that keeps receiving data', async () => { + let hits = 0; + const url = await startServer((req, res) => { + hits += 1; + res.writeHead(200, { 'Content-Length': `${body.length}` }); + // Slower than socketTimeout in total, but never idle for that long + let sent = 0; + const interval = setInterval(() => { + const next = Math.min(sent + 40, body.length); + res.write(body.subarray(sent, next)); + sent = next; + if (sent === body.length) { + clearInterval(interval); + res.end(); + } + }, 50); + }); + const handler = createHandler(url, { socketTimeout: 150 }); + + const { value, error, elapsedMs } = await fetchWithin(handler, 3000); + + expect(error).to.be.undefined; + expect(value?.batches).to.deep.eq([body]); + expect(hits).to.be.equal(1); + expect(elapsedMs).to.be.gte(150); + }); + + it('should share one retriesTimeout deadline across link refreshes', async () => { + const url = await startServer((req, res) => { + if (req.url?.includes('sig=0')) { + res.writeHead(403, 'Forbidden'); + res.end(); + } + // ...every refreshed URL stalls + }); + let refreshes = 0; + const { handler } = createRefreshingHandler( + makeLink(`${url}?sig=0`), + () => { + refreshes += 1; + return [makeLink(`${url}?sig=${refreshes}`)]; + }, + { retriesTimeout: 300, retryMaxAttempts: 10 }, + ); + + const { error, elapsedMs } = await fetchWithin(handler, 1500); + + expect(error).to.be.instanceOf(RetryError); + expect(error.errorCode).to.be.equal(RetryErrorCode.TimeoutExceeded); + expect(elapsedMs).to.be.lt(1000); + expect(refreshes).to.be.equal(1); + }); }); it('should not enforce a deadline when retriesTimeout is not positive', async () => { From 1caae6441266ca7d15bf1640e50973e60509c014 Mon Sep 17 00:00:00 2001 From: Vu Anh Phung Date: Sun, 15 Nov 2026 07:23:12 +0000 Subject: [PATCH 03/10] docs: describe the CloudFetch socketTimeout and refresh budget; widen the trickle test margin The CHANGELOG now says the idle retry applies when socketTimeout is shorter than retriesTimeout (both default to 15 minutes). CONNECTION_PARAMETERS.md notes that socketTimeout also bounds each Thrift CloudFetch download attempt and that link refreshes count against retryMaxAttempts. The trickling-body unit test had only 100 ms of slack per gap; it now uses a 300 ms idle timeout over a ~1 s body. Signed-off-by: Vu Anh Phung Co-authored-by: Isaac --- CHANGELOG.md | 2 +- CONNECTION_PARAMETERS.md | 4 ++-- tests/unit/result/CloudFetchResultHandler.test.ts | 11 ++++++----- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 16c8eacc..204f8ccc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,7 +3,7 @@ ## Unreleased - Reduce result chunking and `fetchAll()` overhead for large query results. -- Thrift CloudFetch downloads now recover from more failures. When cloud storage rejects a presigned URL with 401, 403 or 404 (an expired or invalid signature), the driver fetches a fresh link for the same rows from the server and retries with it; each refresh counts as an attempt against `retryMaxAttempts`. Other 4xx responses are retried with the same URL. `socketTimeout` is now also the inactivity timeout of each download attempt, so an attempt that stops receiving data is retried instead of waiting until `retriesTimeout`; one `retriesTimeout` deadline still covers the whole download, including link refreshes. Previously these failures failed the query, and a stalled attempt waited out the whole deadline. +- Thrift CloudFetch downloads now recover from more failures. When cloud storage rejects a presigned URL with 401, 403 or 404 (an expired or invalid signature), the driver fetches a fresh link for the same rows from the server and retries with it; each refresh counts as an attempt against `retryMaxAttempts`. Other 4xx responses are retried with the same URL. `socketTimeout` is now also the inactivity timeout of each download attempt, so when it is shorter than `retriesTimeout` (both default to 15 minutes), an attempt that stops receiving data is retried instead of waiting out the deadline; one `retriesTimeout` deadline still covers the whole download, including link refreshes. Previously these failures failed the query, and a stalled attempt waited out the whole deadline. - Thrift CloudFetch downloads are now bounded end to end by `retriesTimeout` (default 15 minutes), including every attempt and retry backoff. Previously a stalled download, where the response headers or body never arrived, blocked the fetch forever. A download still running when the deadline expires is aborted and fails with `RetryError` (`TIMEOUT_EXCEEDED`, message `CloudFetch result download failed: Retry timeout exceeded`). (#520) - Thrift connections made through `proxy` now apply `checkServerCertificate`, `customCaCert` and `clientCert`/`clientKey`. Previously the proxied path ignored them and verified against Node's process-wide trust store, so a certificate outside the configured trust set could be accepted. **Compatibility:** proxied connections now verify unless `checkServerCertificate: false` is set; as on direct connections, `NODE_TLS_REJECT_UNAUTHORIZED=0` no longer turns verification off. For an `https` proxy these options also apply to the proxy's own certificate. (#485) - Honor the client `useCloudFetch: false` default on the kernel backend so complete results can be fetched without cloud-storage downloads. (#574) diff --git a/CONNECTION_PARAMETERS.md b/CONNECTION_PARAMETERS.md index e0503021..0d0120c9 100644 --- a/CONNECTION_PARAMETERS.md +++ b/CONNECTION_PARAMETERS.md @@ -78,11 +78,11 @@ Per-statement `useCloudFetch` overrides remain unsupported on the kernel backend | Option | Type | Thrift | Kernel | Default Value | Note | | --------------------------- | ------------------------ | :----: | :----: | ------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `socketTimeout` | `number` (ms) | ✅ | ⚠️ | `900000` / `120000` | Kernel forwards only positive values. `0` means no timeout on Thrift but is omitted on Kernel, which then keeps its 120-second default. | +| `socketTimeout` | `number` (ms) | ✅ | ⚠️ | `900000` / `120000` | Kernel forwards only positive values. `0` means no timeout on Thrift but is omitted on Kernel, which then keeps its 120-second default. On Thrift it is also the inactivity timeout of each CloudFetch download attempt. | | `proxy` | `ProxyOptions` | ✅ | ⚠️ | — | `{protocol, host, port, auth}`. Kernel supports HTTP/HTTPS proxies only; Thrift also supports the declared SOCKS variants. | | `noProxy` (internal) | `string` | ❌ | ⚠️ | — | Runtime-only Kernel bypass list, forwarded as `bypassHosts` only when an explicit `proxy` is also supplied. | | `customHeaders` | `Record` | ✅ | ✅ | — | Sent on workspace, telemetry and feature-flag requests, not OAuth. Workspace calls ignore caller `Authorization`, `User-Agent`, `Content-Type`, `Content-Length`, `Transfer-Encoding`, `Connection`; Kernel also ignores `x-databricks-org-id`. | -| `retryMaxAttempts` | `number` | ✅ | ✅ | `5` | Total attempts, including the initial request. Kernel converts this to its retries-after-first representation internally. | +| `retryMaxAttempts` | `number` | ✅ | ✅ | `5` | Total attempts, including the initial request. Kernel converts this to its retries-after-first representation internally. Thrift CloudFetch link refreshes count as attempts. | | `retriesTimeout` | `number` (ms) | ✅ | ✅ | `900000` | Kernel converts milliseconds to whole seconds. | | `retryDelayMin` | `number` (ms) | ✅ | ✅ | `1000` | Kernel converts milliseconds to whole seconds. | | `retryDelayMax` | `number` (ms) | ✅ | ✅ | `60000` | Kernel converts milliseconds to whole seconds. | diff --git a/tests/unit/result/CloudFetchResultHandler.test.ts b/tests/unit/result/CloudFetchResultHandler.test.ts index c725a523..b3f13de5 100644 --- a/tests/unit/result/CloudFetchResultHandler.test.ts +++ b/tests/unit/result/CloudFetchResultHandler.test.ts @@ -1057,7 +1057,8 @@ describe('CloudFetchResultHandler', () => { expect(error.message).to.not.contain('SECRET_SIGNATURE'); }); - it('should not time out an attempt that keeps receiving data', async () => { + it('should not time out an attempt that keeps receiving data', async function () { + this.timeout(5000); let hits = 0; const url = await startServer((req, res) => { hits += 1; @@ -1065,7 +1066,7 @@ describe('CloudFetchResultHandler', () => { // Slower than socketTimeout in total, but never idle for that long let sent = 0; const interval = setInterval(() => { - const next = Math.min(sent + 40, body.length); + const next = Math.min(sent + 20, body.length); res.write(body.subarray(sent, next)); sent = next; if (sent === body.length) { @@ -1074,14 +1075,14 @@ describe('CloudFetchResultHandler', () => { } }, 50); }); - const handler = createHandler(url, { socketTimeout: 150 }); + const handler = createHandler(url, { socketTimeout: 300 }); - const { value, error, elapsedMs } = await fetchWithin(handler, 3000); + const { value, error, elapsedMs } = await fetchWithin(handler, 4000); expect(error).to.be.undefined; expect(value?.batches).to.deep.eq([body]); expect(hits).to.be.equal(1); - expect(elapsedMs).to.be.gte(150); + expect(elapsedMs).to.be.gte(300); }); it('should share one retriesTimeout deadline across link refreshes', async () => { From e3f676dd50df0d650eb9ce2655a7ef01ba2526a3 Mon Sep 17 00:00:00 2001 From: Vu Anh Phung Date: Sun, 15 Nov 2026 08:11:29 +0000 Subject: [PATCH 04/10] test: make the shared-deadline test fail without a shared deadline; fix refresh-budget and socketTimeout docs Every URL in the shared-deadline test now answers 403 after 120 ms, so a fresh retriesTimeout per refreshed URL would refresh 9 times and fail with the 403, while the shared deadline times out within 3 GETs. The docs now say a download tries at most retryMaxAttempts URLs, each retried under its own budget, and the socketTimeout JSDoc gives the Thrift and kernel semantics and defaults. fetch() takes deadlineAt explicitly instead of defaulting it. Signed-off-by: Vu Anh Phung Co-authored-by: Isaac --- CHANGELOG.md | 2 +- CONNECTION_PARAMETERS.md | 2 +- lib/contracts/IDBSQLClient.ts | 8 +++++--- lib/result/CloudFetchResultHandler.ts | 8 ++++---- .../connections/PlainHttpTransport.test.ts | 4 +++- tests/unit/result/CloudFetchResultHandler.test.ts | 15 ++++++++------- 6 files changed, 22 insertions(+), 17 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 204f8ccc..bdf99864 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,7 +3,7 @@ ## Unreleased - Reduce result chunking and `fetchAll()` overhead for large query results. -- Thrift CloudFetch downloads now recover from more failures. When cloud storage rejects a presigned URL with 401, 403 or 404 (an expired or invalid signature), the driver fetches a fresh link for the same rows from the server and retries with it; each refresh counts as an attempt against `retryMaxAttempts`. Other 4xx responses are retried with the same URL. `socketTimeout` is now also the inactivity timeout of each download attempt, so when it is shorter than `retriesTimeout` (both default to 15 minutes), an attempt that stops receiving data is retried instead of waiting out the deadline; one `retriesTimeout` deadline still covers the whole download, including link refreshes. Previously these failures failed the query, and a stalled attempt waited out the whole deadline. +- Thrift CloudFetch downloads now recover from more failures. When cloud storage rejects a presigned URL with 401, 403 or 404 (an expired or invalid signature), the driver fetches a fresh link for the same rows from the server and retries with it. A download tries at most `retryMaxAttempts` URLs (the original and its refreshes), and each URL is retried under its own `retryMaxAttempts` budget. Other 4xx responses are retried with the same URL. `socketTimeout` is now also the inactivity timeout of each download attempt, so when it is shorter than `retriesTimeout` (both default to 15 minutes), an attempt that stops receiving data is retried instead of waiting out the deadline; one `retriesTimeout` deadline still covers the whole download, including link refreshes. Previously these failures failed the query, and a stalled attempt waited out the whole deadline. - Thrift CloudFetch downloads are now bounded end to end by `retriesTimeout` (default 15 minutes), including every attempt and retry backoff. Previously a stalled download, where the response headers or body never arrived, blocked the fetch forever. A download still running when the deadline expires is aborted and fails with `RetryError` (`TIMEOUT_EXCEEDED`, message `CloudFetch result download failed: Retry timeout exceeded`). (#520) - Thrift connections made through `proxy` now apply `checkServerCertificate`, `customCaCert` and `clientCert`/`clientKey`. Previously the proxied path ignored them and verified against Node's process-wide trust store, so a certificate outside the configured trust set could be accepted. **Compatibility:** proxied connections now verify unless `checkServerCertificate: false` is set; as on direct connections, `NODE_TLS_REJECT_UNAUTHORIZED=0` no longer turns verification off. For an `https` proxy these options also apply to the proxy's own certificate. (#485) - Honor the client `useCloudFetch: false` default on the kernel backend so complete results can be fetched without cloud-storage downloads. (#574) diff --git a/CONNECTION_PARAMETERS.md b/CONNECTION_PARAMETERS.md index 0d0120c9..95cc2c40 100644 --- a/CONNECTION_PARAMETERS.md +++ b/CONNECTION_PARAMETERS.md @@ -82,7 +82,7 @@ Per-statement `useCloudFetch` overrides remain unsupported on the kernel backend | `proxy` | `ProxyOptions` | ✅ | ⚠️ | — | `{protocol, host, port, auth}`. Kernel supports HTTP/HTTPS proxies only; Thrift also supports the declared SOCKS variants. | | `noProxy` (internal) | `string` | ❌ | ⚠️ | — | Runtime-only Kernel bypass list, forwarded as `bypassHosts` only when an explicit `proxy` is also supplied. | | `customHeaders` | `Record` | ✅ | ✅ | — | Sent on workspace, telemetry and feature-flag requests, not OAuth. Workspace calls ignore caller `Authorization`, `User-Agent`, `Content-Type`, `Content-Length`, `Transfer-Encoding`, `Connection`; Kernel also ignores `x-databricks-org-id`. | -| `retryMaxAttempts` | `number` | ✅ | ✅ | `5` | Total attempts, including the initial request. Kernel converts this to its retries-after-first representation internally. Thrift CloudFetch link refreshes count as attempts. | +| `retryMaxAttempts` | `number` | ✅ | ✅ | `5` | Total attempts, including the initial request. Kernel converts this to its retries-after-first representation internally. On Thrift it also caps the CloudFetch URLs per download; each URL is retried under its own budget. | | `retriesTimeout` | `number` (ms) | ✅ | ✅ | `900000` | Kernel converts milliseconds to whole seconds. | | `retryDelayMin` | `number` (ms) | ✅ | ✅ | `1000` | Kernel converts milliseconds to whole seconds. | | `retryDelayMax` | `number` (ms) | ✅ | ✅ | `60000` | Kernel converts milliseconds to whole seconds. | diff --git a/lib/contracts/IDBSQLClient.ts b/lib/contracts/IDBSQLClient.ts index fb036cb1..8b7c6ecb 100644 --- a/lib/contracts/IDBSQLClient.ts +++ b/lib/contracts/IDBSQLClient.ts @@ -71,9 +71,11 @@ export type ConnectionOptions = { path: string; userAgentEntry?: string; /** - * Request timeout in milliseconds. On the Thrift backend it is also the inactivity timeout - * of each CloudFetch download attempt: an attempt that receives nothing for this long is - * aborted and retried. Default 900000 (15 minutes). + * Timeout in milliseconds. Thrift: how long each request may wait for its response headers, + * and then for its body (default 900000, 15 minutes; `0` disables it). It is also the + * inactivity timeout of each CloudFetch download attempt: an attempt that receives nothing + * for this long is aborted and retried. Kernel: the socket read timeout, used only when + * positive; otherwise the kernel keeps its 120-second default. */ socketTimeout?: number; proxy?: ProxyOptions; diff --git a/lib/result/CloudFetchResultHandler.ts b/lib/result/CloudFetchResultHandler.ts index 2ec2d3f7..9e3d32d2 100644 --- a/lib/result/CloudFetchResultHandler.ts +++ b/lib/result/CloudFetchResultHandler.ts @@ -147,7 +147,7 @@ export default class CloudFetchResultHandler implements IResultsProvider 0 ? Date.now() + retriesTimeout : undefined; } - // `deadlineAt` is shared by a download and its link refreshes + // `deadlineAt` is shared by a download and its link refreshes; `undefined` means no deadline private async fetch( url: RequestInfo, - init?: RequestInit, - deadlineAt: number | undefined = this.downloadDeadline(), + init: RequestInit, + deadlineAt: number | undefined, ): Promise<{ response: Response; body?: ArrayBuffer }> { const connectionProvider = await this.context.getConnectionProvider(); let requestUrl = url; diff --git a/tests/unit/connection/connections/PlainHttpTransport.test.ts b/tests/unit/connection/connections/PlainHttpTransport.test.ts index 425387d1..c2799da5 100644 --- a/tests/unit/connection/connections/PlainHttpTransport.test.ts +++ b/tests/unit/connection/connections/PlainHttpTransport.test.ts @@ -146,7 +146,9 @@ describe('DBSQLClient plain HTTP transport', () => { const handler = new CloudFetchResultHandler(client, new ResultsProviderStub([], undefined), { status: { statusCode: TStatusCode.SUCCESS_STATUS }, }); - await expectTlsAttempt(() => handler['fetch'](`https://127.0.0.1:${port}/chunk?X-Amz-Signature=secret`)); + await expectTlsAttempt(() => + handler['fetch'](`https://127.0.0.1:${port}/chunk?X-Amz-Signature=secret`, {}, undefined), + ); await client.close(); // The HTTPS attempt never reached the server as an HTTP request. diff --git a/tests/unit/result/CloudFetchResultHandler.test.ts b/tests/unit/result/CloudFetchResultHandler.test.ts index b3f13de5..e5513587 100644 --- a/tests/unit/result/CloudFetchResultHandler.test.ts +++ b/tests/unit/result/CloudFetchResultHandler.test.ts @@ -1085,13 +1085,15 @@ describe('CloudFetchResultHandler', () => { expect(elapsedMs).to.be.gte(300); }); - it('should share one retriesTimeout deadline across link refreshes', async () => { + it('should share one retriesTimeout deadline across link refreshes', async function () { + this.timeout(5000); + // Every URL is rejected after 120 ms. One shared 300 ms deadline ends the download within + // 3 GETs; a fresh deadline per refreshed URL would refresh 9 times and fail with the 403. const url = await startServer((req, res) => { - if (req.url?.includes('sig=0')) { + setTimeout(() => { res.writeHead(403, 'Forbidden'); res.end(); - } - // ...every refreshed URL stalls + }, 120); }); let refreshes = 0; const { handler } = createRefreshingHandler( @@ -1103,12 +1105,11 @@ describe('CloudFetchResultHandler', () => { { retriesTimeout: 300, retryMaxAttempts: 10 }, ); - const { error, elapsedMs } = await fetchWithin(handler, 1500); + const { error } = await fetchWithin(handler, 4000); expect(error).to.be.instanceOf(RetryError); expect(error.errorCode).to.be.equal(RetryErrorCode.TimeoutExceeded); - expect(elapsedMs).to.be.lt(1000); - expect(refreshes).to.be.equal(1); + expect(refreshes).to.be.lessThan(9); }); }); From 041145310bde98f58a600eea13234a1dc8a0a6f4 Mon Sep 17 00:00:00 2001 From: Vu Anh Phung Date: Sun, 15 Nov 2026 08:32:01 +0000 Subject: [PATCH 05/10] test: check that connect() copies socketTimeout into ClientConfig The CloudFetch download idle timeout reads socketTimeout from ClientConfig, so the existing connect() retry-option tests now also cover socketTimeout: a set value, 0, and the default when omitted. Signed-off-by: Vu Anh Phung Co-authored-by: Isaac --- tests/unit/DBSQLClient.test.ts | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/tests/unit/DBSQLClient.test.ts b/tests/unit/DBSQLClient.test.ts index 386f268f..c051920b 100644 --- a/tests/unit/DBSQLClient.test.ts +++ b/tests/unit/DBSQLClient.test.ts @@ -879,6 +879,7 @@ describe('DBSQLClient retry-policy ConnectionOptions', () => { expect(client.getConfig().retriesTimeout).to.equal(15 * 60 * 1000); expect(client.getConfig().retryDelayMin).to.equal(1000); expect(client.getConfig().retryDelayMax).to.equal(60 * 1000); + expect(client.getConfig().socketTimeout).to.equal(15 * 60 * 1000); await client.connect({ ...connectOptions, @@ -886,12 +887,23 @@ describe('DBSQLClient retry-policy ConnectionOptions', () => { retriesTimeout: 5000, retryDelayMin: 100, retryDelayMax: 500, + socketTimeout: 30000, }); expect(client.getConfig().retryMaxAttempts).to.equal(2); expect(client.getConfig().retriesTimeout).to.equal(5000); expect(client.getConfig().retryDelayMin).to.equal(100); expect(client.getConfig().retryDelayMax).to.equal(500); + // Read by the Thrift CloudFetch download idle timeout + expect(client.getConfig().socketTimeout).to.equal(30000); + }); + + it('ingests socketTimeout: 0 from connect() into ClientConfig', async () => { + const client = new DBSQLClient(); + + await client.connect({ ...connectOptions, socketTimeout: 0 }); + + expect(client.getConfig().socketTimeout).to.equal(0); }); it('keeps defaults when retry-policy options are omitted', async () => { @@ -903,6 +915,7 @@ describe('DBSQLClient retry-policy ConnectionOptions', () => { expect(client.getConfig().retriesTimeout).to.equal(15 * 60 * 1000); expect(client.getConfig().retryDelayMin).to.equal(1000); expect(client.getConfig().retryDelayMax).to.equal(60 * 1000); + expect(client.getConfig().socketTimeout).to.equal(15 * 60 * 1000); }); }); From 723056cd9af956ac3132dc7552a2df6046631614 Mon Sep 17 00:00:00 2001 From: Vu Anh Phung Date: Sun, 15 Nov 2026 09:02:37 +0000 Subject: [PATCH 06/10] fix: report the rejected response when a link refresh outlasts retriesTimeout The refresh FetchResults RPC is not bounded by the download deadline. If it ran past it, the next fetch started a GET that aborted at once and failed with a bare 'Retry timeout exceeded'. Now the download fails with the same TIMEOUT_EXCEEDED RetryError, with the rejected 401/403/404 response as its cause, and without the doomed GET. Signed-off-by: Vu Anh Phung Co-authored-by: Isaac --- lib/result/CloudFetchResultHandler.ts | 6 ++++ .../result/CloudFetchResultHandler.test.ts | 32 ++++++++++++++++++- 2 files changed, 37 insertions(+), 1 deletion(-) diff --git a/lib/result/CloudFetchResultHandler.ts b/lib/result/CloudFetchResultHandler.ts index 9e3d32d2..2e8661b7 100644 --- a/lib/result/CloudFetchResultHandler.ts +++ b/lib/result/CloudFetchResultHandler.ts @@ -155,6 +155,12 @@ export default class CloudFetchResultHandler implements IResultsProvider= deadlineAt) { + throw CloudFetchResultHandler.describeDownloadError( + new RetryError(RetryErrorCode.TimeoutExceeded, { response }), + ); + } // eslint-disable-next-line no-await-in-loop ({ response, body } = await this.fetch(currentLink.fileLink, { headers: currentLink.httpHeaders }, deadlineAt)); } diff --git a/tests/unit/result/CloudFetchResultHandler.test.ts b/tests/unit/result/CloudFetchResultHandler.test.ts index e5513587..e95b15db 100644 --- a/tests/unit/result/CloudFetchResultHandler.test.ts +++ b/tests/unit/result/CloudFetchResultHandler.test.ts @@ -918,7 +918,7 @@ describe('CloudFetchResultHandler', () => { // Like `createHandler`, with a source that can refresh links function createRefreshingHandler( link: TSparkArrowResultLink, - refreshedLinks: () => Array, + refreshedLinks: () => Array | Promise>, configOverrides: Partial = {}, ) { const context = new BaseClientContextStub({ cloudFetchConcurrentDownloads: 1, ...configOverrides }); @@ -1111,6 +1111,36 @@ describe('CloudFetchResultHandler', () => { expect(error.errorCode).to.be.equal(RetryErrorCode.TimeoutExceeded); expect(refreshes).to.be.lessThan(9); }); + + it('should time out with the rejected response as the cause when a refresh outlasts retriesTimeout', async () => { + let hits = 0; + const url = await startServer((req, res) => { + hits += 1; + res.writeHead(403, 'Forbidden'); + res.end(); + }); + const { handler, refreshLinks } = createRefreshingHandler( + makeLink(`${url}?sig=0`), + async () => { + await new Promise((resolve) => { + setTimeout(resolve, 400); + }); + return [makeLink(`${url}?sig=1`)]; + }, + { retriesTimeout: 300 }, + ); + + const { error } = await fetchWithin(handler, 1500); + + expect(error).to.be.instanceOf(RetryError); + expect(error.errorCode).to.be.equal(RetryErrorCode.TimeoutExceeded); + expect(error.message).to.be.equal( + 'CloudFetch result download failed: Retry timeout exceeded (last error: HTTP 403: Forbidden)', + ); + expect(refreshLinks.callCount).to.be.equal(1); + // No doomed GET of the refreshed URL + expect(hits).to.be.equal(1); + }); }); it('should not enforce a deadline when retriesTimeout is not positive', async () => { From 3a34b1f3f1e8138f6be7cf61862d54a54462951c Mon Sep 17 00:00:00 2001 From: Vu Anh Phung Date: Sun, 15 Nov 2026 23:17:40 +0000 Subject: [PATCH 07/10] fix: keep the CloudFetch download deadline active across link refreshes The retriesTimeout deadline was a timer inside each GET, cleared before a link refresh, so a refresh that was queued behind another FetchResults call, or whose RPC was slow, kept the download pending past the deadline (forever, if it never returned). One deadline now spans the whole download, refreshes included; a refresh that returns late can't resume the download, and a refresh still queued when its download gives up is skipped. Signed-off-by: Vu Anh Phung Co-authored-by: Isaac --- lib/result/CloudFetchResultHandler.ts | 233 ++++++++++-------- lib/result/RowSetProvider.ts | 11 +- .../result/CloudFetchResultHandler.test.ts | 189 +++++++++++++- tests/unit/result/RowSetProvider.test.ts | 29 +++ 4 files changed, 353 insertions(+), 109 deletions(-) diff --git a/lib/result/CloudFetchResultHandler.ts b/lib/result/CloudFetchResultHandler.ts index 2e8661b7..cf74e868 100644 --- a/lib/result/CloudFetchResultHandler.ts +++ b/lib/result/CloudFetchResultHandler.ts @@ -27,13 +27,71 @@ const MAX_TIMER_DELAY_MS = 2 ** 31 - 1; const LINK_REFRESH_STATUSES: ReadonlySet = new Set([401, 403, 404]); interface LinkRefresher { - refreshLinks(startRowOffset: Int64, maxRows: Int64): Promise>; + // An aborted `signal` means the caller gave up; a refresh still queued may then be skipped + refreshLinks(startRowOffset: Int64, maxRows: Int64, signal?: AbortSignal): Promise>; } function canRefreshLinks(source: unknown): source is LinkRefresher { return typeof (source as Partial | undefined)?.refreshLinks === 'function'; } +// The `retriesTimeout` deadline of one download: its GET attempts, the backoff between them and +// its link refreshes. On expiry `signal` is aborted with a `TimeoutExceeded` error. +class DownloadDeadline { + public readonly signal?: AbortSignal; + + // Reported as the cause of the timeout error + public lastFailure: unknown; + + private readonly timer?: NodeJS.Timeout; + + constructor(timeoutMs: number) { + if (timeoutMs > 0) { + const controller = new AbortController(); + this.signal = controller.signal; + this.timer = setTimeout(() => { + controller.abort(new RetryError(RetryErrorCode.TimeoutExceeded, this.lastFailure)); + }, Math.min(timeoutMs, MAX_TIMER_DELAY_MS)); + } + } + + public throwIfExpired(): void { + if (this.signal?.aborted) { + throw this.signal.reason; + } + } + + // Settles like `promise`, or rejects with the timeout error as soon as the deadline expires + public race(promise: Promise): Promise { + const { signal } = this; + if (!signal) { + return promise; + } + return new Promise((resolve, reject) => { + const onExpired = () => reject(signal.reason); + if (signal.aborted) { + onExpired(); + } else { + signal.addEventListener('abort', onExpired, { once: true }); + } + promise.then( + (value) => { + signal.removeEventListener('abort', onExpired); + resolve(value); + }, + (error) => { + signal.removeEventListener('abort', onExpired); + reject(error); + }, + ); + }); + } + + public clear(): void { + clearTimeout(this.timer); + } +} + export default class CloudFetchResultHandler implements IResultsProvider { private readonly context: IClientContext; @@ -141,32 +199,41 @@ export default class CloudFetchResultHandler implements IResultsProvider { const startTime = Date.now(); const { retryMaxAttempts } = this.context.getConfig(); - // One `retriesTimeout` deadline covers the download and all of its link refreshes - const deadlineAt = this.downloadDeadline(); let currentLink = link; - let { response, body } = await this.fetch(currentLink.fileLink, { headers: currentLink.httpHeaders }, deadlineAt); + let { response, body } = await this.fetch(currentLink.fileLink, { headers: currentLink.httpHeaders }, deadline); // At most `retryMaxAttempts` URLs per download; each `fetch` retries its URL under its own budget for (let attempt = 1; LINK_REFRESH_STATUSES.has(response.status) && attempt < retryMaxAttempts; attempt += 1) { // eslint-disable-next-line no-await-in-loop - const freshLink = await this.refreshLink(currentLink, response.status); + const freshLink = await this.refreshLink(currentLink, response.status, deadline.signal); if (!freshLink) { break; } + // The download already failed if the refresh outlasted the deadline; don't GET the fresh URL + deadline.throwIfExpired(); currentLink = freshLink; - // The refresh RPC isn't bounded by the deadline; if it ran past it, report the rejected response - if (deadlineAt !== undefined && Date.now() >= deadlineAt) { - throw CloudFetchResultHandler.describeDownloadError( - new RetryError(RetryErrorCode.TimeoutExceeded, { response }), - ); - } // eslint-disable-next-line no-await-in-loop - ({ response, body } = await this.fetch(currentLink.fileLink, { headers: currentLink.httpHeaders }, deadlineAt)); + ({ response, body } = await this.fetch(currentLink.fileLink, { headers: currentLink.httpHeaders }, deadline)); } if (!response.ok) { throw new Error(`CloudFetch HTTP error ${response.status} ${response.statusText}`); } + // A download that outlasted the deadline was already reported as failed + deadline.throwIfExpired(); // `body` is always set when `response.ok` is true — `fetch` reads it inside // the retried block on success. const result = body!; @@ -185,7 +252,11 @@ export default class CloudFetchResultHandler implements IResultsProvider { + private async refreshLink( + link: TSparkArrowResultLink, + status: number, + signal: AbortSignal | undefined, + ): Promise { if (!canRefreshLinks(this.source)) { return undefined; } @@ -193,21 +264,14 @@ export default class CloudFetchResultHandler implements IResultsProvider candidate.startRowOffset.toNumber() === offset); } - // End (epoch ms) of a download's `retriesTimeout` deadline; `undefined` when it is off - private downloadDeadline(): number | undefined { - const { retriesTimeout } = this.context.getConfig(); - return retriesTimeout > 0 ? Date.now() + retriesTimeout : undefined; - } - - // `deadlineAt` is shared by a download and its link refreshes; `undefined` means no deadline private async fetch( url: RequestInfo, init: RequestInit, - deadlineAt: number | undefined, + deadline: DownloadDeadline = new DownloadDeadline(0), ): Promise<{ response: Response; body?: ArrayBuffer }> { const connectionProvider = await this.context.getConnectionProvider(); let requestUrl = url; @@ -222,18 +286,7 @@ export default class CloudFetchResultHandler implements IResultsProvider 0 ? Math.min(socketTimeout, MAX_TIMER_DELAY_MS) : undefined; const requestConfig: RequestInit = { agent, ...init }; - // The retry policy checks `retriesTimeout` only between attempts, so a stalled attempt - // (headers or body never arrive) would block forever. Enforce it as a deadline over the - // whole download - every attempt plus the backoff between them - and abort the in-flight - // request and any pending backoff sleep when it expires (#520). Non-positive values keep - // the deadline off. - const deadlineMs = - deadlineAt === undefined ? undefined : Math.min(Math.max(deadlineAt - Date.now(), 0), MAX_TIMER_DELAY_MS); - // Aborted only once, by the deadline, so a single controller serves every attempt and backoff. - const controller = deadlineMs === undefined ? undefined : new AbortController(); - let timeoutError: RetryError | undefined; - let lastFailure: unknown; - let timer: NodeJS.Timeout | undefined; + const { signal } = deadline; // Read the body inside the retried block. CloudFetch downloads are large // GETs against pre-signed cloud-storage URLs that frequently surface @@ -241,78 +294,54 @@ export default class CloudFetchResultHandler implements IResultsProvider { - // The timeout error is not retryable, so this also ends the retry loop when the - // deadline expired during a backoff sleep or aborted the attempt. - if (timeoutError) { - throw timeoutError; + // An expired deadline aborts the in-flight request and rejects a pending backoff sleep + const result = await retryPolicy.invokeWithRetry(async () => { + // The timeout error is not retryable, so this also ends the retry loop when the + // deadline expired during a backoff sleep or aborted the attempt. + deadline.throwIfExpired(); + downloaded = undefined; + // Aborted by the deadline or when the attempt goes idle + const attempt = new AbortController(); + const abortAttempt = () => attempt.abort(); + signal?.addEventListener('abort', abortAttempt, { once: true }); + let attemptTimedOut = false; + let idleTimer: NodeJS.Timeout | undefined; + const restartIdleTimer = () => { + if (idleTimeoutMs !== undefined) { + clearTimeout(idleTimer); + idleTimer = setTimeout(() => { + attemptTimedOut = true; + attempt.abort(); + }, idleTimeoutMs); } - downloaded = undefined; - // Aborted by the deadline or when the attempt goes idle - const attempt = new AbortController(); - const abortAttempt = () => attempt.abort(); - controller?.signal.addEventListener('abort', abortAttempt, { once: true }); - let attemptTimedOut = false; - let idleTimer: NodeJS.Timeout | undefined; - const restartIdleTimer = () => { - if (idleTimeoutMs !== undefined) { - clearTimeout(idleTimer); - idleTimer = setTimeout(() => { - attemptTimedOut = true; - attempt.abort(); - }, idleTimeoutMs); - } - }; + }; + restartIdleTimer(); + const request = new Request(url, { + ...requestConfig, + signal: attempt.signal as unknown as RequestInit['signal'], + }); + try { + const response = await fetch(request); restartIdleTimer(); - const request = new Request(url, { - ...requestConfig, - signal: attempt.signal as unknown as RequestInit['signal'], - }); - try { - const response = await fetch(request); - restartIdleTimer(); - if (response.ok) { - const bodyRead = response.arrayBuffer(); - response.body.on('data', restartIdleTimer); - downloaded = await bodyRead; - } else { - lastFailure = { request, response }; - } - return { request, response }; - } catch (error) { - if (timeoutError) { - throw timeoutError; - } - lastFailure = attemptTimedOut - ? Object.assign(new Error('CloudFetch download attempt timed out'), { code: 'ETIMEDOUT' }) - : error; - throw lastFailure; - } finally { - clearTimeout(idleTimer); - controller?.signal.removeEventListener('abort', abortAttempt); + if (response.ok) { + const bodyRead = response.arrayBuffer(); + response.body.on('data', restartIdleTimer); + downloaded = await bodyRead; + } else { + deadline.lastFailure = { request, response }; } - }, controller?.signal); - - if (deadlineMs === undefined) { - result = await download; - } else { - const deadline = new Promise((_, reject) => { - timer = setTimeout(() => { - timeoutError = new RetryError(RetryErrorCode.TimeoutExceeded, lastFailure); - // Aborts the in-flight request, and rejects a pending backoff sleep with `timeoutError` - controller?.abort(timeoutError); - reject(timeoutError); - }, deadlineMs); - }); - result = await Promise.race([download, deadline]); + return { request, response }; + } catch (error) { + deadline.throwIfExpired(); + deadline.lastFailure = attemptTimedOut + ? Object.assign(new Error('CloudFetch download attempt timed out'), { code: 'ETIMEDOUT' }) + : error; + throw deadline.lastFailure; + } finally { + clearTimeout(idleTimer); + signal?.removeEventListener('abort', abortAttempt); } - } catch (error) { - throw CloudFetchResultHandler.describeDownloadError(error); - } finally { - clearTimeout(timer); - } + }, signal); // Fall back to reading the body here if the retry policy returned a // response without consuming it via our operation (e.g. unit-test stubs // that hand back a pre-baked Response without invoking the operation diff --git a/lib/result/RowSetProvider.ts b/lib/result/RowSetProvider.ts index 6830cfe9..726c0aa0 100644 --- a/lib/result/RowSetProvider.ts +++ b/lib/result/RowSetProvider.ts @@ -157,13 +157,22 @@ export default class RowSetProvider implements IResultsProvider> { + public async refreshLinks( + startRowOffset: Int64, + maxRows: Int64, + signal?: AbortSignal, + ): Promise> { if (this.returnOnlyPrefetchedResults) { return []; } const driver = await this.context.getDriver(); const response = await this.serializeFetch(async () => { + if (signal?.aborted) { + throw signal.reason; + } this.fetchByOffset = true; return driver.fetchResults({ operationHandle: this.operationHandle, diff --git a/tests/unit/result/CloudFetchResultHandler.test.ts b/tests/unit/result/CloudFetchResultHandler.test.ts index e95b15db..e522c5f7 100644 --- a/tests/unit/result/CloudFetchResultHandler.test.ts +++ b/tests/unit/result/CloudFetchResultHandler.test.ts @@ -2,6 +2,7 @@ import { expect, AssertionError } from 'chai'; import sinon, { SinonStub } from 'sinon'; import http from 'http'; import { AddressInfo } from 'net'; +import { PassThrough } from 'stream'; import Int64 from 'node-int64'; import { compressFrameSync } from 'lz4-napi'; import { Request, Response } from 'node-fetch'; @@ -9,8 +10,15 @@ import { ShouldRetryResult } from '../../../lib/connection/contracts/IRetryPolic import { HttpTransactionDetails } from '../../../lib/connection/contracts/IConnectionProvider'; import { LogLevel } from '../../../lib/contracts/IDBSQLLogger'; import CloudFetchResultHandler from '../../../lib/result/CloudFetchResultHandler'; +import RowSetProvider from '../../../lib/result/RowSetProvider'; import ResultsProviderStub from '../.stubs/ResultsProviderStub'; -import { TRowSet, TSparkArrowResultLink, TStatusCode } from '../../../thrift/TCLIService_types'; +import { + TFetchResultsResp, + TOperationType, + TRowSet, + TSparkArrowResultLink, + TStatusCode, +} from '../../../thrift/TCLIService_types'; import { ArrowBatch } from '../../../lib/result/utils'; import RetryError, { RetryErrorCode } from '../../../lib/errors/RetryError'; @@ -1119,13 +1127,15 @@ describe('CloudFetchResultHandler', () => { res.writeHead(403, 'Forbidden'); res.end(); }); + let refreshed: Promise = Promise.resolve(); const { handler, refreshLinks } = createRefreshingHandler( makeLink(`${url}?sig=0`), - async () => { - await new Promise((resolve) => { + () => { + const links = new Promise((resolve) => { setTimeout(resolve, 400); - }); - return [makeLink(`${url}?sig=1`)]; + }).then(() => [makeLink(`${url}?sig=1`)]); + refreshed = links; + return links; }, { retriesTimeout: 300 }, ); @@ -1138,9 +1148,176 @@ describe('CloudFetchResultHandler', () => { 'CloudFetch result download failed: Retry timeout exceeded (last error: HTTP 403: Forbidden)', ); expect(refreshLinks.callCount).to.be.equal(1); - // No doomed GET of the refreshed URL + // The late refresh doesn't resume the download with a doomed GET of the refreshed URL + await refreshed; + await new Promise((resolve) => { + setImmediate(resolve); + }); expect(hits).to.be.equal(1); }); + + it('should time out on schedule while a link refresh never returns', async () => { + const url = await startServer((req, res) => { + res.writeHead(403, 'Forbidden'); + res.end(); + }); + const { handler, refreshLinks } = createRefreshingHandler( + makeLink(`${url}?sig=0`), + () => new Promise(() => {}), + { retriesTimeout: 100 }, + ); + + const { error, elapsedMs } = await fetchWithin(handler, 1000); + + expect(error).to.be.instanceOf(RetryError); + expect(error.errorCode).to.be.equal(RetryErrorCode.TimeoutExceeded); + expect(error.message).to.be.equal( + 'CloudFetch result download failed: Retry timeout exceeded (last error: HTTP 403: Forbidden)', + ); + expect(elapsedMs).to.be.lt(500); + // The refresh is told that the download gave up + expect(refreshLinks.firstCall.args[2].aborted).to.be.true; + }); + + it('should not leak the rejection of a link refresh that fails after retriesTimeout', async () => { + const url = await startServer((req, res) => { + res.writeHead(403, 'Forbidden'); + res.end(); + }); + let refreshed: Promise = Promise.resolve(); + const { handler } = createRefreshingHandler( + makeLink(`${url}?sig=0`), + () => { + const links = new Promise((_, reject) => { + setTimeout(() => reject(new Error('FetchResults failed')), 200); + }); + refreshed = links.catch(() => undefined); + return links; + }, + { retriesTimeout: 100 }, + ); + const unhandled: Array = []; + const onUnhandled = (reason: unknown) => unhandled.push(reason); + process.on('unhandledRejection', onUnhandled); + try { + const { error } = await fetchWithin(handler, 1000); + expect(error.errorCode).to.be.equal(RetryErrorCode.TimeoutExceeded); + await refreshed; + await new Promise((resolve) => { + setTimeout(resolve, 10); + }); + } finally { + process.removeListener('unhandledRejection', onUnhandled); + } + expect(unhandled).to.deep.equal([]); + }); + + it('should time out on schedule while its link refresh waits for another FetchResults call', async () => { + const url = await startServer((req, res) => { + res.writeHead(403, 'Forbidden'); + res.end(); + }); + const context = new BaseClientContextStub({ cloudFetchConcurrentDownloads: 1, retriesTimeout: 100 }); + sinon + .stub(context.connectionProvider, 'getRetryPolicy') + .callsFake(async (options) => new HttpRetryPolicy(context, options)); + const page = (resultLinks: Array): TFetchResultsResp => ({ + status: { statusCode: TStatusCode.SUCCESS_STATUS }, + hasMoreRows: true, + results: { startRowOffset: new Int64(0), rows: [], resultLinks }, + }); + let releasePage!: () => void; + const pageHeld = new Promise((resolve) => { + releasePage = resolve; + }); + const fetchResults = sinon.stub(context.driver, 'fetchResults'); + fetchResults.onFirstCall().resolves(page([makeLink(`${url}?sig=0`)])); + // The next page's FetchResults hangs, and the link refresh is queued behind it + fetchResults.onSecondCall().callsFake(async () => { + await pageHeld; + return page([]); + }); + fetchResults.resolves(page([makeLink(`${url}?sig=1`)])); + const operationHandle = { + operationId: { guid: Buffer.alloc(16), secret: Buffer.alloc(16) }, + operationType: TOperationType.EXECUTE_STATEMENT, + hasResultSet: true, + }; + const handler = new CloudFetchResultHandler(context, new RowSetProvider(context, operationHandle, [], false), { + lz4Compressed: false, + status: { statusCode: TStatusCode.SUCCESS_STATUS }, + }); + + const first = fetchWithin(handler, 1000); + const second = handler.fetchNext({ limit: 10000 }); + const { error, elapsedMs } = await first; + + expect(error).to.be.instanceOf(RetryError); + expect(error.errorCode).to.be.equal(RetryErrorCode.TimeoutExceeded); + expect(elapsedMs).to.be.lt(500); + + releasePage(); + await second; + // The abandoned refresh was skipped when its turn came: FetchResults ran only for the two pages + expect(fetchResults.callCount).to.be.equal(2); + }); + }); + + it('should include a body read after the retry loop in the deadline', async () => { + // A retry policy that returns a response without running the attempt leaves its body unread + const context = new ClientContextStub({ cloudFetchConcurrentDownloads: 1, retriesTimeout: 100 }); + const stalledBody = new PassThrough(); + stalledBody.write(Buffer.alloc(10)); + context.invokeWithRetryStub.callsFake(async () => ({ + request: new Request('http://localhost/chunk'), + response: new Response(stalledBody, { status: 200 }), + })); + const rowSet: TRowSet = { + startRowOffset: new Int64(0), + rows: [], + resultLinks: [ + { + fileLink: 'http://localhost/chunk', + expiryTime: new Int64(defaultLinkExpiryTime), + rowCount: new Int64(1), + startRowOffset: new Int64(0), + bytesNum: new Int64(0), + }, + ], + }; + const handler = new CloudFetchResultHandler(context, new ResultsProviderStub([rowSet], undefined), { + lz4Compressed: false, + status: { statusCode: TStatusCode.SUCCESS_STATUS }, + }); + + const { error, elapsedMs } = await fetchWithin(handler, 1000); + + expect(error).to.be.instanceOf(RetryError); + expect(error.errorCode).to.be.equal(RetryErrorCode.TimeoutExceeded); + expect(elapsedMs).to.be.lt(500); + }); + + it('should clear the deadline timer when a download completes', async () => { + const body = Buffer.concat([sampleArrowSchema, sampleArrowBatch]); + const url = await startServer((req, res) => { + res.end(body); + }); + const handler = createHandler(url, { retriesTimeout: 4321 }); + const setTimeoutSpy = sinon.spy(global, 'setTimeout'); + const clearTimeoutSpy = sinon.spy(global, 'clearTimeout'); + try { + const { error } = await fetchWithin(handler, 1500); + expect(error).to.be.undefined; + const deadlineTimers = setTimeoutSpy + .getCalls() + .filter((call) => call.args[1] === 4321) + .map((call) => call.returnValue); + expect(deadlineTimers).to.have.length(1); + expect(clearTimeoutSpy.calledWith(deadlineTimers[0])).to.be.true; + } finally { + setTimeoutSpy.restore(); + clearTimeoutSpy.restore(); + } }); it('should not enforce a deadline when retriesTimeout is not positive', async () => { diff --git a/tests/unit/result/RowSetProvider.test.ts b/tests/unit/result/RowSetProvider.test.ts index ab29bff1..390abe92 100644 --- a/tests/unit/result/RowSetProvider.test.ts +++ b/tests/unit/result/RowSetProvider.test.ts @@ -229,6 +229,35 @@ describe('RowSetProvider', () => { expect(maxInFlight).to.equal(1); }); + it('skips a refresh whose caller gave up while it waited for a page fetch', async () => { + const context = new ClientContextStub(); + const provider = new RowSetProvider(context, makeOperationHandle(), [], false); + context.driver.fetchResultsResp = linksResponse('https://storage/0?sig=old', 0, 100); + await provider.fetchNext({ limit: 1000 }); + + let releasePage!: () => void; + const pageHeld = new Promise((resolve) => { + releasePage = resolve; + }); + const fetchResults = sinon.stub(context.driver, 'fetchResults').callsFake(async () => { + await pageHeld; + return linksResponse('https://storage/100?sig=a', 100, 50); + }); + const page = provider.fetchNext({ limit: 1000 }); + const controller = new AbortController(); + const refresh = provider.refreshLinks(new Int64(0), new Int64(100), controller.signal); + const reason = new Error('download timed out'); + controller.abort(reason); + releasePage(); + + expect(await refresh.catch((error) => error)).to.equal(reason); + await page; + expect(fetchResults.callCount).to.equal(1); + // The skipped refresh didn't move the cursor, so the next page isn't requested by offset + await provider.fetchNext({ limit: 1000 }); + expect(fetchResults.secondCall.args[0].startRowOffset).to.be.undefined; + }); + it('returns no links for an operation closed by direct results', async () => { const context = new ClientContextStub(); const fetchResults = sinon.spy(context.driver, 'fetchResults'); From f47d1a63b441b3b55e7eddd0b327003de56eae28 Mon Sep 17 00:00:00 2001 From: Vu Anh Phung Date: Sun, 15 Nov 2026 23:45:57 +0000 Subject: [PATCH 08/10] test: pin the offset match that picks a refreshed CloudFetch link A refresh response without a link at the rejected link's row offset (only links for other rows, or none) must fail the download with the original HTTP error after one refresh, and must never download another chunk's link in its place. Also adds the PR number to the CHANGELOG entry. Signed-off-by: Vu Anh Phung Co-authored-by: Isaac --- CHANGELOG.md | 2 +- .../result/CloudFetchResultHandler.test.ts | 29 +++++++++++++++++++ 2 files changed, 30 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bdf99864..d2abb9a1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,7 +3,7 @@ ## Unreleased - Reduce result chunking and `fetchAll()` overhead for large query results. -- Thrift CloudFetch downloads now recover from more failures. When cloud storage rejects a presigned URL with 401, 403 or 404 (an expired or invalid signature), the driver fetches a fresh link for the same rows from the server and retries with it. A download tries at most `retryMaxAttempts` URLs (the original and its refreshes), and each URL is retried under its own `retryMaxAttempts` budget. Other 4xx responses are retried with the same URL. `socketTimeout` is now also the inactivity timeout of each download attempt, so when it is shorter than `retriesTimeout` (both default to 15 minutes), an attempt that stops receiving data is retried instead of waiting out the deadline; one `retriesTimeout` deadline still covers the whole download, including link refreshes. Previously these failures failed the query, and a stalled attempt waited out the whole deadline. +- Thrift CloudFetch downloads now recover from more failures. When cloud storage rejects a presigned URL with 401, 403 or 404 (an expired or invalid signature), the driver fetches a fresh link for the same rows from the server and retries with it. A download tries at most `retryMaxAttempts` URLs (the original and its refreshes), and each URL is retried under its own `retryMaxAttempts` budget. Other 4xx responses are retried with the same URL. `socketTimeout` is now also the inactivity timeout of each download attempt, so when it is shorter than `retriesTimeout` (both default to 15 minutes), an attempt that stops receiving data is retried instead of waiting out the deadline; one `retriesTimeout` deadline still covers the whole download, including link refreshes. Previously these failures failed the query, and a stalled attempt waited out the whole deadline. (#610) - Thrift CloudFetch downloads are now bounded end to end by `retriesTimeout` (default 15 minutes), including every attempt and retry backoff. Previously a stalled download, where the response headers or body never arrived, blocked the fetch forever. A download still running when the deadline expires is aborted and fails with `RetryError` (`TIMEOUT_EXCEEDED`, message `CloudFetch result download failed: Retry timeout exceeded`). (#520) - Thrift connections made through `proxy` now apply `checkServerCertificate`, `customCaCert` and `clientCert`/`clientKey`. Previously the proxied path ignored them and verified against Node's process-wide trust store, so a certificate outside the configured trust set could be accepted. **Compatibility:** proxied connections now verify unless `checkServerCertificate: false` is set; as on direct connections, `NODE_TLS_REJECT_UNAUTHORIZED=0` no longer turns verification off. For an `https` proxy these options also apply to the proxy's own certificate. (#485) - Honor the client `useCloudFetch: false` default on the kernel backend so complete results can be fetched without cloud-storage downloads. (#574) diff --git a/tests/unit/result/CloudFetchResultHandler.test.ts b/tests/unit/result/CloudFetchResultHandler.test.ts index e522c5f7..72e86e80 100644 --- a/tests/unit/result/CloudFetchResultHandler.test.ts +++ b/tests/unit/result/CloudFetchResultHandler.test.ts @@ -984,6 +984,35 @@ describe('CloudFetchResultHandler', () => { expect(error.message).to.be.equal('CloudFetch HTTP error 403 Forbidden'); }); + [ + { + name: 'only a link for other rows', + links: (url: string) => [{ ...makeLink(`${url}?sig=other`), startRowOffset: new Int64(8) }], + }, + { name: 'no links', links: () => [] }, + ].forEach(({ name, links }) => { + it(`should fail with the HTTP error when the refresh returns ${name}`, async () => { + const hits: Array = []; + const url = await startServer((req, res) => { + hits.push(req.url ?? ''); + if (req.url?.includes('sig=old')) { + res.writeHead(403, 'Forbidden'); + res.end(); + } else { + res.end(body); + } + }); + const { handler, refreshLinks } = createRefreshingHandler(makeLink(`${url}?sig=old`), () => links(url)); + + const { error } = await fetchWithin(handler, 1500); + + expect(error?.message).to.be.equal('CloudFetch HTTP error 403 Forbidden'); + expect(refreshLinks.callCount).to.be.equal(1); + // Another chunk's link must never be downloaded in place of the rejected one + expect(hits).to.deep.eq(['/chunk?sig=old']); + }); + }); + it('should count link refreshes against retryMaxAttempts', async () => { const url = await startServer((req, res) => { res.writeHead(403, 'Forbidden'); From 36bdba4a8d59e164c0eb7e6cc8bfa26899277ba0 Mon Sep 17 00:00:00 2001 From: Vu Anh Phung Date: Mon, 16 Nov 2026 01:59:07 +0000 Subject: [PATCH 09/10] refactor: trim the CloudFetch recovery change to what the spec cells need Drop the hardening that no CLOUDFETCH cell or regression guard needs: skipping a queued link refresh after its download gave up (its late result is already ignored), two deadline checks that the per-attempt check already covers, and the socketTimeout/retryMaxAttempts notes in CONNECTION_PARAMETERS.md and the ConnectionOptions JSDoc. DownloadDeadline.race is now a plain Promise.race, fetch() takes init as optional again so PlainHttpTransport.test.ts is unchanged, and the CHANGELOG entry is shorter. Tests that only covered the removed parts are dropped. Signed-off-by: Vu Anh Phung Co-authored-by: Isaac --- CHANGELOG.md | 2 +- CONNECTION_PARAMETERS.md | 4 +- lib/contracts/IDBSQLClient.ts | 7 --- lib/result/CloudFetchResultHandler.ts | 39 ++++----------- lib/result/RowSetProvider.ts | 11 +---- .../connections/PlainHttpTransport.test.ts | 4 +- .../result/CloudFetchResultHandler.test.ts | 47 ++----------------- tests/unit/result/RowSetProvider.test.ts | 29 ------------ 8 files changed, 16 insertions(+), 127 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d2abb9a1..526cb627 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,7 +3,7 @@ ## Unreleased - Reduce result chunking and `fetchAll()` overhead for large query results. -- Thrift CloudFetch downloads now recover from more failures. When cloud storage rejects a presigned URL with 401, 403 or 404 (an expired or invalid signature), the driver fetches a fresh link for the same rows from the server and retries with it. A download tries at most `retryMaxAttempts` URLs (the original and its refreshes), and each URL is retried under its own `retryMaxAttempts` budget. Other 4xx responses are retried with the same URL. `socketTimeout` is now also the inactivity timeout of each download attempt, so when it is shorter than `retriesTimeout` (both default to 15 minutes), an attempt that stops receiving data is retried instead of waiting out the deadline; one `retriesTimeout` deadline still covers the whole download, including link refreshes. Previously these failures failed the query, and a stalled attempt waited out the whole deadline. (#610) +- Thrift CloudFetch downloads now recover when cloud storage rejects a presigned URL with 401, 403 or 404: the driver fetches a fresh link for the same rows (up to `retryMaxAttempts` URLs per download) and retries with it. Other 4xx responses are retried with the same URL. `socketTimeout` is now also the inactivity timeout of each download attempt, so a stalled attempt is retried when `socketTimeout` is shorter than `retriesTimeout`. One `retriesTimeout` deadline covers the whole download, link refreshes included. Previously these failures failed the query. (#610) - Thrift CloudFetch downloads are now bounded end to end by `retriesTimeout` (default 15 minutes), including every attempt and retry backoff. Previously a stalled download, where the response headers or body never arrived, blocked the fetch forever. A download still running when the deadline expires is aborted and fails with `RetryError` (`TIMEOUT_EXCEEDED`, message `CloudFetch result download failed: Retry timeout exceeded`). (#520) - Thrift connections made through `proxy` now apply `checkServerCertificate`, `customCaCert` and `clientCert`/`clientKey`. Previously the proxied path ignored them and verified against Node's process-wide trust store, so a certificate outside the configured trust set could be accepted. **Compatibility:** proxied connections now verify unless `checkServerCertificate: false` is set; as on direct connections, `NODE_TLS_REJECT_UNAUTHORIZED=0` no longer turns verification off. For an `https` proxy these options also apply to the proxy's own certificate. (#485) - Honor the client `useCloudFetch: false` default on the kernel backend so complete results can be fetched without cloud-storage downloads. (#574) diff --git a/CONNECTION_PARAMETERS.md b/CONNECTION_PARAMETERS.md index 95cc2c40..e0503021 100644 --- a/CONNECTION_PARAMETERS.md +++ b/CONNECTION_PARAMETERS.md @@ -78,11 +78,11 @@ Per-statement `useCloudFetch` overrides remain unsupported on the kernel backend | Option | Type | Thrift | Kernel | Default Value | Note | | --------------------------- | ------------------------ | :----: | :----: | ------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `socketTimeout` | `number` (ms) | ✅ | ⚠️ | `900000` / `120000` | Kernel forwards only positive values. `0` means no timeout on Thrift but is omitted on Kernel, which then keeps its 120-second default. On Thrift it is also the inactivity timeout of each CloudFetch download attempt. | +| `socketTimeout` | `number` (ms) | ✅ | ⚠️ | `900000` / `120000` | Kernel forwards only positive values. `0` means no timeout on Thrift but is omitted on Kernel, which then keeps its 120-second default. | | `proxy` | `ProxyOptions` | ✅ | ⚠️ | — | `{protocol, host, port, auth}`. Kernel supports HTTP/HTTPS proxies only; Thrift also supports the declared SOCKS variants. | | `noProxy` (internal) | `string` | ❌ | ⚠️ | — | Runtime-only Kernel bypass list, forwarded as `bypassHosts` only when an explicit `proxy` is also supplied. | | `customHeaders` | `Record` | ✅ | ✅ | — | Sent on workspace, telemetry and feature-flag requests, not OAuth. Workspace calls ignore caller `Authorization`, `User-Agent`, `Content-Type`, `Content-Length`, `Transfer-Encoding`, `Connection`; Kernel also ignores `x-databricks-org-id`. | -| `retryMaxAttempts` | `number` | ✅ | ✅ | `5` | Total attempts, including the initial request. Kernel converts this to its retries-after-first representation internally. On Thrift it also caps the CloudFetch URLs per download; each URL is retried under its own budget. | +| `retryMaxAttempts` | `number` | ✅ | ✅ | `5` | Total attempts, including the initial request. Kernel converts this to its retries-after-first representation internally. | | `retriesTimeout` | `number` (ms) | ✅ | ✅ | `900000` | Kernel converts milliseconds to whole seconds. | | `retryDelayMin` | `number` (ms) | ✅ | ✅ | `1000` | Kernel converts milliseconds to whole seconds. | | `retryDelayMax` | `number` (ms) | ✅ | ✅ | `60000` | Kernel converts milliseconds to whole seconds. | diff --git a/lib/contracts/IDBSQLClient.ts b/lib/contracts/IDBSQLClient.ts index 8b7c6ecb..5c302bb0 100644 --- a/lib/contracts/IDBSQLClient.ts +++ b/lib/contracts/IDBSQLClient.ts @@ -70,13 +70,6 @@ export type ConnectionOptions = { port?: number; path: string; userAgentEntry?: string; - /** - * Timeout in milliseconds. Thrift: how long each request may wait for its response headers, - * and then for its body (default 900000, 15 minutes; `0` disables it). It is also the - * inactivity timeout of each CloudFetch download attempt: an attempt that receives nothing - * for this long is aborted and retried. Kernel: the socket read timeout, used only when - * positive; otherwise the kernel keeps its 120-second default. - */ socketTimeout?: number; proxy?: ProxyOptions; enableMetricViewMetadata?: boolean; diff --git a/lib/result/CloudFetchResultHandler.ts b/lib/result/CloudFetchResultHandler.ts index cf74e868..e4a8a84c 100644 --- a/lib/result/CloudFetchResultHandler.ts +++ b/lib/result/CloudFetchResultHandler.ts @@ -27,8 +27,7 @@ const MAX_TIMER_DELAY_MS = 2 ** 31 - 1; const LINK_REFRESH_STATUSES: ReadonlySet = new Set([401, 403, 404]); interface LinkRefresher { - // An aborted `signal` means the caller gave up; a refresh still queued may then be skipped - refreshLinks(startRowOffset: Int64, maxRows: Int64, signal?: AbortSignal): Promise>; + refreshLinks(startRowOffset: Int64, maxRows: Int64): Promise>; } function canRefreshLinks(source: unknown): source is LinkRefresher { @@ -67,24 +66,10 @@ class DownloadDeadline { if (!signal) { return promise; } - return new Promise((resolve, reject) => { - const onExpired = () => reject(signal.reason); - if (signal.aborted) { - onExpired(); - } else { - signal.addEventListener('abort', onExpired, { once: true }); - } - promise.then( - (value) => { - signal.removeEventListener('abort', onExpired); - resolve(value); - }, - (error) => { - signal.removeEventListener('abort', onExpired); - reject(error); - }, - ); + const expired = new Promise((_, reject) => { + signal.addEventListener('abort', () => reject(signal.reason), { once: true }); }); + return Promise.race([promise, expired]); } public clear(): void { @@ -219,12 +204,10 @@ export default class CloudFetchResultHandler implements IResultsProvider { + private async refreshLink(link: TSparkArrowResultLink, status: number): Promise { if (!canRefreshLinks(this.source)) { return undefined; } @@ -264,13 +241,13 @@ export default class CloudFetchResultHandler implements IResultsProvider candidate.startRowOffset.toNumber() === offset); } private async fetch( url: RequestInfo, - init: RequestInit, + init?: RequestInit, deadline: DownloadDeadline = new DownloadDeadline(0), ): Promise<{ response: Response; body?: ArrayBuffer }> { const connectionProvider = await this.context.getConnectionProvider(); diff --git a/lib/result/RowSetProvider.ts b/lib/result/RowSetProvider.ts index 726c0aa0..6830cfe9 100644 --- a/lib/result/RowSetProvider.ts +++ b/lib/result/RowSetProvider.ts @@ -157,22 +157,13 @@ export default class RowSetProvider implements IResultsProvider> { + public async refreshLinks(startRowOffset: Int64, maxRows: Int64): Promise> { if (this.returnOnlyPrefetchedResults) { return []; } const driver = await this.context.getDriver(); const response = await this.serializeFetch(async () => { - if (signal?.aborted) { - throw signal.reason; - } this.fetchByOffset = true; return driver.fetchResults({ operationHandle: this.operationHandle, diff --git a/tests/unit/connection/connections/PlainHttpTransport.test.ts b/tests/unit/connection/connections/PlainHttpTransport.test.ts index c2799da5..425387d1 100644 --- a/tests/unit/connection/connections/PlainHttpTransport.test.ts +++ b/tests/unit/connection/connections/PlainHttpTransport.test.ts @@ -146,9 +146,7 @@ describe('DBSQLClient plain HTTP transport', () => { const handler = new CloudFetchResultHandler(client, new ResultsProviderStub([], undefined), { status: { statusCode: TStatusCode.SUCCESS_STATUS }, }); - await expectTlsAttempt(() => - handler['fetch'](`https://127.0.0.1:${port}/chunk?X-Amz-Signature=secret`, {}, undefined), - ); + await expectTlsAttempt(() => handler['fetch'](`https://127.0.0.1:${port}/chunk?X-Amz-Signature=secret`)); await client.close(); // The HTTPS attempt never reached the server as an HTTP request. diff --git a/tests/unit/result/CloudFetchResultHandler.test.ts b/tests/unit/result/CloudFetchResultHandler.test.ts index 72e86e80..fd3ba934 100644 --- a/tests/unit/result/CloudFetchResultHandler.test.ts +++ b/tests/unit/result/CloudFetchResultHandler.test.ts @@ -2,7 +2,6 @@ import { expect, AssertionError } from 'chai'; import sinon, { SinonStub } from 'sinon'; import http from 'http'; import { AddressInfo } from 'net'; -import { PassThrough } from 'stream'; import Int64 from 'node-int64'; import { compressFrameSync } from 'lz4-napi'; import { Request, Response } from 'node-fetch'; @@ -1190,11 +1189,9 @@ describe('CloudFetchResultHandler', () => { res.writeHead(403, 'Forbidden'); res.end(); }); - const { handler, refreshLinks } = createRefreshingHandler( - makeLink(`${url}?sig=0`), - () => new Promise(() => {}), - { retriesTimeout: 100 }, - ); + const { handler } = createRefreshingHandler(makeLink(`${url}?sig=0`), () => new Promise(() => {}), { + retriesTimeout: 100, + }); const { error, elapsedMs } = await fetchWithin(handler, 1000); @@ -1204,8 +1201,6 @@ describe('CloudFetchResultHandler', () => { 'CloudFetch result download failed: Retry timeout exceeded (last error: HTTP 403: Forbidden)', ); expect(elapsedMs).to.be.lt(500); - // The refresh is told that the download gave up - expect(refreshLinks.firstCall.args[2].aborted).to.be.true; }); it('should not leak the rejection of a link refresh that fails after retriesTimeout', async () => { @@ -1287,43 +1282,7 @@ describe('CloudFetchResultHandler', () => { releasePage(); await second; - // The abandoned refresh was skipped when its turn came: FetchResults ran only for the two pages - expect(fetchResults.callCount).to.be.equal(2); - }); - }); - - it('should include a body read after the retry loop in the deadline', async () => { - // A retry policy that returns a response without running the attempt leaves its body unread - const context = new ClientContextStub({ cloudFetchConcurrentDownloads: 1, retriesTimeout: 100 }); - const stalledBody = new PassThrough(); - stalledBody.write(Buffer.alloc(10)); - context.invokeWithRetryStub.callsFake(async () => ({ - request: new Request('http://localhost/chunk'), - response: new Response(stalledBody, { status: 200 }), - })); - const rowSet: TRowSet = { - startRowOffset: new Int64(0), - rows: [], - resultLinks: [ - { - fileLink: 'http://localhost/chunk', - expiryTime: new Int64(defaultLinkExpiryTime), - rowCount: new Int64(1), - startRowOffset: new Int64(0), - bytesNum: new Int64(0), - }, - ], - }; - const handler = new CloudFetchResultHandler(context, new ResultsProviderStub([rowSet], undefined), { - lz4Compressed: false, - status: { statusCode: TStatusCode.SUCCESS_STATUS }, }); - - const { error, elapsedMs } = await fetchWithin(handler, 1000); - - expect(error).to.be.instanceOf(RetryError); - expect(error.errorCode).to.be.equal(RetryErrorCode.TimeoutExceeded); - expect(elapsedMs).to.be.lt(500); }); it('should clear the deadline timer when a download completes', async () => { diff --git a/tests/unit/result/RowSetProvider.test.ts b/tests/unit/result/RowSetProvider.test.ts index 390abe92..ab29bff1 100644 --- a/tests/unit/result/RowSetProvider.test.ts +++ b/tests/unit/result/RowSetProvider.test.ts @@ -229,35 +229,6 @@ describe('RowSetProvider', () => { expect(maxInFlight).to.equal(1); }); - it('skips a refresh whose caller gave up while it waited for a page fetch', async () => { - const context = new ClientContextStub(); - const provider = new RowSetProvider(context, makeOperationHandle(), [], false); - context.driver.fetchResultsResp = linksResponse('https://storage/0?sig=old', 0, 100); - await provider.fetchNext({ limit: 1000 }); - - let releasePage!: () => void; - const pageHeld = new Promise((resolve) => { - releasePage = resolve; - }); - const fetchResults = sinon.stub(context.driver, 'fetchResults').callsFake(async () => { - await pageHeld; - return linksResponse('https://storage/100?sig=a', 100, 50); - }); - const page = provider.fetchNext({ limit: 1000 }); - const controller = new AbortController(); - const refresh = provider.refreshLinks(new Int64(0), new Int64(100), controller.signal); - const reason = new Error('download timed out'); - controller.abort(reason); - releasePage(); - - expect(await refresh.catch((error) => error)).to.equal(reason); - await page; - expect(fetchResults.callCount).to.equal(1); - // The skipped refresh didn't move the cursor, so the next page isn't requested by offset - await provider.fetchNext({ limit: 1000 }); - expect(fetchResults.secondCall.args[0].startRowOffset).to.be.undefined; - }); - it('returns no links for an operation closed by direct results', async () => { const context = new ClientContextStub(); const fetchResults = sinon.spy(context.driver, 'fetchResults'); From 812281e6cd74bf6ab1e405757510445bf6d5deb1 Mon Sep 17 00:00:00 2001 From: Vu Anh Phung Date: Mon, 16 Nov 2026 03:06:37 +0000 Subject: [PATCH 10/10] fix: retry a one-shot CloudFetch 401/403/404 with the same URL instead of refreshing the link Per the relaxed CloudFetch spec (a one-shot 401/403/404 must not fail the query; a refresh or a same-URL retry both count, and other 4xx may fail with a clear error), drop the link refresh, the RowSetProvider offset paging and serialization, the cross-refresh download deadline and the other-4xx retry opt-in. A CloudFetch download now marks a 401/403/404 response retryable, so HttpRetryPolicy retries the same URL within its existing attempt budget, backoff and the #520 per-download deadline. 400/405/412 keep failing with 'CloudFetch HTTP error '. The per-attempt socketTimeout idle timeout stays. Signed-off-by: Vu Anh Phung Co-authored-by: Isaac --- CHANGELOG.md | 2 +- lib/connection/connections/HttpConnection.ts | 6 +- lib/connection/connections/HttpRetryPolicy.ts | 18 +- .../contracts/IConnectionProvider.ts | 12 +- lib/result/CloudFetchResultHandler.ts | 234 +++++------- lib/result/RowSetProvider.ts | 60 +--- tests/unit/.stubs/ConnectionProviderStub.ts | 8 +- .../connections/HttpRetryPolicy.test.ts | 38 +- .../result/CloudFetchResultHandler.test.ts | 334 ++---------------- tests/unit/result/RowSetProvider.test.ts | 81 +---- 10 files changed, 157 insertions(+), 636 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 526cb627..e6c4b382 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,7 +3,7 @@ ## Unreleased - Reduce result chunking and `fetchAll()` overhead for large query results. -- Thrift CloudFetch downloads now recover when cloud storage rejects a presigned URL with 401, 403 or 404: the driver fetches a fresh link for the same rows (up to `retryMaxAttempts` URLs per download) and retries with it. Other 4xx responses are retried with the same URL. `socketTimeout` is now also the inactivity timeout of each download attempt, so a stalled attempt is retried when `socketTimeout` is shorter than `retriesTimeout`. One `retriesTimeout` deadline covers the whole download, link refreshes included. Previously these failures failed the query. (#610) +- Thrift CloudFetch downloads now retry a 401, 403 or 404 from cloud storage with the same URL, like 429 and 5xx, within the `retryMaxAttempts`/`retriesTimeout` budget; other 4xx responses still fail with `CloudFetch HTTP error `. `socketTimeout` is now also the inactivity timeout of each download attempt, so a stalled attempt is retried when `socketTimeout` is shorter than `retriesTimeout`. Previously a one-shot 401/403/404 failed the query and a stalled attempt waited out `retriesTimeout`. (#610) - Thrift CloudFetch downloads are now bounded end to end by `retriesTimeout` (default 15 minutes), including every attempt and retry backoff. Previously a stalled download, where the response headers or body never arrived, blocked the fetch forever. A download still running when the deadline expires is aborted and fails with `RetryError` (`TIMEOUT_EXCEEDED`, message `CloudFetch result download failed: Retry timeout exceeded`). (#520) - Thrift connections made through `proxy` now apply `checkServerCertificate`, `customCaCert` and `clientCert`/`clientKey`. Previously the proxied path ignored them and verified against Node's process-wide trust store, so a certificate outside the configured trust set could be accepted. **Compatibility:** proxied connections now verify unless `checkServerCertificate: false` is set; as on direct connections, `NODE_TLS_REJECT_UNAUTHORIZED=0` no longer turns verification off. For an `https` proxy these options also apply to the proxy's own certificate. (#485) - Honor the client `useCloudFetch: false` default on the kernel backend so complete results can be fetched without cloud-storage downloads. (#574) diff --git a/lib/connection/connections/HttpConnection.ts b/lib/connection/connections/HttpConnection.ts index 5dc1a3ce..6472265a 100644 --- a/lib/connection/connections/HttpConnection.ts +++ b/lib/connection/connections/HttpConnection.ts @@ -4,7 +4,7 @@ import http from 'http'; import { HeadersInit } from 'node-fetch'; import { ProxyAgent } from 'proxy-agent'; -import IConnectionProvider, { HttpRetryPolicyOptions, HttpTransactionDetails } from '../contracts/IConnectionProvider'; +import IConnectionProvider, { HttpTransactionDetails } from '../contracts/IConnectionProvider'; import IConnectionOptions, { ProxyOptions } from '../contracts/IConnectionOptions'; import IClientContext from '../../contracts/IClientContext'; import { LogLevel } from '../../contracts/IDBSQLLogger'; @@ -227,7 +227,7 @@ export default class HttpConnection implements IConnectionProvider { return this.connection; } - public async getRetryPolicy(options?: HttpRetryPolicyOptions): Promise> { - return new HttpRetryPolicy(this.context, options); + public async getRetryPolicy(): Promise> { + return new HttpRetryPolicy(this.context); } } diff --git a/lib/connection/connections/HttpRetryPolicy.ts b/lib/connection/connections/HttpRetryPolicy.ts index 41cb4633..4c5d996d 100644 --- a/lib/connection/connections/HttpRetryPolicy.ts +++ b/lib/connection/connections/HttpRetryPolicy.ts @@ -1,5 +1,5 @@ import IRetryPolicy, { ShouldRetryResult, RetryableOperation } from '../contracts/IRetryPolicy'; -import { HttpRetryPolicyOptions, HttpTransactionDetails } from '../contracts/IConnectionProvider'; +import { HttpTransactionDetails } from '../contracts/IConnectionProvider'; import IClientContext from '../../contracts/IClientContext'; import RetryError, { RetryErrorCode } from '../../errors/RetryError'; @@ -53,13 +53,10 @@ export default class HttpRetryPolicy implements IRetryPolicy { @@ -194,22 +191,17 @@ export default class HttpRetryPolicy implements IRetryPolicy= 500 && statusCode !== 501) || - // ...and, when enabled, on the other `4xx` codes - (retryClientErrorsExcept !== undefined && - statusCode >= 400 && - statusCode < 500 && - !retryClientErrorsExcept.has(statusCode)); + (statusCode >= 500 && statusCode !== 501); return result; } diff --git a/lib/connection/contracts/IConnectionProvider.ts b/lib/connection/contracts/IConnectionProvider.ts index 935ead37..2ba9e884 100644 --- a/lib/connection/contracts/IConnectionProvider.ts +++ b/lib/connection/contracts/IConnectionProvider.ts @@ -5,14 +5,8 @@ import IRetryPolicy from './IRetryPolicy'; export interface HttpTransactionDetails { request: Request; response: Response; -} - -export interface HttpRetryPolicyOptions { - /** - * Also retry 4xx responses, except these statuses. For idempotent GETs of presigned - * URLs (CloudFetch), where a 4xx from storage does not mean the request was wrong. - */ - retryClientErrorsExcept?: ReadonlySet; + // Set by the caller to retry this response whatever its status + retryable?: boolean; } export default interface IConnectionProvider { @@ -26,5 +20,5 @@ export default interface IConnectionProvider { setHeaders(headers: HeadersInit): void; - getRetryPolicy(options?: HttpRetryPolicyOptions): Promise>; + getRetryPolicy(): Promise>; } diff --git a/lib/result/CloudFetchResultHandler.ts b/lib/result/CloudFetchResultHandler.ts index e4a8a84c..38af6425 100644 --- a/lib/result/CloudFetchResultHandler.ts +++ b/lib/result/CloudFetchResultHandler.ts @@ -1,5 +1,4 @@ import fetch, { RequestInfo, RequestInit, Request, Response } from 'node-fetch'; -import Int64 from 'node-int64'; import { TGetResultSetMetadataResp, TRowSet, TSparkArrowResultLink } from '../../thrift/TCLIService_types'; import HiveDriverError from '../errors/HiveDriverError'; import RetryError, { RetryErrorCode } from '../errors/RetryError'; @@ -21,61 +20,9 @@ type SettledDownload = { ok: true; batch: ArrowBatch } | { ok: false; error: unk // Largest delay `setTimeout` accepts; longer values fire immediately, so they are clamped. const MAX_TIMER_DELAY_MS = 2 ** 31 - 1; -// Statuses that mean the presigned URL itself was rejected (expired or invalid signature; GCS -// answers 404 for an expired one). Retrying the same URL can't succeed, so the link is refreshed. -// Other 4xx responses are retried with the same URL. -const LINK_REFRESH_STATUSES: ReadonlySet = new Set([401, 403, 404]); - -interface LinkRefresher { - refreshLinks(startRowOffset: Int64, maxRows: Int64): Promise>; -} - -function canRefreshLinks(source: unknown): source is LinkRefresher { - return typeof (source as Partial | undefined)?.refreshLinks === 'function'; -} - -// The `retriesTimeout` deadline of one download: its GET attempts, the backoff between them and -// its link refreshes. On expiry `signal` is aborted with a `TimeoutExceeded` error. -class DownloadDeadline { - public readonly signal?: AbortSignal; - - // Reported as the cause of the timeout error - public lastFailure: unknown; - - private readonly timer?: NodeJS.Timeout; - - constructor(timeoutMs: number) { - if (timeoutMs > 0) { - const controller = new AbortController(); - this.signal = controller.signal; - this.timer = setTimeout(() => { - controller.abort(new RetryError(RetryErrorCode.TimeoutExceeded, this.lastFailure)); - }, Math.min(timeoutMs, MAX_TIMER_DELAY_MS)); - } - } - - public throwIfExpired(): void { - if (this.signal?.aborted) { - throw this.signal.reason; - } - } - - // Settles like `promise`, or rejects with the timeout error as soon as the deadline expires - public race(promise: Promise): Promise { - const { signal } = this; - if (!signal) { - return promise; - } - const expired = new Promise((_, reject) => { - signal.addEventListener('abort', () => reject(signal.reason), { once: true }); - }); - return Promise.race([promise, expired]); - } - - public clear(): void { - clearTimeout(this.timer); - } -} +// A one-shot 401/403/404 from cloud storage is retried with the same URL, like 429 and 5xx. +// Other 4xx responses fail the download. +const SAME_URL_RETRY_STATUSES: ReadonlySet = new Set([401, 403, 404]); export default class CloudFetchResultHandler implements IResultsProvider { private readonly context: IClientContext; @@ -184,34 +131,8 @@ export default class CloudFetchResultHandler implements IResultsProvider { const startTime = Date.now(); - const { retryMaxAttempts } = this.context.getConfig(); - let currentLink = link; - let { response, body } = await this.fetch(currentLink.fileLink, { headers: currentLink.httpHeaders }, deadline); - // At most `retryMaxAttempts` URLs per download; each `fetch` retries its URL under its own budget - for (let attempt = 1; LINK_REFRESH_STATUSES.has(response.status) && attempt < retryMaxAttempts; attempt += 1) { - // eslint-disable-next-line no-await-in-loop - const freshLink = await this.refreshLink(currentLink, response.status); - if (!freshLink) { - break; - } - currentLink = freshLink; - // eslint-disable-next-line no-await-in-loop - ({ response, body } = await this.fetch(currentLink.fileLink, { headers: currentLink.httpHeaders }, deadline)); - } + const { response, body } = await this.fetch(link.fileLink, { headers: link.httpHeaders }); if (!response.ok) { throw new Error(`CloudFetch HTTP error ${response.status} ${response.statusText}`); } @@ -232,38 +153,31 @@ export default class CloudFetchResultHandler implements IResultsProvider { - if (!canRefreshLinks(this.source)) { - return undefined; - } - const offset = link.startRowOffset.toNumber(); - this.context - .getLogger() - .log(LogLevel.debug, `CloudFetch download got HTTP ${status}; refreshing the link at row offset ${offset}`); - const links = await this.source.refreshLinks(link.startRowOffset, link.rowCount); - return links.find((candidate) => candidate.startRowOffset.toNumber() === offset); - } - - private async fetch( - url: RequestInfo, - init?: RequestInit, - deadline: DownloadDeadline = new DownloadDeadline(0), - ): Promise<{ response: Response; body?: ArrayBuffer }> { + private async fetch(url: RequestInfo, init?: RequestInit): Promise<{ response: Response; body?: ArrayBuffer }> { const connectionProvider = await this.context.getConnectionProvider(); let requestUrl = url; if (typeof requestUrl !== 'string') { requestUrl = 'href' in requestUrl ? requestUrl.href : requestUrl.url; } const agent = await connectionProvider.getAgent(requestUrl); - const retryPolicy = await connectionProvider.getRetryPolicy({ retryClientErrorsExcept: LINK_REFRESH_STATUSES }); - const { socketTimeout } = this.context.getConfig(); + const retryPolicy = await connectionProvider.getRetryPolicy(); + const { retriesTimeout, socketTimeout } = this.context.getConfig(); // `socketTimeout` is an inactivity timeout for each attempt: an attempt that receives // nothing for that long is aborted and retried like a network timeout. const idleTimeoutMs = socketTimeout > 0 ? Math.min(socketTimeout, MAX_TIMER_DELAY_MS) : undefined; const requestConfig: RequestInit = { agent, ...init }; - const { signal } = deadline; + // The retry policy checks `retriesTimeout` only between attempts, so a stalled attempt + // (headers or body never arrive) would block forever. Enforce it as a deadline over the + // whole download - every attempt plus the backoff between them - and abort the in-flight + // request and any pending backoff sleep when it expires (#520). Non-positive values keep + // the deadline off. + const deadlineMs = retriesTimeout > 0 ? Math.min(retriesTimeout, MAX_TIMER_DELAY_MS) : undefined; + // Aborted only once, by the deadline, so a single controller serves every attempt and backoff. + const controller = deadlineMs === undefined ? undefined : new AbortController(); + let timeoutError: RetryError | undefined; + let lastFailure: unknown; + let timer: NodeJS.Timeout | undefined; // Read the body inside the retried block. CloudFetch downloads are large // GETs against pre-signed cloud-storage URLs that frequently surface @@ -271,54 +185,78 @@ export default class CloudFetchResultHandler implements IResultsProvider { - // The timeout error is not retryable, so this also ends the retry loop when the - // deadline expired during a backoff sleep or aborted the attempt. - deadline.throwIfExpired(); - downloaded = undefined; - // Aborted by the deadline or when the attempt goes idle - const attempt = new AbortController(); - const abortAttempt = () => attempt.abort(); - signal?.addEventListener('abort', abortAttempt, { once: true }); - let attemptTimedOut = false; - let idleTimer: NodeJS.Timeout | undefined; - const restartIdleTimer = () => { - if (idleTimeoutMs !== undefined) { - clearTimeout(idleTimer); - idleTimer = setTimeout(() => { - attemptTimedOut = true; - attempt.abort(); - }, idleTimeoutMs); + let result: { request: Request; response: Response }; + try { + const download = retryPolicy.invokeWithRetry(async () => { + // The timeout error is not retryable, so this also ends the retry loop when the + // deadline expired during a backoff sleep or aborted the attempt. + if (timeoutError) { + throw timeoutError; } - }; - restartIdleTimer(); - const request = new Request(url, { - ...requestConfig, - signal: attempt.signal as unknown as RequestInit['signal'], - }); - try { - const response = await fetch(request); + downloaded = undefined; + // Aborted by the deadline or when the attempt goes idle + const attempt = new AbortController(); + const abortAttempt = () => attempt.abort(); + controller?.signal.addEventListener('abort', abortAttempt, { once: true }); + let attemptTimedOut = false; + let idleTimer: NodeJS.Timeout | undefined; + const restartIdleTimer = () => { + if (idleTimeoutMs !== undefined) { + clearTimeout(idleTimer); + idleTimer = setTimeout(() => { + attemptTimedOut = true; + attempt.abort(); + }, idleTimeoutMs); + } + }; restartIdleTimer(); - if (response.ok) { - const bodyRead = response.arrayBuffer(); - response.body.on('data', restartIdleTimer); - downloaded = await bodyRead; - } else { - deadline.lastFailure = { request, response }; + const request = new Request(url, { + ...requestConfig, + signal: attempt.signal as unknown as RequestInit['signal'], + }); + try { + const response = await fetch(request); + restartIdleTimer(); + if (response.ok) { + const bodyRead = response.arrayBuffer(); + response.body.on('data', restartIdleTimer); + downloaded = await bodyRead; + } else { + lastFailure = { request, response }; + } + return { request, response, retryable: SAME_URL_RETRY_STATUSES.has(response.status) }; + } catch (error) { + if (timeoutError) { + throw timeoutError; + } + lastFailure = attemptTimedOut + ? Object.assign(new Error('CloudFetch download attempt timed out'), { code: 'ETIMEDOUT' }) + : error; + throw lastFailure; + } finally { + clearTimeout(idleTimer); + controller?.signal.removeEventListener('abort', abortAttempt); } - return { request, response }; - } catch (error) { - deadline.throwIfExpired(); - deadline.lastFailure = attemptTimedOut - ? Object.assign(new Error('CloudFetch download attempt timed out'), { code: 'ETIMEDOUT' }) - : error; - throw deadline.lastFailure; - } finally { - clearTimeout(idleTimer); - signal?.removeEventListener('abort', abortAttempt); + }, controller?.signal); + + if (deadlineMs === undefined) { + result = await download; + } else { + const deadline = new Promise((_, reject) => { + timer = setTimeout(() => { + timeoutError = new RetryError(RetryErrorCode.TimeoutExceeded, lastFailure); + // Aborts the in-flight request, and rejects a pending backoff sleep with `timeoutError` + controller?.abort(timeoutError); + reject(timeoutError); + }, deadlineMs); + }); + result = await Promise.race([download, deadline]); } - }, signal); + } catch (error) { + throw CloudFetchResultHandler.describeDownloadError(error); + } finally { + clearTimeout(timer); + } // Fall back to reading the body here if the retry policy returned a // response without consuming it via our operation (e.g. unit-test stubs // that hand back a pre-baked Response without invoking the operation diff --git a/lib/result/RowSetProvider.ts b/lib/result/RowSetProvider.ts index 6830cfe9..8c67a9f4 100644 --- a/lib/result/RowSetProvider.ts +++ b/lib/result/RowSetProvider.ts @@ -5,7 +5,6 @@ import { TFetchResultsResp, TOperationHandle, TRowSet, - TSparkArrowResultLink, } from '../../thrift/TCLIService_types'; import Status from '../dto/Status'; import IClientContext from '../contracts/IClientContext'; @@ -99,21 +98,6 @@ export default class RowSetProvider implements IResultsProvider = Promise.resolve(); - - private serializeFetch(fetchResults: () => Promise): Promise { - const result = this.lastFetch.then(fetchResults, fetchResults); - this.lastFetch = result.catch(() => undefined); - return result; - } - private get hasMoreRows(): boolean { // `hasMoreRowsFlag` is populated only after fetching the first row set. // Prior to that, we use a `operationHandle.hasResultSet` flag which @@ -145,38 +129,9 @@ export default class RowSetProvider implements IResultsProvider { - this.nextLinkRowOffset = Math.max( - this.nextLinkRowOffset, - link.startRowOffset.toNumber() + link.rowCount.toNumber(), - ); - }); return response.results; } - /** - * Fetches fresh result links starting at `startRowOffset`, e.g. to replace a presigned - * URL that cloud storage rejected. Returns no links when the operation is already closed. - */ - public async refreshLinks(startRowOffset: Int64, maxRows: Int64): Promise> { - if (this.returnOnlyPrefetchedResults) { - return []; - } - const driver = await this.context.getDriver(); - const response = await this.serializeFetch(async () => { - this.fetchByOffset = true; - return driver.fetchResults({ - operationHandle: this.operationHandle, - orientation: TFetchOrientation.FETCH_NEXT, - maxRows, - fetchType: FetchType.Data, - startRowOffset, - }); - }); - Status.assert(response.status); - return response.results?.resultLinks ?? []; - } - public async fetchNext({ limit }: ResultsProviderFetchNextOptions) { const prefetchedResponse = this.prefetchedResults.shift(); if (prefetchedResponse) { @@ -195,15 +150,12 @@ export default class RowSetProvider implements IResultsProvider - driver.fetchResults({ - operationHandle: this.operationHandle, - orientation: this.fetchOrientation, - maxRows: new Int64(limit), - fetchType: FetchType.Data, - ...(this.fetchByOffset ? { startRowOffset: new Int64(this.nextLinkRowOffset) } : {}), - }), - ); + const response = await driver.fetchResults({ + operationHandle: this.operationHandle, + orientation: this.fetchOrientation, + maxRows: new Int64(limit), + fetchType: FetchType.Data, + }); const latencyMs = Date.now() - startTime; this.emitChunkEvent(latencyMs, response); diff --git a/tests/unit/.stubs/ConnectionProviderStub.ts b/tests/unit/.stubs/ConnectionProviderStub.ts index 579d441b..20b54dfd 100644 --- a/tests/unit/.stubs/ConnectionProviderStub.ts +++ b/tests/unit/.stubs/ConnectionProviderStub.ts @@ -1,9 +1,6 @@ import http from 'http'; import { HeadersInit } from 'node-fetch'; -import IConnectionProvider, { - HttpRetryPolicyOptions, - HttpTransactionDetails, -} from '../../../lib/connection/contracts/IConnectionProvider'; +import IConnectionProvider, { HttpTransactionDetails } from '../../../lib/connection/contracts/IConnectionProvider'; import IRetryPolicy from '../../../lib/connection/contracts/IRetryPolicy'; import NullRetryPolicy from '../../../lib/connection/connections/NullRetryPolicy'; @@ -22,8 +19,7 @@ export default class ConnectionProviderStub implements IConnectionProvider { this.headers = headers; } - // eslint-disable-next-line @typescript-eslint/no-unused-vars - public async getRetryPolicy(options?: HttpRetryPolicyOptions): Promise> { + public async getRetryPolicy(): Promise> { return new NullRetryPolicy(); } } diff --git a/tests/unit/connection/connections/HttpRetryPolicy.test.ts b/tests/unit/connection/connections/HttpRetryPolicy.test.ts index 96ccece0..0577c9c9 100644 --- a/tests/unit/connection/connections/HttpRetryPolicy.test.ts +++ b/tests/unit/connection/connections/HttpRetryPolicy.test.ts @@ -103,27 +103,27 @@ describe('HttpRetryPolicy', () => { expect(result.shouldRetry).to.be.false; }); - it('should retry 4xx responses only when enabled, except the listed statuses', async () => { - function createStub(status: number) { - return { - request: new Request('http://localhost'), - response: new Response(undefined, { status }), - }; - } - - const context = new ClientContextStub({ retryMaxAttempts: 100 }); - const defaultPolicy = new HttpRetryPolicy(context); - const cloudFetchPolicy = new HttpRetryPolicy(context, { retryClientErrorsExcept: new Set([401, 403, 404]) }); + it('should retry a response the caller marks retryable, within the attempts budget', async () => { + const context = new ClientContextStub({ retryMaxAttempts: 3 }); + const policy = new HttpRetryPolicy(context); + const forbidden = () => ({ + request: new Request('http://localhost'), + response: new Response(undefined, { status: 403 }), + }); - for (const status of [400, 405, 408, 412]) { - expect((await defaultPolicy.shouldRetry(createStub(status))).shouldRetry, `${status}`).to.be.false; - expect((await cloudFetchPolicy.shouldRetry(createStub(status))).shouldRetry, `${status}`).to.be.true; - } - for (const status of [401, 403, 404]) { - expect((await cloudFetchPolicy.shouldRetry(createStub(status))).shouldRetry, `${status}`).to.be.false; + expect((await policy.shouldRetry(forbidden())).shouldRetry).to.be.false; + expect((await policy.shouldRetry({ ...forbidden(), retryable: true })).shouldRetry).to.be.true; + expect((await policy.shouldRetry({ ...forbidden(), retryable: true })).shouldRetry).to.be.true; + try { + await policy.shouldRetry({ ...forbidden(), retryable: true }); + expect.fail('It should throw an error'); + } catch (error) { + if (error instanceof AssertionError) { + throw error; + } + expect(error).to.be.instanceOf(RetryError); + expect((error as RetryError).errorCode).to.equal(RetryErrorCode.AttemptsExceeded); } - expect((await cloudFetchPolicy.shouldRetry(createStub(501))).shouldRetry).to.be.false; - expect((await cloudFetchPolicy.shouldRetry(createStub(503))).shouldRetry).to.be.true; }); it('should use `Retry-After` header as a base for backoff', async () => { diff --git a/tests/unit/result/CloudFetchResultHandler.test.ts b/tests/unit/result/CloudFetchResultHandler.test.ts index fd3ba934..0539e13c 100644 --- a/tests/unit/result/CloudFetchResultHandler.test.ts +++ b/tests/unit/result/CloudFetchResultHandler.test.ts @@ -9,15 +9,8 @@ import { ShouldRetryResult } from '../../../lib/connection/contracts/IRetryPolic import { HttpTransactionDetails } from '../../../lib/connection/contracts/IConnectionProvider'; import { LogLevel } from '../../../lib/contracts/IDBSQLLogger'; import CloudFetchResultHandler from '../../../lib/result/CloudFetchResultHandler'; -import RowSetProvider from '../../../lib/result/RowSetProvider'; import ResultsProviderStub from '../.stubs/ResultsProviderStub'; -import { - TFetchResultsResp, - TOperationType, - TRowSet, - TSparkArrowResultLink, - TStatusCode, -} from '../../../thrift/TCLIService_types'; +import { TRowSet, TSparkArrowResultLink, TStatusCode } from '../../../thrift/TCLIService_types'; import { ArrowBatch } from '../../../lib/result/utils'; import RetryError, { RetryErrorCode } from '../../../lib/errors/RetryError'; @@ -911,146 +904,68 @@ describe('CloudFetchResultHandler', () => { describe('recovery', () => { const body = Buffer.concat([sampleArrowSchema, sampleArrowBatch]); - - function makeLink(fileLink: string): TSparkArrowResultLink { - return { - fileLink, - expiryTime: new Int64(defaultLinkExpiryTime), - rowCount: new Int64(1), - startRowOffset: new Int64(7), - bytesNum: new Int64(0), - }; - } - - // Like `createHandler`, with a source that can refresh links - function createRefreshingHandler( - link: TSparkArrowResultLink, - refreshedLinks: () => Array | Promise>, - configOverrides: Partial = {}, - ) { - const context = new BaseClientContextStub({ cloudFetchConcurrentDownloads: 1, ...configOverrides }); - sinon - .stub(context.connectionProvider, 'getRetryPolicy') - .callsFake(async (options) => new HttpRetryPolicy(context, options)); - const source = new ResultsProviderStub( - [{ startRowOffset: new Int64(0), rows: [], resultLinks: [link] }], - undefined, - ); - const refreshLinks = sinon.stub().callsFake(async () => refreshedLinks()); - const handler = new CloudFetchResultHandler(context, Object.assign(source, { refreshLinks }), { - lz4Compressed: false, - status: { statusCode: TStatusCode.SUCCESS_STATUS }, - }); - return { handler, refreshLinks }; - } + const fastRetries = { retryDelayMin: 10, retryDelayMax: 20 }; [401, 403, 404].forEach((status) => { - it(`should refresh the link and download the fresh URL after HTTP ${status}`, async () => { + it(`should retry a one-shot HTTP ${status} with the same URL`, async () => { const hits: Array = []; const url = await startServer((req, res) => { hits.push(req.url ?? ''); - if (req.url?.includes('sig=old')) { + if (hits.length === 1) { res.writeHead(status); res.end(); } else { res.end(body); } }); - const { handler, refreshLinks } = createRefreshingHandler(makeLink(`${url}?sig=old`), () => [ - makeLink(`${url}?sig=new`), - ]); + const handler = createHandler(`${url}?sig=a`, fastRetries); const { value, error } = await fetchWithin(handler, 1500); expect(error).to.be.undefined; expect(value?.batches).to.deep.eq([body]); - expect(hits).to.deep.eq(['/chunk?sig=old', '/chunk?sig=new']); - expect(refreshLinks.callCount).to.be.equal(1); - expect(refreshLinks.firstCall.args[0].toNumber()).to.be.equal(7); - expect(refreshLinks.firstCall.args[1].toNumber()).to.be.equal(1); + expect(hits).to.deep.eq(['/chunk?sig=a', '/chunk?sig=a']); }); }); - it('should fail with the HTTP error when the source cannot refresh links', async () => { + it('should fail with the HTTP status once a persistent 403 exhausts the retries', async () => { + let hits = 0; const url = await startServer((req, res) => { - res.writeHead(403, 'Forbidden'); + hits += 1; + res.writeHead(403); res.end(); }); - const handler = createHandler(url, {}); + const handler = createHandler(url, { ...fastRetries, retryMaxAttempts: 3 }); const { error } = await fetchWithin(handler, 1500); - expect(error.message).to.be.equal('CloudFetch HTTP error 403 Forbidden'); + expect(error).to.be.instanceOf(RetryError); + expect(error.errorCode).to.be.equal(RetryErrorCode.AttemptsExceeded); + expect(error.message).to.be.equal( + 'CloudFetch result download failed: Max retry count exceeded (last error: HTTP 403: Forbidden)', + ); + expect(hits).to.be.equal(3); }); [ - { - name: 'only a link for other rows', - links: (url: string) => [{ ...makeLink(`${url}?sig=other`), startRowOffset: new Int64(8) }], - }, - { name: 'no links', links: () => [] }, - ].forEach(({ name, links }) => { - it(`should fail with the HTTP error when the refresh returns ${name}`, async () => { - const hits: Array = []; + [400, 'Bad Request'], + [405, 'Method Not Allowed'], + [412, 'Precondition Failed'], + ].forEach(([status, statusText]) => { + it(`should fail at once with the HTTP error on ${status}`, async () => { + let hits = 0; const url = await startServer((req, res) => { - hits.push(req.url ?? ''); - if (req.url?.includes('sig=old')) { - res.writeHead(403, 'Forbidden'); - res.end(); - } else { - res.end(body); - } + hits += 1; + res.writeHead(status as number); + res.end(); }); - const { handler, refreshLinks } = createRefreshingHandler(makeLink(`${url}?sig=old`), () => links(url)); + const handler = createHandler(url, fastRetries); const { error } = await fetchWithin(handler, 1500); - expect(error?.message).to.be.equal('CloudFetch HTTP error 403 Forbidden'); - expect(refreshLinks.callCount).to.be.equal(1); - // Another chunk's link must never be downloaded in place of the rejected one - expect(hits).to.deep.eq(['/chunk?sig=old']); - }); - }); - - it('should count link refreshes against retryMaxAttempts', async () => { - const url = await startServer((req, res) => { - res.writeHead(403, 'Forbidden'); - res.end(); - }); - const { handler, refreshLinks } = createRefreshingHandler( - makeLink(`${url}?sig=0`), - () => [makeLink(`${url}?sig=${Math.random()}`)], - { retryMaxAttempts: 3 }, - ); - - const { error } = await fetchWithin(handler, 1500); - - expect(error.message).to.be.equal('CloudFetch HTTP error 403 Forbidden'); - expect(refreshLinks.callCount).to.be.equal(2); - }); - - it('should retry other 4xx responses with the same URL', async () => { - const hits: Array = []; - const url = await startServer((req, res) => { - hits.push(req.url ?? ''); - if (hits.length === 1) { - res.writeHead(412, 'Precondition Failed'); - res.end(); - } else { - res.end(body); - } + expect(error.message).to.be.equal(`CloudFetch HTTP error ${status} ${statusText}`); + expect(hits).to.be.equal(1); }); - const { handler, refreshLinks } = createRefreshingHandler(makeLink(`${url}?sig=a`), () => [], { - retryDelayMin: 10, - retryDelayMax: 20, - }); - - const { value, error } = await fetchWithin(handler, 1500); - - expect(error).to.be.undefined; - expect(value?.batches).to.deep.eq([body]); - expect(hits).to.deep.eq(['/chunk?sig=a', '/chunk?sig=a']); - expect(refreshLinks.called).to.be.false; }); it('should time out a stalled attempt after socketTimeout and retry it', async () => { @@ -1062,7 +977,7 @@ describe('CloudFetchResultHandler', () => { } // ...the first request never gets a response }); - const handler = createHandler(url, { socketTimeout: 200, retryDelayMin: 10, retryDelayMax: 20 }); + const handler = createHandler(url, { ...fastRetries, socketTimeout: 200 }); const { value, error, elapsedMs } = await fetchWithin(handler, 1500); @@ -1077,10 +992,9 @@ describe('CloudFetchResultHandler', () => { // never respond }); const handler = createHandler(`${url}?sig=SECRET_SIGNATURE`, { + ...fastRetries, socketTimeout: 100, retryMaxAttempts: 2, - retryDelayMin: 10, - retryDelayMax: 20, }); const { error } = await fetchWithin(handler, 1500); @@ -1120,192 +1034,6 @@ describe('CloudFetchResultHandler', () => { expect(hits).to.be.equal(1); expect(elapsedMs).to.be.gte(300); }); - - it('should share one retriesTimeout deadline across link refreshes', async function () { - this.timeout(5000); - // Every URL is rejected after 120 ms. One shared 300 ms deadline ends the download within - // 3 GETs; a fresh deadline per refreshed URL would refresh 9 times and fail with the 403. - const url = await startServer((req, res) => { - setTimeout(() => { - res.writeHead(403, 'Forbidden'); - res.end(); - }, 120); - }); - let refreshes = 0; - const { handler } = createRefreshingHandler( - makeLink(`${url}?sig=0`), - () => { - refreshes += 1; - return [makeLink(`${url}?sig=${refreshes}`)]; - }, - { retriesTimeout: 300, retryMaxAttempts: 10 }, - ); - - const { error } = await fetchWithin(handler, 4000); - - expect(error).to.be.instanceOf(RetryError); - expect(error.errorCode).to.be.equal(RetryErrorCode.TimeoutExceeded); - expect(refreshes).to.be.lessThan(9); - }); - - it('should time out with the rejected response as the cause when a refresh outlasts retriesTimeout', async () => { - let hits = 0; - const url = await startServer((req, res) => { - hits += 1; - res.writeHead(403, 'Forbidden'); - res.end(); - }); - let refreshed: Promise = Promise.resolve(); - const { handler, refreshLinks } = createRefreshingHandler( - makeLink(`${url}?sig=0`), - () => { - const links = new Promise((resolve) => { - setTimeout(resolve, 400); - }).then(() => [makeLink(`${url}?sig=1`)]); - refreshed = links; - return links; - }, - { retriesTimeout: 300 }, - ); - - const { error } = await fetchWithin(handler, 1500); - - expect(error).to.be.instanceOf(RetryError); - expect(error.errorCode).to.be.equal(RetryErrorCode.TimeoutExceeded); - expect(error.message).to.be.equal( - 'CloudFetch result download failed: Retry timeout exceeded (last error: HTTP 403: Forbidden)', - ); - expect(refreshLinks.callCount).to.be.equal(1); - // The late refresh doesn't resume the download with a doomed GET of the refreshed URL - await refreshed; - await new Promise((resolve) => { - setImmediate(resolve); - }); - expect(hits).to.be.equal(1); - }); - - it('should time out on schedule while a link refresh never returns', async () => { - const url = await startServer((req, res) => { - res.writeHead(403, 'Forbidden'); - res.end(); - }); - const { handler } = createRefreshingHandler(makeLink(`${url}?sig=0`), () => new Promise(() => {}), { - retriesTimeout: 100, - }); - - const { error, elapsedMs } = await fetchWithin(handler, 1000); - - expect(error).to.be.instanceOf(RetryError); - expect(error.errorCode).to.be.equal(RetryErrorCode.TimeoutExceeded); - expect(error.message).to.be.equal( - 'CloudFetch result download failed: Retry timeout exceeded (last error: HTTP 403: Forbidden)', - ); - expect(elapsedMs).to.be.lt(500); - }); - - it('should not leak the rejection of a link refresh that fails after retriesTimeout', async () => { - const url = await startServer((req, res) => { - res.writeHead(403, 'Forbidden'); - res.end(); - }); - let refreshed: Promise = Promise.resolve(); - const { handler } = createRefreshingHandler( - makeLink(`${url}?sig=0`), - () => { - const links = new Promise((_, reject) => { - setTimeout(() => reject(new Error('FetchResults failed')), 200); - }); - refreshed = links.catch(() => undefined); - return links; - }, - { retriesTimeout: 100 }, - ); - const unhandled: Array = []; - const onUnhandled = (reason: unknown) => unhandled.push(reason); - process.on('unhandledRejection', onUnhandled); - try { - const { error } = await fetchWithin(handler, 1000); - expect(error.errorCode).to.be.equal(RetryErrorCode.TimeoutExceeded); - await refreshed; - await new Promise((resolve) => { - setTimeout(resolve, 10); - }); - } finally { - process.removeListener('unhandledRejection', onUnhandled); - } - expect(unhandled).to.deep.equal([]); - }); - - it('should time out on schedule while its link refresh waits for another FetchResults call', async () => { - const url = await startServer((req, res) => { - res.writeHead(403, 'Forbidden'); - res.end(); - }); - const context = new BaseClientContextStub({ cloudFetchConcurrentDownloads: 1, retriesTimeout: 100 }); - sinon - .stub(context.connectionProvider, 'getRetryPolicy') - .callsFake(async (options) => new HttpRetryPolicy(context, options)); - const page = (resultLinks: Array): TFetchResultsResp => ({ - status: { statusCode: TStatusCode.SUCCESS_STATUS }, - hasMoreRows: true, - results: { startRowOffset: new Int64(0), rows: [], resultLinks }, - }); - let releasePage!: () => void; - const pageHeld = new Promise((resolve) => { - releasePage = resolve; - }); - const fetchResults = sinon.stub(context.driver, 'fetchResults'); - fetchResults.onFirstCall().resolves(page([makeLink(`${url}?sig=0`)])); - // The next page's FetchResults hangs, and the link refresh is queued behind it - fetchResults.onSecondCall().callsFake(async () => { - await pageHeld; - return page([]); - }); - fetchResults.resolves(page([makeLink(`${url}?sig=1`)])); - const operationHandle = { - operationId: { guid: Buffer.alloc(16), secret: Buffer.alloc(16) }, - operationType: TOperationType.EXECUTE_STATEMENT, - hasResultSet: true, - }; - const handler = new CloudFetchResultHandler(context, new RowSetProvider(context, operationHandle, [], false), { - lz4Compressed: false, - status: { statusCode: TStatusCode.SUCCESS_STATUS }, - }); - - const first = fetchWithin(handler, 1000); - const second = handler.fetchNext({ limit: 10000 }); - const { error, elapsedMs } = await first; - - expect(error).to.be.instanceOf(RetryError); - expect(error.errorCode).to.be.equal(RetryErrorCode.TimeoutExceeded); - expect(elapsedMs).to.be.lt(500); - - releasePage(); - await second; - }); - }); - - it('should clear the deadline timer when a download completes', async () => { - const body = Buffer.concat([sampleArrowSchema, sampleArrowBatch]); - const url = await startServer((req, res) => { - res.end(body); - }); - const handler = createHandler(url, { retriesTimeout: 4321 }); - const setTimeoutSpy = sinon.spy(global, 'setTimeout'); - const clearTimeoutSpy = sinon.spy(global, 'clearTimeout'); - try { - const { error } = await fetchWithin(handler, 1500); - expect(error).to.be.undefined; - const deadlineTimers = setTimeoutSpy - .getCalls() - .filter((call) => call.args[1] === 4321) - .map((call) => call.returnValue); - expect(deadlineTimers).to.have.length(1); - expect(clearTimeoutSpy.calledWith(deadlineTimers[0])).to.be.true; - } finally { - setTimeoutSpy.restore(); - clearTimeoutSpy.restore(); - } }); it('should not enforce a deadline when retriesTimeout is not positive', async () => { diff --git a/tests/unit/result/RowSetProvider.test.ts b/tests/unit/result/RowSetProvider.test.ts index ab29bff1..f2aebe44 100644 --- a/tests/unit/result/RowSetProvider.test.ts +++ b/tests/unit/result/RowSetProvider.test.ts @@ -11,13 +11,7 @@ import { expect } from 'chai'; import sinon from 'sinon'; import Int64 from 'node-int64'; -import { - TFetchResultsResp, - TOperationHandle, - TOperationType, - THandleIdentifier, - TStatusCode, -} from '../../../thrift/TCLIService_types'; +import { TOperationHandle, TOperationType, THandleIdentifier } from '../../../thrift/TCLIService_types'; import RowSetProvider from '../../../lib/result/RowSetProvider'; import ClientContextStub from '../.stubs/ClientContextStub'; @@ -165,77 +159,4 @@ describe('RowSetProvider', () => { expect(emitCloudFetchChunk.firstCall.args[0].bytes).to.equal(64); }); }); - - describe('link refresh', () => { - function linksResponse(fileLink: string, startRowOffset: number, rowCount: number): TFetchResultsResp { - return { - status: { statusCode: TStatusCode.SUCCESS_STATUS }, - hasMoreRows: true, - results: { - startRowOffset: new Int64(startRowOffset), - rows: [], - resultLinks: [ - { - fileLink, - expiryTime: new Int64(Date.now() + 60_000), - startRowOffset: new Int64(startRowOffset), - rowCount: new Int64(rowCount), - bytesNum: new Int64(0), - }, - ], - }, - }; - } - - it('fetches fresh links from the offset, then pages by offset', async () => { - const context = new ClientContextStub(); - const fetchResults = sinon.spy(context.driver, 'fetchResults'); - const provider = new RowSetProvider(context, makeOperationHandle(), [], false); - - context.driver.fetchResultsResp = linksResponse('https://storage/0?sig=old', 0, 100); - await provider.fetchNext({ limit: 1000 }); - expect(fetchResults.firstCall.args[0].startRowOffset).to.be.undefined; - - context.driver.fetchResultsResp = linksResponse('https://storage/0?sig=new', 0, 100); - const links = await provider.refreshLinks(new Int64(0), new Int64(100)); - expect(links.map((link) => link.fileLink)).to.deep.equal(['https://storage/0?sig=new']); - expect(fetchResults.secondCall.args[0].startRowOffset?.toNumber()).to.equal(0); - expect(fetchResults.secondCall.args[0].maxRows.toNumber()).to.equal(100); - - // The refresh may have moved the server cursor, so the next page is requested by offset - context.driver.fetchResultsResp = linksResponse('https://storage/100?sig=a', 100, 50); - await provider.fetchNext({ limit: 1000 }); - expect(fetchResults.thirdCall.args[0].startRowOffset?.toNumber()).to.equal(100); - }); - - it('serializes a refresh with a page fetch', async () => { - const context = new ClientContextStub(); - const provider = new RowSetProvider(context, makeOperationHandle(), [], false); - context.driver.fetchResultsResp = linksResponse('https://storage/0?sig=old', 0, 100); - await provider.fetchNext({ limit: 1000 }); - - let inFlight = 0; - let maxInFlight = 0; - sinon.stub(context.driver, 'fetchResults').callsFake(async () => { - inFlight += 1; - maxInFlight = Math.max(maxInFlight, inFlight); - await new Promise((resolve) => { - setTimeout(resolve, 10); - }); - inFlight -= 1; - return linksResponse('https://storage/0?sig=new', 0, 100); - }); - await Promise.all([provider.refreshLinks(new Int64(0), new Int64(100)), provider.fetchNext({ limit: 1000 })]); - expect(maxInFlight).to.equal(1); - }); - - it('returns no links for an operation closed by direct results', async () => { - const context = new ClientContextStub(); - const fetchResults = sinon.spy(context.driver, 'fetchResults'); - const provider = new RowSetProvider(context, makeOperationHandle(), [], true); - - expect(await provider.refreshLinks(new Int64(0), new Int64(100))).to.deep.equal([]); - expect(fetchResults.called).to.be.false; - }); - }); });