Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
## Unreleased

- Reduce result chunking and `fetchAll()` overhead for large query results.
- Thrift CloudFetch downloads now retry a 401, 403 or 404 from cloud storage with the same URL, like 429 and 5xx, within the `retryMaxAttempts`/`retriesTimeout` budget; other 4xx responses still fail with `CloudFetch HTTP error <status>`. `socketTimeout` is now also the inactivity timeout of each download attempt, so a stalled attempt is retried when `socketTimeout` is shorter than `retriesTimeout`. Previously a one-shot 401/403/404 failed the query and a stalled attempt waited out `retriesTimeout`. (#610)
- Thrift CloudFetch downloads are now bounded end to end by `retriesTimeout` (default 15 minutes), including every attempt and retry backoff. Previously a stalled download, where the response headers or body never arrived, blocked the fetch forever. A download still running when the deadline expires is aborted and fails with `RetryError` (`TIMEOUT_EXCEEDED`, message `CloudFetch result download failed: Retry timeout exceeded`). (#520)
- Thrift connections made through `proxy` now apply `checkServerCertificate`, `customCaCert` and `clientCert`/`clientKey`. Previously the proxied path ignored them and verified against Node's process-wide trust store, so a certificate outside the configured trust set could be accepted. **Compatibility:** proxied connections now verify unless `checkServerCertificate: false` is set; as on direct connections, `NODE_TLS_REJECT_UNAUTHORIZED=0` no longer turns verification off. For an `https` proxy these options also apply to the proxy's own certificate. (#485)
- Honor the client `useCloudFetch: false` default on the kernel backend so complete results can be fetched without cloud-storage downloads. (#574)
Expand Down
4 changes: 4 additions & 0 deletions lib/DBSQLClient.ts
Original file line number Diff line number Diff line change
Expand Up @@ -742,6 +742,10 @@ export default class DBSQLClient extends EventEmitter implements IDBSQLClient, I
if (options.retryDelayMax !== undefined) {
this.config.retryDelayMax = options.retryDelayMax;
}
// Also bounds each CloudFetch download attempt on the Thrift backend
if (options.socketTimeout !== undefined) {
Comment thread
peco-review-bot[bot] marked this conversation as resolved.
Resolved
this.config.socketTimeout = options.socketTimeout;
}

// Override telemetry config if provided in options. Per-key narrowed copy
// preserves the structural type system: `ConnectionOptions` and
Expand Down
3 changes: 2 additions & 1 deletion lib/connection/connections/HttpRetryPolicy.ts
Original file line number Diff line number Diff line change
Expand Up @@ -191,10 +191,11 @@ export default class HttpRetryPolicy implements IRetryPolicy<HttpTransactionDeta
return false;
}

protected isRetryable({ response }: HttpTransactionDetails): boolean {
protected isRetryable({ response, retryable }: HttpTransactionDetails): boolean {
const statusCode = response.status;

const result =
retryable === true ||
// Retry on all codes below 100
statusCode < 100 ||
// ...and on `429 Too Many Requests`
Expand Down
2 changes: 2 additions & 0 deletions lib/connection/contracts/IConnectionProvider.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,8 @@ import IRetryPolicy from './IRetryPolicy';
export interface HttpTransactionDetails {
request: Request;
response: Response;
// Set by the caller to retry this response whatever its status
retryable?: boolean;
}

export default interface IConnectionProvider {
Expand Down
43 changes: 37 additions & 6 deletions lib/result/CloudFetchResultHandler.ts
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,10 @@ type SettledDownload = { ok: true; batch: ArrowBatch } | { ok: false; error: unk
// Largest delay `setTimeout` accepts; longer values fire immediately, so they are clamped.
const MAX_TIMER_DELAY_MS = 2 ** 31 - 1;

// A one-shot 401/403/404 from cloud storage is retried with the same URL, like 429 and 5xx.
// Other 4xx responses fail the download.
const SAME_URL_RETRY_STATUSES: ReadonlySet<number> = new Set([401, 403, 404]);

export default class CloudFetchResultHandler implements IResultsProvider<ArrowBatch> {
private readonly context: IClientContext;

Expand Down Expand Up @@ -157,7 +161,10 @@ export default class CloudFetchResultHandler implements IResultsProvider<ArrowBa
}
const agent = await connectionProvider.getAgent(requestUrl);
const retryPolicy = await connectionProvider.getRetryPolicy();
const { retriesTimeout } = this.context.getConfig();
const { retriesTimeout, socketTimeout } = this.context.getConfig();
// `socketTimeout` is an inactivity timeout for each attempt: an attempt that receives
// nothing for that long is aborted and retried like a network timeout.
const idleTimeoutMs = socketTimeout > 0 ? Math.min(socketTimeout, MAX_TIMER_DELAY_MS) : undefined;

const requestConfig: RequestInit = { agent, ...init };
// The retry policy checks `retriesTimeout` only between attempts, so a stalled attempt
Expand Down Expand Up @@ -187,24 +194,48 @@ export default class CloudFetchResultHandler implements IResultsProvider<ArrowBa
throw timeoutError;
}
downloaded = undefined;
// Aborted by the deadline or when the attempt goes idle
const attempt = new AbortController();
const abortAttempt = () => attempt.abort();
controller?.signal.addEventListener('abort', abortAttempt, { once: true });
let attemptTimedOut = false;
let idleTimer: NodeJS.Timeout | undefined;
const restartIdleTimer = () => {
if (idleTimeoutMs !== undefined) {
clearTimeout(idleTimer);
idleTimer = setTimeout(() => {
attemptTimedOut = true;
attempt.abort();
}, idleTimeoutMs);
}
};
restartIdleTimer();
const request = new Request(url, {
...requestConfig,
signal: controller?.signal as unknown as RequestInit['signal'],
signal: attempt.signal as unknown as RequestInit['signal'],
});
try {
const response = await fetch(request);
restartIdleTimer();
if (response.ok) {
downloaded = await response.arrayBuffer();
const bodyRead = response.arrayBuffer();
response.body.on('data', restartIdleTimer);
downloaded = await bodyRead;
} else {
lastFailure = { request, response };
}
return { request, response };
return { request, response, retryable: SAME_URL_RETRY_STATUSES.has(response.status) };
} catch (error) {
if (timeoutError) {
throw timeoutError;
}
lastFailure = error;
throw error;
lastFailure = attemptTimedOut
? Object.assign(new Error('CloudFetch download attempt timed out'), { code: 'ETIMEDOUT' })
: error;
throw lastFailure;
} finally {
clearTimeout(idleTimer);
controller?.signal.removeEventListener('abort', abortAttempt);
}
}, controller?.signal);

Expand Down
13 changes: 13 additions & 0 deletions tests/unit/DBSQLClient.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -879,19 +879,31 @@ describe('DBSQLClient retry-policy ConnectionOptions', () => {
expect(client.getConfig().retriesTimeout).to.equal(15 * 60 * 1000);
expect(client.getConfig().retryDelayMin).to.equal(1000);
expect(client.getConfig().retryDelayMax).to.equal(60 * 1000);
expect(client.getConfig().socketTimeout).to.equal(15 * 60 * 1000);

await client.connect({
...connectOptions,
retryMaxAttempts: 2,
retriesTimeout: 5000,
retryDelayMin: 100,
retryDelayMax: 500,
socketTimeout: 30000,
});

expect(client.getConfig().retryMaxAttempts).to.equal(2);
expect(client.getConfig().retriesTimeout).to.equal(5000);
expect(client.getConfig().retryDelayMin).to.equal(100);
expect(client.getConfig().retryDelayMax).to.equal(500);
// Read by the Thrift CloudFetch download idle timeout
expect(client.getConfig().socketTimeout).to.equal(30000);
});

it('ingests socketTimeout: 0 from connect() into ClientConfig', async () => {
const client = new DBSQLClient();

await client.connect({ ...connectOptions, socketTimeout: 0 });

expect(client.getConfig().socketTimeout).to.equal(0);
});

it('keeps defaults when retry-policy options are omitted', async () => {
Expand All @@ -903,6 +915,7 @@ describe('DBSQLClient retry-policy ConnectionOptions', () => {
expect(client.getConfig().retriesTimeout).to.equal(15 * 60 * 1000);
expect(client.getConfig().retryDelayMin).to.equal(1000);
expect(client.getConfig().retryDelayMax).to.equal(60 * 1000);
expect(client.getConfig().socketTimeout).to.equal(15 * 60 * 1000);
});
});

Expand Down
23 changes: 23 additions & 0 deletions tests/unit/connection/connections/HttpRetryPolicy.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -103,6 +103,29 @@ describe('HttpRetryPolicy', () => {
expect(result.shouldRetry).to.be.false;
});

it('should retry a response the caller marks retryable, within the attempts budget', async () => {
const context = new ClientContextStub({ retryMaxAttempts: 3 });
const policy = new HttpRetryPolicy(context);
const forbidden = () => ({
request: new Request('http://localhost'),
response: new Response(undefined, { status: 403 }),
});

expect((await policy.shouldRetry(forbidden())).shouldRetry).to.be.false;
expect((await policy.shouldRetry({ ...forbidden(), retryable: true })).shouldRetry).to.be.true;
expect((await policy.shouldRetry({ ...forbidden(), retryable: true })).shouldRetry).to.be.true;
try {
await policy.shouldRetry({ ...forbidden(), retryable: true });
expect.fail('It should throw an error');
} catch (error) {
if (error instanceof AssertionError) {
throw error;
}
expect(error).to.be.instanceOf(RetryError);
expect((error as RetryError).errorCode).to.equal(RetryErrorCode.AttemptsExceeded);
}
});

it('should use `Retry-After` header as a base for backoff', async () => {
const context = new ClientContextStub({ retryDelayMin: 3, retryDelayMax: 100, retryMaxAttempts: 10 });
const policy = new HttpRetryPolicy(context);
Expand Down
134 changes: 134 additions & 0 deletions tests/unit/result/CloudFetchResultHandler.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -902,6 +902,140 @@ describe('CloudFetchResultHandler', () => {
}
});

describe('recovery', () => {
const body = Buffer.concat([sampleArrowSchema, sampleArrowBatch]);
const fastRetries = { retryDelayMin: 10, retryDelayMax: 20 };

[401, 403, 404].forEach((status) => {
it(`should retry a one-shot HTTP ${status} with the same URL`, async () => {
const hits: Array<string> = [];
const url = await startServer((req, res) => {
hits.push(req.url ?? '');
if (hits.length === 1) {
res.writeHead(status);
res.end();
} else {
res.end(body);
}
});
const handler = createHandler(`${url}?sig=a`, fastRetries);

const { value, error } = await fetchWithin(handler, 1500);

expect(error).to.be.undefined;
expect(value?.batches).to.deep.eq([body]);
expect(hits).to.deep.eq(['/chunk?sig=a', '/chunk?sig=a']);
});
});

it('should fail with the HTTP status once a persistent 403 exhausts the retries', async () => {
let hits = 0;
const url = await startServer((req, res) => {
hits += 1;
res.writeHead(403);
res.end();
});
const handler = createHandler(url, { ...fastRetries, retryMaxAttempts: 3 });

const { error } = await fetchWithin(handler, 1500);

expect(error).to.be.instanceOf(RetryError);
expect(error.errorCode).to.be.equal(RetryErrorCode.AttemptsExceeded);
expect(error.message).to.be.equal(
'CloudFetch result download failed: Max retry count exceeded (last error: HTTP 403: Forbidden)',
);
expect(hits).to.be.equal(3);
});

[
[400, 'Bad Request'],
[405, 'Method Not Allowed'],
[412, 'Precondition Failed'],
].forEach(([status, statusText]) => {
it(`should fail at once with the HTTP error on ${status}`, async () => {
let hits = 0;
const url = await startServer((req, res) => {
hits += 1;
res.writeHead(status as number);
res.end();
});
const handler = createHandler(url, fastRetries);

const { error } = await fetchWithin(handler, 1500);

expect(error.message).to.be.equal(`CloudFetch HTTP error ${status} ${statusText}`);
expect(hits).to.be.equal(1);
});
});

it('should time out a stalled attempt after socketTimeout and retry it', async () => {
let hits = 0;
const url = await startServer((req, res) => {
hits += 1;
if (hits > 1) {
res.end(body);
}
// ...the first request never gets a response
});
const handler = createHandler(url, { ...fastRetries, socketTimeout: 200 });

const { value, error, elapsedMs } = await fetchWithin(handler, 1500);

expect(error).to.be.undefined;
expect(value?.batches).to.deep.eq([body]);
expect(hits).to.be.equal(2);
expect(elapsedMs).to.be.gte(150);
});

it('should report a timed-out last attempt without the presigned URL', async () => {
const url = await startServer(() => {
// never respond
});
const handler = createHandler(`${url}?sig=SECRET_SIGNATURE`, {
...fastRetries,
socketTimeout: 100,
retryMaxAttempts: 2,
});

const { error } = await fetchWithin(handler, 1500);

expect(error).to.be.instanceOf(RetryError);
expect(error.errorCode).to.be.equal(RetryErrorCode.AttemptsExceeded);
expect(error.message).to.be.equal(
'CloudFetch result download failed: Max retry count exceeded (last error: network error ETIMEDOUT)',
);
expect(error.message).to.not.contain('SECRET_SIGNATURE');
});

it('should not time out an attempt that keeps receiving data', async function () {
this.timeout(5000);
let hits = 0;
const url = await startServer((req, res) => {
hits += 1;
res.writeHead(200, { 'Content-Length': `${body.length}` });
// Slower than socketTimeout in total, but never idle for that long
let sent = 0;
const interval = setInterval(() => {
const next = Math.min(sent + 20, body.length);
res.write(body.subarray(sent, next));
sent = next;
if (sent === body.length) {
clearInterval(interval);
res.end();
}
}, 50);
});
const handler = createHandler(url, { socketTimeout: 300 });

const { value, error, elapsedMs } = await fetchWithin(handler, 4000);

expect(error).to.be.undefined;
expect(value?.batches).to.deep.eq([body]);
expect(hits).to.be.equal(1);
expect(elapsedMs).to.be.gte(300);
});
});

it('should not enforce a deadline when retriesTimeout is not positive', async () => {
const body = Buffer.concat([sampleArrowSchema, sampleArrowBatch]);
const url = await startServer((req, res) => {
Expand Down
Loading