diff --git a/CHANGELOG.md b/CHANGELOG.md index cd6726b5..c76492f1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ## Unreleased +- Thrift requests now retry the same HTTP statuses and network errors as the kernel backend, within the existing backoff and `retryMaxAttempts` / `retriesTimeout` budget. `OpenSession` is now retried, idempotent calls and CloudFetch downloads retry every status except 400, 401, 403–405 and 409–416, and idempotent calls also retry a request timeout when `retriesTimeout` leaves time. `ExecuteStatement` is retried only when the server didn't run it: on HTTP 429, HTTP 503 with `Retry-After`, or a failure to connect. Certificate verification failures, and `ENOTFOUND` except on CloudFetch downloads, are no longer retried and fail at once with node-fetch's `FetchError`. A retry waits as long as `Retry-After` asks, within `retryDelayMin` and `retryDelayMax`, and the feature-flag fetch that `connect()` waits for gives up after 10 seconds and uses the defaults. Once retries run out, CloudFetch downloads fail with `RetryError`; `OpenSession`, `ExecuteStatement`, request timeouts and newly retried statuses fail with the same error as before (such as `THTTPException` or the network error), and other calls with `HiveDriverError`. (#458, #605) - The Thrift token federation exchange (`enableTokenFederation`) now goes through the connection's agent, so `proxy`, `checkServerCertificate`, `customCaCert` and `clientCert`/`clientKey` apply to it. Previously it connected directly with Node's default agent, so behind a required proxy the exchange failed and the driver fell back to the original token. (#611) - Kernel backend: a failed status poll while waiting for a statement (a `runAsync: true` statement, or a default execution that outlives the server's inline wait) and a failed result fetch in `hasMoreRows()` now throw the mapped driver error, such as `HiveDriverError: HTTP 500: Internal Server Error` once retries on HTTP 5xx run out, instead of a raw `Error` whose message is the internal `__databricks_error__:{...}` envelope. As in `fetchChunk()`, a failed fetch in `hasMoreRows()` also closes the statement. (#608) - Reduce result chunking and `fetchAll()` overhead for large query results. @@ -10,7 +11,6 @@ - Honor the client `useCloudFetch: false` default on the kernel backend so complete results can be fetched without cloud-storage downloads. (#574) - Thrift HTTP errors (`THTTPException`) now append server detail from the error headers (at most 500 characters). Kernel-backend errors now carry a non-enumerable `cause` summarizing the kernel error code and message (also at most 500 characters). Reyden fallback preserves KP001 after any existing kernel cause. (#467) - Add `Symbol.asyncDispose` to clients, sessions, and operations, delegating to `close()`. Use `asyncDisposable()` with interface-typed results in TypeScript; existing interfaces and factory return types remain unchanged. (#331) -- Thrift `ExecuteStatement` requests are now retried when the connection fails before the request is sent (`ECONNREFUSED`, `ENOTFOUND`, `EAI_AGAIN`), within the existing `retryMaxAttempts` / `retriesTimeout` budget; once retries run out, the original connection error is thrown. Failures after the request may have reached the server are still not retried, so a statement can't run twice. (#458) - Add the `https` connection option: set `https: false` to connect to `host` over plain HTTP (port 80 by default), for example through a trusted local reverse proxy. HTTPS remains the default, and plain HTTP should only be used on a trusted local or private link. (#503) - Add the `rowMode` option to `executeStatement()`: `rowMode: 'array'` returns each row as an array of values in column order, so columns with the same name (e.g. `SELECT carat AS a, color AS a`) are no longer lost; the default `'object'` rows are unchanged, and the driver now logs a warning when a result has duplicate column names in object mode. (#118) - CloudFetch retry-exhaustion errors now identify the download and the last failure: the `RetryError` message reads `CloudFetch result download failed: (last error: )`. `` is `HTTP : ` (or `HTTP ` when the status text is empty) for HTTP failures, and `network error `, `network error ()` (e.g. `aborted`, when there is no `code`) or a bare `network error` for network failures (the raw network-error message, which can contain the presigned URL, is not copied). If no cause can be determined, the ` (last error: …)` suffix is omitted. The `RetryError` instance, `errorCode` and `payload` (including the node-fetch `Request` / `Response`) are unchanged. (#471) diff --git a/lib/FeatureFlagCache.ts b/lib/FeatureFlagCache.ts index 8e7f323c..6fe90247 100644 --- a/lib/FeatureFlagCache.ts +++ b/lib/FeatureFlagCache.ts @@ -53,6 +53,9 @@ export default class FeatureFlagCache { private readonly MAX_CACHE_DURATION_S = 3600; + // The flags fetch's total time, retries included: the 10 s a single attempt could already take + private readonly FETCH_DEADLINE_MS = 10 * 1000; + constructor(private context: IClientContext, private authProvider?: IAuthentication) { this.userAgent = buildUserAgentString(this.context.getConfig().userAgentEntry); } @@ -236,11 +239,13 @@ export default class FeatureFlagCache { const connectionProvider = await this.context.getConnectionProvider(); const agent = await connectionProvider.getAgent(url); const retryPolicy = await connectionProvider.getRetryPolicy(); - const requestConfig: RequestInit = { agent, ...init }; + // connect() waits for the flags, so the attempts, the waits between them and reading the body share one deadline + const signal = AbortSignal.timeout(this.FETCH_DEADLINE_MS); + const requestConfig: RequestInit = { agent, ...init, signal: signal as unknown as RequestInit['signal'] }; const result = await retryPolicy.invokeWithRetry(() => { const request = new Request(url, requestConfig); return fetch(request).then((response) => ({ request, response })); - }); + }, signal); return result.response; } diff --git a/lib/connection/connections/HttpConnection.ts b/lib/connection/connections/HttpConnection.ts index 6472265a..5068e076 100644 --- a/lib/connection/connections/HttpConnection.ts +++ b/lib/connection/connections/HttpConnection.ts @@ -11,9 +11,9 @@ import { LogLevel } from '../../contracts/IDBSQLLogger'; import HiveDriverError from '../../errors/HiveDriverError'; import ThriftHttpConnection from './ThriftHttpConnection'; -import IRetryPolicy from '../contracts/IRetryPolicy'; +import IRetryPolicy, { RetryRequestClass } from '../contracts/IRetryPolicy'; import HttpRetryPolicy from './HttpRetryPolicy'; -import UnsentRequestRetryPolicy from './UnsentRequestRetryPolicy'; +import NonIdempotentRequestRetryPolicy from './NonIdempotentRequestRetryPolicy'; // Set by the driver, auth provider or Thrift transport. node-fetch merges // same-named headers case-insensitively into one value, so a caller copy @@ -214,7 +214,7 @@ export default class HttpConnection implements IConnectionProvider { transport: thrift.TBufferedTransport, protocol: thrift.TBinaryProtocol, getRetryPolicy: () => this.getRetryPolicy(), - getUnsentRequestRetryPolicy: async () => new UnsentRequestRetryPolicy(this.context), + getNonIdempotentRequestRetryPolicy: async () => new NonIdempotentRequestRetryPolicy(this.context), }, { agent, @@ -227,7 +227,7 @@ export default class HttpConnection implements IConnectionProvider { return this.connection; } - public async getRetryPolicy(): Promise> { - return new HttpRetryPolicy(this.context); + public async getRetryPolicy(requestClass?: RetryRequestClass): Promise> { + return new HttpRetryPolicy(this.context, requestClass); } } diff --git a/lib/connection/connections/HttpRetryPolicy.ts b/lib/connection/connections/HttpRetryPolicy.ts index 3e6a2e53..d1b35925 100644 --- a/lib/connection/connections/HttpRetryPolicy.ts +++ b/lib/connection/connections/HttpRetryPolicy.ts @@ -1,4 +1,4 @@ -import IRetryPolicy, { ShouldRetryResult, RetryableOperation } from '../contracts/IRetryPolicy'; +import IRetryPolicy, { ShouldRetryResult, RetryableOperation, RetryRequestClass } from '../contracts/IRetryPolicy'; import { HttpTransactionDetails } from '../contracts/IConnectionProvider'; import IClientContext from '../../contracts/IClientContext'; import RetryError, { RetryErrorCode } from '../../errors/RetryError'; @@ -36,10 +36,81 @@ const RETRYABLE_NETWORK_ERROR_CODES = new Set([ 'EHOSTUNREACH', 'ENETUNREACH', 'EPIPE', - 'ENOTFOUND', 'EAI_AGAIN', ]); +// The client's certificate verification (OpenSSL's X509 codes as Node names them, and its hostname check). +// Node reports every X509 code it doesn't name as `UNSPECIFIED` (ncrypto's X509Pointer::ErrorCode), e.g. a CA +// without keyCertSign or a SHA-1 signature; the kernel doesn't retry any certificate verification failure either. +const CERTIFICATE_ERROR_CODES = [ + 'CERT_CHAIN_TOO_LONG', + 'CERT_HAS_EXPIRED', + 'CERT_NOT_YET_VALID', + 'CERT_REJECTED', + 'CERT_REVOKED', + 'CERT_SIGNATURE_FAILURE', + 'CERT_UNTRUSTED', + 'DEPTH_ZERO_SELF_SIGNED_CERT', + 'ERROR_IN_CERT_NOT_AFTER_FIELD', + 'ERROR_IN_CERT_NOT_BEFORE_FIELD', + 'ERR_TLS_CERT_ALTNAME_FORMAT', + 'ERR_TLS_CERT_ALTNAME_INVALID', + 'HOSTNAME_MISMATCH', + 'INVALID_CA', + 'INVALID_PURPOSE', + 'PATH_LENGTH_EXCEEDED', + 'SELF_SIGNED_CERT_IN_CHAIN', + 'UNABLE_TO_DECODE_ISSUER_PUBLIC_KEY', + 'UNABLE_TO_DECRYPT_CERT_SIGNATURE', + 'UNABLE_TO_GET_ISSUER_CERT', + 'UNABLE_TO_GET_ISSUER_CERT_LOCALLY', + 'UNABLE_TO_VERIFY_LEAF_SIGNATURE', + 'UNSPECIFIED', +]; + +// Which statuses and network errors are retried matches the kernel (databricks-sql-kernel src/client/retry.rs +// and the CloudFetch chunk retry); backoff and budget stay Node's. +// Network errors retrying can't fix, thrown at once: certificate failures, and for RPCs a host name that doesn't +// resolve (`ENOTFOUND`; a temporary DNS failure is `EAI_AGAIN`). The kernel retries CloudFetch downloads on it. +const PERMANENT_NETWORK_ERROR_CODES: Record> = { + rpc: new Set([...CERTIFICATE_ERROR_CODES, 'ENOTFOUND']), + cloudFetch: new Set(CERTIFICATE_ERROR_CODES), +}; + +// Idempotent requests retry every status except these (the kernel's IDEMPOTENT_NON_RETRYABLE); non-idempotent +// ones only a 429, or a 503 with `Retry-After`. +const IDEMPOTENT_NON_RETRYABLE_STATUSES = new Set([400, 401, 403, 404, 405, 409, 410, 411, 412, 413, 414, 415, 416]); + +export function isRetryableStatus({ response }: HttpTransactionDetails, idempotent: boolean): boolean { + const { status } = response; + if (status >= 200 && status < 400) { + return false; + } + if (idempotent) { + return !IDEMPOTENT_NON_RETRYABLE_STATUSES.has(status); + } + return status === 429 || (status === 503 && response.headers.has('Retry-After')); +} + +// A failure to connect (the kernel's `is_connect`), so the request never left the client and even a +// non-idempotent one can be retried. Node reports these as `connect
`, as a socket closed +// before the TLS handshake finished, or, for a host with several addresses, as an AggregateError of the +// attempts, which has a code but no message (node-fetch's reason is then empty). +export function isConnectError(error: unknown): boolean { + const { code, message } = (error ?? {}) as { code?: unknown; message?: unknown }; + if (typeof code !== 'string' || PERMANENT_NETWORK_ERROR_CODES.rpc.has(code)) { + return false; + } + return ( + code === 'ECONNREFUSED' || + code === 'EAI_AGAIN' || + (typeof message === 'string' && + (message.includes(`connect ${code}`) || + message.includes('before secure TLS connection was established') || + message.endsWith('reason: '))) + ); +} + // Fallback message patterns for errors that don't carry an errno. node-fetch surfaces // "socket hang up" as a generic FetchError, and "Premature close" when the response // body stream closes before all data is received — both occur regularly when a @@ -53,10 +124,13 @@ export default class HttpRetryPolicy implements IRetryPolicy { @@ -135,13 +209,17 @@ export default class HttpRetryPolicy implements IRetryPolicy clientConfig.retriesTimeout) { + throw new RetryError(RetryErrorCode.TimeoutExceeded, details); + } + return { shouldRetry: true, retryAfter }; + } + const retryAfter = this.getBackoffDelay(this.attempt, clientConfig.retryDelayMin, clientConfig.retryDelayMax); return { shouldRetry: true, retryAfter }; } @@ -172,11 +250,14 @@ export default class HttpRetryPolicy implements IRetryPolicy= 500 && statusCode !== 501); - - return result; + protected isRetryable(details: HttpTransactionDetails): boolean { + return isRetryableStatus(details, true); } protected getRetryAfterHeader({ response }: HttpTransactionDetails, delayMin: number): number | undefined { @@ -210,12 +281,12 @@ export default class HttpRetryPolicy implements IRetryPolicy 0) { - return Math.max(delayMin, value); + return Math.max(delayMin, value * 1000); } } return undefined; diff --git a/lib/connection/connections/UnsentRequestRetryPolicy.ts b/lib/connection/connections/NonIdempotentRequestRetryPolicy.ts similarity index 57% rename from lib/connection/connections/UnsentRequestRetryPolicy.ts rename to lib/connection/connections/NonIdempotentRequestRetryPolicy.ts index 14357c52..2171ffe9 100644 --- a/lib/connection/connections/UnsentRequestRetryPolicy.ts +++ b/lib/connection/connections/NonIdempotentRequestRetryPolicy.ts @@ -1,14 +1,12 @@ -import HttpRetryPolicy from './HttpRetryPolicy'; +import HttpRetryPolicy, { isConnectError, isRetryableStatus } from './HttpRetryPolicy'; import { HttpTransactionDetails } from '../contracts/IConnectionProvider'; import { RetryableOperation } from '../contracts/IRetryPolicy'; import RetryError from '../../errors/RetryError'; -// Raised before a connection exists, so no request bytes were written. Failures on a connected -// socket (ECONNRESET, EPIPE, timeouts, ...) are excluded: the server may already have the request. -const UNSENT_REQUEST_ERROR_CODES = new Set(['ECONNREFUSED', 'ENOTFOUND', 'EAI_AGAIN']); - -// For operations that are not safe to repeat: retries only requests that never left the client. -export default class UnsentRequestRetryPolicy extends HttpRetryPolicy { +// For operations that are not safe to repeat: retries only requests the server didn't run, as the kernel's +// non-idempotent strategy does: a failure to connect, a 429, or a 503 with `Retry-After`. Failures on a +// connected socket (ECONNRESET, EPIPE, timeouts, ...) are not retried: the server may already have the request. +export default class NonIdempotentRequestRetryPolicy extends HttpRetryPolicy { public async invokeWithRetry(operation: RetryableOperation): Promise { try { return await super.invokeWithRetry(operation); @@ -21,12 +19,11 @@ export default class UnsentRequestRetryPolicy extends HttpRetryPolicy { } } - protected isRetryable(): boolean { - return false; + protected isRetryable(details: HttpTransactionDetails): boolean { + return isRetryableStatus(details, false); } protected isRetryableNetworkError(error: unknown): boolean { - const code = (error as { code?: unknown } | undefined)?.code; - return typeof code === 'string' && UNSENT_REQUEST_ERROR_CODES.has(code); + return isConnectError(error); } } diff --git a/lib/connection/connections/ThriftHttpConnection.ts b/lib/connection/connections/ThriftHttpConnection.ts index fa686596..52bbb4f1 100644 --- a/lib/connection/connections/ThriftHttpConnection.ts +++ b/lib/connection/connections/ThriftHttpConnection.ts @@ -12,6 +12,7 @@ import InputBufferUnderrunError from 'thrift/lib/nodejs/lib/thrift/input_buffer_ import IRetryPolicy from '../contracts/IRetryPolicy'; import { HttpTransactionDetails } from '../contracts/IConnectionProvider'; import NullRetryPolicy from './NullRetryPolicy'; +import RetryError from '../../errors/RetryError'; export class THTTPException extends Thrift.TApplicationException { public readonly statusCode: unknown; @@ -60,7 +61,7 @@ interface ThriftHttpConnectionOptions { transport?: TTransportType; protocol?: TProtocolConstructor; getRetryPolicy(): Promise>; - getUnsentRequestRetryPolicy?(): Promise>; + getNonIdempotentRequestRetryPolicy?(): Promise>; } // This type describes a shape of internals of Thrift client object. @@ -76,6 +77,8 @@ type ThriftClient = { }; const retryableThriftMethods = new Set([ + // A repeated OpenSession leaves at most an unused session, which the server expires + 'OpenSession', 'GetOperationStatus', 'CancelOperation', 'CloseOperation', @@ -93,8 +96,30 @@ const retryableThriftMethods = new Set([ 'GetCrossReference', ]); -// Not safe to repeat once the server may have the request, so retried only if it was never sent -const unsentRequestRetryableThriftMethods = new Set(['ExecuteStatement']); +// Not safe to repeat once the server may have run the request, so retried only if it was never sent or +// the server rejected it unrun (see NonIdempotentRequestRetryPolicy) +const nonIdempotentRetryableThriftMethods = new Set(['ExecuteStatement']); + +// Once retries run out, fail with the last attempt's error, as without retries: THTTPException (status and server +// detail) or the network error. Idempotent RPCs other than OpenSession have always retried a status below 100, a 429, +// a 5xx other than 501 and a network error other than a request timeout; those keep the RetryError (a HiveDriverError +// in BaseCommand). +function getRetryExhaustedError(thriftMethod: string | undefined, error: RetryError): unknown { + const { payload } = error; + const response = (payload as { response?: unknown } | undefined)?.response; + const status = response instanceof Response ? response.status : undefined; + const alwaysRetried = + thriftMethod !== undefined && + thriftMethod !== 'OpenSession' && + retryableThriftMethods.has(thriftMethod) && + (status === undefined + ? !(payload instanceof FetchError && payload.type === 'request-timeout') + : status < 100 || status === 429 || (status >= 500 && status !== 501)); + if (alwaysRetried) { + return error; + } + return response instanceof Response ? new THTTPException(response) : payload ?? error; +} export default class ThriftHttpConnection extends EventEmitter { private readonly url: string; @@ -128,10 +153,10 @@ export default class ThriftHttpConnection extends EventEmitter { } if ( thriftMethodName && - unsentRequestRetryableThriftMethods.has(thriftMethodName) && - this.options.getUnsentRequestRetryPolicy + nonIdempotentRetryableThriftMethods.has(thriftMethodName) && + this.options.getNonIdempotentRequestRetryPolicy ) { - return this.options.getUnsentRequestRetryPolicy(); + return this.options.getNonIdempotentRequestRetryPolicy(); } // Don't retry everything that is not explicitly allowed to retry return new NullRetryPolicy(); @@ -164,9 +189,13 @@ export default class ThriftHttpConnection extends EventEmitter { // transient network error — the body Buffer is captured via closure so the // post-retry caller still gets it. let responseBuffer: Buffer | undefined; + let thriftMethodName: string | undefined; this.getThriftMethodName(data) - .then((thriftMethod) => this.getRetryPolicy(thriftMethod)) + .then((thriftMethod) => { + thriftMethodName = thriftMethod; + return this.getRetryPolicy(thriftMethod); + }) .then((retryPolicy) => { const makeRequest = async () => { responseBuffer = undefined; @@ -198,6 +227,9 @@ export default class ThriftHttpConnection extends EventEmitter { )(buffer); }) .catch((error) => { + if (error instanceof RetryError) { + error = getRetryExhaustedError(thriftMethodName, error); + } if (error instanceof FetchError) { if (error.type === 'request-timeout') { error = new Thrift.TApplicationException( diff --git a/lib/connection/contracts/IConnectionProvider.ts b/lib/connection/contracts/IConnectionProvider.ts index a81bbe82..ff24f552 100644 --- a/lib/connection/contracts/IConnectionProvider.ts +++ b/lib/connection/contracts/IConnectionProvider.ts @@ -1,6 +1,6 @@ import http from 'http'; import { HeadersInit, Request, Response } from 'node-fetch'; -import IRetryPolicy from './IRetryPolicy'; +import IRetryPolicy, { RetryRequestClass } from './IRetryPolicy'; export interface HttpTransactionDetails { request: Request; @@ -18,5 +18,5 @@ export default interface IConnectionProvider { setHeaders(headers: HeadersInit): void; - getRetryPolicy(): Promise>; + getRetryPolicy(requestClass?: RetryRequestClass): Promise>; } diff --git a/lib/connection/contracts/IRetryPolicy.ts b/lib/connection/contracts/IRetryPolicy.ts index 900671f0..ff022950 100644 --- a/lib/connection/contracts/IRetryPolicy.ts +++ b/lib/connection/contracts/IRetryPolicy.ts @@ -9,6 +9,9 @@ export type ShouldRetryResult = export type RetryableOperation = () => Promise; +// Request classes whose retried network errors differ (see HttpRetryPolicy) +export type RetryRequestClass = 'rpc' | 'cloudFetch'; + export default interface IRetryPolicy { shouldRetry(details: R): Promise; diff --git a/lib/result/CloudFetchResultHandler.ts b/lib/result/CloudFetchResultHandler.ts index fbfe5c7c..9be28825 100644 --- a/lib/result/CloudFetchResultHandler.ts +++ b/lib/result/CloudFetchResultHandler.ts @@ -1,4 +1,4 @@ -import fetch, { RequestInfo, RequestInit, Request, Response } from 'node-fetch'; +import fetch, { FetchError, RequestInfo, RequestInit, Request, Response } from 'node-fetch'; import { TGetResultSetMetadataResp, TRowSet, TSparkArrowResultLink } from '../../thrift/TCLIService_types'; import HiveDriverError from '../errors/HiveDriverError'; import RetryError, { RetryErrorCode } from '../errors/RetryError'; @@ -156,7 +156,7 @@ export default class CloudFetchResultHandler implements IResultsProvider> { + // eslint-disable-next-line @typescript-eslint/no-unused-vars + public async getRetryPolicy(requestClass?: RetryRequestClass): Promise> { return new NullRetryPolicy(); } } diff --git a/tests/unit/connection/connections/HttpRetryPolicy.test.ts b/tests/unit/connection/connections/HttpRetryPolicy.test.ts index 35e0c0fc..258f48c8 100644 --- a/tests/unit/connection/connections/HttpRetryPolicy.test.ts +++ b/tests/unit/connection/connections/HttpRetryPolicy.test.ts @@ -34,18 +34,20 @@ describe('HttpRetryPolicy', () => { // Missing `Retry-After` header expect(policy['getRetryAfterHeader'](createStub({}), retryDelayMin)).to.be.undefined; - // Valid `Retry-After`, several header name variants - expect(policy['getRetryAfterHeader'](createStub({ 'Retry-After': '10' }), retryDelayMin)).to.equal(10); - expect(policy['getRetryAfterHeader'](createStub({ 'retry-after': '10' }), retryDelayMin)).to.equal(10); - expect(policy['getRetryAfterHeader'](createStub({ 'RETRY-AFTER': '10' }), retryDelayMin)).to.equal(10); + // Valid `Retry-After` in seconds, several header name variants, returned in milliseconds + expect(policy['getRetryAfterHeader'](createStub({ 'Retry-After': '10' }), retryDelayMin)).to.equal(10_000); + expect(policy['getRetryAfterHeader'](createStub({ 'retry-after': '10' }), retryDelayMin)).to.equal(10_000); + expect(policy['getRetryAfterHeader'](createStub({ 'RETRY-AFTER': '10' }), retryDelayMin)).to.equal(10_000); // Invalid header values (non-numeric, negative) expect(policy['getRetryAfterHeader'](createStub({ 'Retry-After': 'test' }), retryDelayMin)).to.be.undefined; expect(policy['getRetryAfterHeader'](createStub({ 'Retry-After': '-10' }), retryDelayMin)).to.be.undefined; // It should not be smaller than min value, but can be greater than max value - expect(policy['getRetryAfterHeader'](createStub({ 'Retry-After': '1' }), retryDelayMin)).to.equal(retryDelayMin); - expect(policy['getRetryAfterHeader'](createStub({ 'Retry-After': '200' }), retryDelayMin)).to.equal(200); + expect(policy['getRetryAfterHeader'](createStub({ 'Retry-After': '0.001' }), retryDelayMin)).to.equal( + retryDelayMin, + ); + expect(policy['getRetryAfterHeader'](createStub({ 'Retry-After': '200' }), retryDelayMin)).to.equal(200_000); }); it('should check if HTTP transaction is safe to retry', async () => { @@ -58,22 +60,16 @@ describe('HttpRetryPolicy', () => { }; } - // Status codes below 100 can be retried - for (let status = 1; status < 100; status += 1) { - expect(policy['isRetryable'](createStub(status))).to.be.true; + // As the kernel's idempotent strategy: every status except success, redirects and these rejections + const nonRetryable = [400, 401, 403, 404, 405, 409, 410, 411, 412, 413, 414, 415, 416]; + for (let status = 1; status < 1000; status += 1) { + const expectedResult = !(status >= 200 && status < 400) && !nonRetryable.includes(status); + expect(policy['isRetryable'](createStub(status)), `status ${status}`).to.equal(expectedResult); } - // Status codes between 100 (including) and 500 (excluding) should not be retried - // The only exception is 429 (Too many requests) - for (let status = 100; status < 500; status += 1) { - const expectedResult = status === 429 ? true : false; - expect(policy['isRetryable'](createStub(status))).to.equal(expectedResult); - } - - // Status codes above 500 can be retried, except for 501 - for (let status = 500; status < 1000; status += 1) { - const expectedResult = status === 501 ? false : true; - expect(policy['isRetryable'](createStub(status))).to.equal(expectedResult); + // Statuses whose class changed to match the kernel + for (const status of [402, 406, 407, 417, 422, 425, 431, 451, 501]) { + expect(policy['isRetryable'](createStub(status)), `status ${status}`).to.be.true; } }); @@ -103,35 +99,65 @@ describe('HttpRetryPolicy', () => { expect(result.shouldRetry).to.be.false; }); - it('should use `Retry-After` header as a base for backoff', async () => { - const context = new ClientContextStub({ retryDelayMin: 3, retryDelayMax: 100, retryMaxAttempts: 10 }); + it('should wait as long as `Retry-After` asks, within retryDelayMin and retryDelayMax', async () => { + const context = new ClientContextStub({ retryDelayMin: 1000, retryDelayMax: 60_000, retryMaxAttempts: 10 }); const policy = new HttpRetryPolicy(context); function createStub(headers: HeadersInit) { return { request: new Request('http://localhost'), - response: new Response(undefined, { status: 500, headers }), + response: new Response(undefined, { status: 503, headers }), }; } - const result1 = await policy.shouldRetry(createStub({ 'Retry-After': '5' })); - expect(result1.shouldRetry).to.be.true; - if (result1.shouldRetry) { - expect(result1.retryAfter).to.equal(10); - } + // The header is the wait, whatever the attempt (no exponential growth) + expect(await policy.shouldRetry(createStub({ 'Retry-After': '5' }))).to.deep.equal({ + shouldRetry: true, + retryAfter: 5_000, + }); + expect(await policy.shouldRetry(createStub({ 'Retry-After': '30' }))).to.deep.equal({ + shouldRetry: true, + retryAfter: 30_000, + }); + // Clamped up to retryDelayMin and down to retryDelayMax + expect(await policy.shouldRetry(createStub({ 'Retry-After': '0.2' }))).to.deep.equal({ + shouldRetry: true, + retryAfter: 1_000, + }); + expect(await policy.shouldRetry(createStub({ 'Retry-After': '600' }))).to.deep.equal({ + shouldRetry: true, + retryAfter: 60_000, + }); + }); - const result2 = await policy.shouldRetry(createStub({ 'Retry-After': '8' })); - expect(result2.shouldRetry).to.be.true; - if (result2.shouldRetry) { - expect(result2.retryAfter).to.equal(32); + it('should not wait for a `Retry-After` that would end after retriesTimeout', async () => { + const shortBudget = { retryDelayMin: 1000, retryDelayMax: 60_000, retriesTimeout: 5000 }; + + function createStub(retryAfter: string) { + return { + request: new Request('http://localhost', { method: 'POST' }), + response: new Response(undefined, { status: 429, headers: { 'Retry-After': retryAfter } }), + }; } - policy['attempt'] = 4; - const result3 = await policy.shouldRetry(createStub({ 'Retry-After': '10' })); - expect(result3.shouldRetry).to.be.true; - if (result3.shouldRetry) { - expect(result3.retryAfter).to.equal(100); + const operation = sinon.stub().resolves(createStub('60')); + const started = Date.now(); + try { + await new HttpRetryPolicy(new ClientContextStub(shortBudget)).invokeWithRetry(operation); + expect.fail('It should throw an error'); + } catch (error) { + if (error instanceof AssertionError || !(error instanceof Error)) { + throw error; + } + expect(error).to.be.instanceOf(RetryError); + expect((error as RetryError).errorCode).to.equal(RetryErrorCode.TimeoutExceeded); } + expect(operation.callCount).to.equal(1); + expect(Date.now() - started).to.be.lt(1000); + + // A wait that fits is kept + const fitting = new HttpRetryPolicy(new ClientContextStub(shortBudget)); + expect(await fitting.shouldRetry(createStub('3'))).to.deep.equal({ shouldRetry: true, retryAfter: 3000 }); }); it('should use backoff when `Retry-After` header is missing', async () => { @@ -401,6 +427,85 @@ describe('HttpRetryPolicy', () => { } }); + describe('network error classification', () => { + function systemError(code: string): FetchError { + return new FetchError(`request to https://example.com failed, reason: ${code}`, 'system', { + code, + } as unknown as NodeJS.ErrnoException); + } + + function createPolicy(requestClass?: 'rpc' | 'cloudFetch') { + return new HttpRetryPolicy( + new ClientContextStub({ retryDelayMin: 1, retryDelayMax: 2, retryMaxAttempts: 5 }), + requestClass, + ); + } + + async function expectThrownAtOnce(error: Error, requestClass?: 'rpc' | 'cloudFetch') { + const operation = sinon.stub().rejects(error); + try { + await createPolicy(requestClass).invokeWithRetry(operation); + expect.fail('It should throw the error'); + } catch (e) { + if (e instanceof AssertionError) { + throw e; + } + expect(e).to.equal(error); + } + expect(operation.callCount).to.equal(1); + } + + async function expectRetried(error: Error, requestClass?: 'rpc' | 'cloudFetch') { + const operation = sinon.stub(); + operation.onCall(0).rejects(error); + operation.onCall(1).resolves({ + request: new Request('http://localhost'), + response: new Response(undefined, { status: 200 }), + }); + + const result = await createPolicy(requestClass).invokeWithRetry(operation); + + expect(result.response.status).to.equal(200); + expect(operation.callCount).to.equal(2); + } + + // Certificate verification failures, and for RPCs a host name that doesn't resolve + for (const code of [ + 'DEPTH_ZERO_SELF_SIGNED_CERT', + 'UNABLE_TO_VERIFY_LEAF_SIGNATURE', + 'SELF_SIGNED_CERT_IN_CHAIN', + 'CERT_HAS_EXPIRED', + 'ERR_TLS_CERT_ALTNAME_INVALID', + // Node's code for the X509 errors it doesn't name + 'UNSPECIFIED', + 'ENOTFOUND', + ]) { + it(`throws ${code} at once on an RPC, without retrying`, () => expectThrownAtOnce(systemError(code))); + } + + // Transient connect/DNS failures, a TLS alert sent by the server, and unavailable revocation data + for (const code of [ + 'ECONNREFUSED', + 'ECONNRESET', + 'EAI_AGAIN', + 'ERR_SSL_TLSV13_ALERT_CERTIFICATE_REQUIRED', + 'UNABLE_TO_GET_CRL', + ]) { + it(`retries ${code}`, () => expectRetried(systemError(code))); + } + + it('retries a request timeout on an RPC, as the kernel', () => + expectRetried(new FetchError('network timeout at: https://example.com', 'request-timeout'))); + + // As the kernel's CloudFetch chunk retry: everything but certificate failures + it('retries ENOTFOUND on a CloudFetch download', () => expectRetried(systemError('ENOTFOUND'), 'cloudFetch')); + + for (const code of ['CERT_HAS_EXPIRED', 'UNSPECIFIED']) { + it(`throws a certificate failure (${code}) at once on a CloudFetch download`, () => + expectThrownAtOnce(systemError(code), 'cloudFetch')); + } + }); + it('should cancel a pending backoff sleep when the signal is aborted', async () => { const context = new ClientContextStub({ retryDelayMin: 10000, diff --git a/tests/unit/connection/connections/NonIdempotentRequestRetryPolicy.test.ts b/tests/unit/connection/connections/NonIdempotentRequestRetryPolicy.test.ts new file mode 100644 index 00000000..f353a47e --- /dev/null +++ b/tests/unit/connection/connections/NonIdempotentRequestRetryPolicy.test.ts @@ -0,0 +1,181 @@ +import { expect, AssertionError } from 'chai'; +import sinon from 'sinon'; +import { Request, Response, FetchError } from 'node-fetch'; +import NonIdempotentRequestRetryPolicy from '../../../../lib/connection/connections/NonIdempotentRequestRetryPolicy'; +import RetryError, { RetryErrorCode } from '../../../../lib/errors/RetryError'; + +import ClientContextStub from '../../.stubs/ClientContextStub'; + +function systemError(code: string, reason = code): FetchError { + return new FetchError(`request to https://example.com failed, reason: ${reason}`, 'system', { + code, + } as unknown as NodeJS.ErrnoException); +} + +function okResponse() { + return { request: new Request('http://localhost'), response: new Response(undefined, { status: 200 }) }; +} + +function createPolicy(retryMaxAttempts = 5) { + return new NonIdempotentRequestRetryPolicy( + new ClientContextStub({ retryDelayMin: 1, retryDelayMax: 2, retryMaxAttempts }), + ); +} + +async function expectRejection(promise: Promise): Promise { + try { + await promise; + } catch (error) { + if (error instanceof AssertionError) { + throw error; + } + return error; + } + return expect.fail('It should throw an error'); +} + +describe('NonIdempotentRequestRetryPolicy', () => { + // Failures to connect (the kernel's `is_connect`): the request never left the client + const connectErrors: Array<[string, Error]> = [ + ['ECONNREFUSED', systemError('ECONNREFUSED', 'connect ECONNREFUSED 10.0.0.1:443')], + ['EAI_AGAIN', systemError('EAI_AGAIN', 'getaddrinfo EAI_AGAIN example.com')], + ['a connect timeout', systemError('ETIMEDOUT', 'connect ETIMEDOUT 10.0.0.1:443')], + ['an unreachable host', systemError('EHOSTUNREACH', 'connect EHOSTUNREACH 10.0.0.1:443')], + ['an unreachable network', systemError('ENETUNREACH', 'connect ENETUNREACH 10.0.0.1:443')], + [ + 'a socket closed during the TLS handshake', + systemError('ECONNRESET', 'Client network socket disconnected before secure TLS connection was established'), + ], + // A host with several addresses: Node's AggregateError has the code but no message + ['a connect timeout on every address', systemError('ETIMEDOUT', '')], + ['an unreachable host on every address', systemError('EHOSTUNREACH', '')], + ['an unreachable network on every address', systemError('ENETUNREACH', '')], + ]; + + for (const [label, error] of connectErrors) { + it(`retries ${label}, which is raised before the request is sent`, async () => { + const operation = sinon.stub(); + operation.onCall(0).rejects(error); + operation.onCall(1).resolves(okResponse()); + + const result = await createPolicy().invokeWithRetry(operation); + + expect(result.response.status).to.equal(200); + expect(operation.callCount).to.equal(2); + }); + } + + const sentRequestErrors: Array<[string, Error]> = [ + ['ECONNRESET (socket hang up)', systemError('ECONNRESET', 'socket hang up')], + ['EPIPE', systemError('EPIPE')], + ['ETIMEDOUT on a connected socket', systemError('ETIMEDOUT', 'read ETIMEDOUT')], + ['EHOSTUNREACH on a connected socket', systemError('EHOSTUNREACH', 'write EHOSTUNREACH')], + ['a body timeout', new FetchError('Response timeout while trying to fetch', 'body-timeout')], + ['a premature close', new Error('Premature close')], + ['ENOTFOUND, a host name that does not resolve', systemError('ENOTFOUND')], + ['a request timeout', new FetchError('network timeout at: https://example.com', 'request-timeout')], + ]; + + for (const [label, error] of sentRequestErrors) { + it(`does not retry ${label}`, async () => { + const operation = sinon.stub().rejects(error); + + expect(await expectRejection(createPolicy().invokeWithRetry(operation))).to.equal(error); + expect(operation.callCount).to.equal(1); + }); + } + + function statusResponse(status: number, headers: Record = {}) { + return { request: new Request('http://localhost'), response: new Response(undefined, { status, headers }) }; + } + + for (const [status, headers] of [ + [500, {}], + [501, {}], + [502, {}], + [503, {}], + [504, {}], + [408, {}], + [500, { 'Retry-After': '1' }], + ] as const) { + it(`does not retry an HTTP ${status} response ${ + Object.keys(headers).length ? 'with' : 'without' + } Retry-After`, async () => { + const details = statusResponse(status, headers); + const policy = createPolicy(); + const operation = sinon.stub().resolves(details); + + expect(await policy.invokeWithRetry(operation)).to.equal(details); + expect(await policy.shouldRetry(details)).to.deep.equal({ shouldRetry: false }); + expect(operation.callCount).to.equal(1); + }); + } + + for (const [label, rejection] of [ + ['an HTTP 503 with Retry-After', statusResponse(503, { 'Retry-After': '1' })], + ['an HTTP 429', statusResponse(429)], + ['an HTTP 429 with Retry-After', statusResponse(429, { 'Retry-After': '1' })], + ] as const) { + it(`retries ${label}, which the server did not run`, async () => { + const operation = sinon.stub(); + operation.onCall(0).resolves(rejection); + operation.onCall(1).resolves(okResponse()); + + const result = await createPolicy().invokeWithRetry(operation); + + expect(result.response.status).to.equal(200); + expect(operation.callCount).to.equal(2); + }); + } + + it('waits the Retry-After delay before retrying a 503, with the default config', async () => { + const policy = new NonIdempotentRequestRetryPolicy(new ClientContextStub()); + + expect(await policy.shouldRetry(statusResponse(503, { 'Retry-After': '5' }))).to.deep.equal({ + shouldRetry: true, + retryAfter: 5_000, + }); + }); + + it('throws a RetryError once a persistent 503 with Retry-After runs out of attempts', async () => { + const operation = sinon.stub().resolves(statusResponse(503, { 'Retry-After': '1' })); + + const error = await expectRejection(createPolicy(3).invokeWithRetry(operation)); + + expect(error).to.be.instanceOf(RetryError); + expect((error as RetryError).errorCode).to.equal(RetryErrorCode.AttemptsExceeded); + expect(operation.callCount).to.equal(3); + }); + + it('rethrows the original error once max attempts is reached', async () => { + const error = systemError('ECONNREFUSED'); + const operation = sinon.stub().rejects(error); + + expect(await expectRejection(createPolicy(3).invokeWithRetry(operation))).to.equal(error); + expect(operation.callCount).to.equal(3); + }); + + it('rethrows the original error once the retry timeout is exceeded', async () => { + const error = systemError('ECONNREFUSED'); + const policy = new NonIdempotentRequestRetryPolicy( + new ClientContextStub({ retryDelayMin: 1, retryDelayMax: 2, retryMaxAttempts: 100, retriesTimeout: 0 }), + ); + const operation = sinon.stub().rejects(error); + + expect(await expectRejection(policy.invokeWithRetry(operation))).to.equal(error); + expect(operation.callCount).to.equal(1); + }); + + it('passes through a RetryError raised by the operation itself', async () => { + for (const error of [ + new RetryError(RetryErrorCode.AttemptsExceeded), + new RetryError(RetryErrorCode.TimeoutExceeded, new TypeError('unrelated')), + ]) { + const operation = sinon.stub().rejects(error); + + // eslint-disable-next-line no-await-in-loop + expect(await expectRejection(createPolicy().invokeWithRetry(operation))).to.equal(error); + expect(operation.callCount).to.equal(1); + } + }); +}); diff --git a/tests/unit/connection/connections/ThriftHttpConnectionRetry.test.ts b/tests/unit/connection/connections/ThriftHttpConnectionRetry.test.ts index 4492f69b..4fe0b696 100644 --- a/tests/unit/connection/connections/ThriftHttpConnectionRetry.test.ts +++ b/tests/unit/connection/connections/ThriftHttpConnectionRetry.test.ts @@ -1,9 +1,16 @@ +import { execFileSync } from 'child_process'; +import fs from 'fs'; import http from 'http'; +import https from 'https'; import net, { AddressInfo } from 'net'; +import os from 'os'; +import path from 'path'; import { expect } from 'chai'; import { FetchError } from 'node-fetch'; import HttpConnection from '../../../../lib/connection/connections/HttpConnection'; import ThriftHttpConnection, { THTTPException } from '../../../../lib/connection/connections/ThriftHttpConnection'; +import { ClientConfig } from '../../../../lib/contracts/IClientContext'; +import RetryError, { RetryErrorCode } from '../../../../lib/errors/RetryError'; import ClientContextStub from '../../.stubs/ClientContextStub'; // Header-only Thrift binary messages: enough for method-name detection and for the fake `recv_*` below. @@ -20,7 +27,18 @@ function thriftMessage(name: string, type: number, seqId: number): Buffer { const CALL = 1; const REPLY = 2; -type ServerAction = 'reply' | 'close-after-request' | 'status-503'; +type ServerAction = + | 'reply' + | 'close-after-request' + | 'stall' + | 'status-099' + | 'status-408' + | 'status-429' + | 'status-501' + | 'status-503' + | 'status-503-retry-after'; + +const ERROR_DETAIL = 'server error detail'; interface TestServer { port: number; @@ -39,8 +57,16 @@ async function startServer(actions: ServerAction[]): Promise { if (action === 'close-after-request') { // The full request has been received; drop the connection without a response req.socket.destroy(); - } else if (action === 'status-503') { - res.writeHead(503); + } else if (action === 'stall') { + // Never respond; the client's request timeout ends it + } else if (action === 'status-099') { + // ServerResponse rejects a status below 100, so write the status line itself + req.socket.end('HTTP/1.1 099 X\r\nContent-Length: 0\r\n\r\n'); + } else if (action === 'status-503-retry-after') { + res.writeHead(503, { 'Retry-After': '1', 'x-thriftserver-error-message': ERROR_DETAIL }); + res.end(); + } else if (action.startsWith('status-')) { + res.writeHead(Number(action.slice('status-'.length)), { 'x-thriftserver-error-message': ERROR_DETAIL }); res.end(); } else { const body = Buffer.concat(chunks); @@ -95,9 +121,48 @@ class RefusingAgent extends http.Agent { } } -async function createThriftConnection(port: number, agent: http.Agent): Promise { - const context = new ClientContextStub({ retryDelayMin: 1, retryDelayMax: 2, retryMaxAttempts: 3 }); - const httpConnection = new HttpConnection({ host: '127.0.0.1', port, path: '/' }, context); +// Keep-alive agent whose first connection tries two closed addresses, as for a host with several addresses. +// Node fails it with an AggregateError that has a code but no message. The code is set to ETIMEDOUT, as when +// every address times out, which a test can't wait for. +class MultiAddressTimeoutAgent extends http.Agent { + public connectAttempts = 0; + + constructor(private readonly closedPort: number) { + super({ keepAlive: true }); + } + + createConnection(options: net.NetConnectOpts): net.Socket { + this.connectAttempts += 1; + if (this.connectAttempts > 1) { + return net.createConnection(options); + } + const addresses = [ + { address: '127.0.0.1', family: 4 }, + { address: '::1', family: 6 }, + ]; + const connectOptions = { + host: 'multi-address.test', + port: this.closedPort, + autoSelectFamily: true, + lookup: (hostname: string, lookupOptions: unknown, callback: (error: null, results: typeof addresses) => void) => + callback(null, addresses), + }; + const socket = net.createConnection(connectOptions as unknown as net.NetConnectOpts); + socket.prependListener('error', (error: NodeJS.ErrnoException) => { + error.code = 'ETIMEDOUT'; + }); + return socket; + } +} + +async function createThriftConnection( + port: number, + agent: http.Agent, + config: Partial, +): Promise { + const context = new ClientContextStub({ retryDelayMin: 1, retryDelayMax: 2, retryMaxAttempts: 3, ...config }); + // A short socketTimeout, so a stalled response becomes a request timeout quickly + const httpConnection = new HttpConnection({ host: '127.0.0.1', port, path: '/', socketTimeout: 300 }, context); httpConnection['httpAgent'] = agent; return httpConnection.getThriftConnection(); } @@ -135,7 +200,7 @@ describe('ThriftHttpConnection retries', () => { cleanup.splice(0).forEach((fn) => fn()); }); - async function setup(actions: ServerAction[], refusals: number) { + async function setup(actions: ServerAction[], refusals: number, config: Partial = {}) { const closedPort = await getClosedPort(); const server = await startServer(actions); const agent = new RefusingAgent(refusals, closedPort); @@ -143,7 +208,7 @@ describe('ThriftHttpConnection retries', () => { () => server.close(), () => agent.destroy(), ); - const connection = await createThriftConnection(server.port, agent); + const connection = await createThriftConnection(server.port, agent, config); return { server, agent, connection }; } @@ -157,6 +222,22 @@ describe('ThriftHttpConnection retries', () => { expect(server.requests).to.equal(1); }); + it('retries ExecuteStatement when connecting to every address of the host times out', async () => { + const server = await startServer(['reply']); + const agent = new MultiAddressTimeoutAgent(await getClosedPort()); + cleanup.push( + () => server.close(), + () => agent.destroy(), + ); + const connection = await createThriftConnection(server.port, agent, {}); + + const result = await send(connection, 'ExecuteStatement', 1); + + expect(result).to.equal('ExecuteStatement result'); + expect(agent.connectAttempts).to.equal(2); + expect(server.requests).to.equal(1); + }); + it('surfaces the original error once retries for a refused ExecuteStatement are exhausted', async () => { const { server, agent, connection } = await setup([], Infinity); @@ -179,20 +260,90 @@ describe('ThriftHttpConnection retries', () => { expect(server.requests).to.equal(2); }); - it('does not retry ExecuteStatement on an HTTP error status', async () => { - const { server, connection } = await setup(['status-503'], 0); + for (const status of ['status-503', 'status-408'] as const) { + it(`does not retry ExecuteStatement on an HTTP error status (${status})`, async () => { + const { server, connection } = await setup([status], 0); + + const error = await expectRejection(send(connection, 'ExecuteStatement', 1)); + + expect(error).to.be.instanceOf(THTTPException); + expect(error.statusCode).to.equal(Number(status.slice('status-'.length))); + expect(server.requests).to.equal(1); + }); + } + + for (const status of ['status-503-retry-after', 'status-429'] as const) { + it(`retries ExecuteStatement when the server rejects it without running it (${status})`, async () => { + const { server, connection } = await setup([status, 'reply'], 0); + + const result = await send(connection, 'ExecuteStatement', 1); + + expect(result).to.equal('ExecuteStatement result'); + expect(server.requests).to.equal(2); + }); + } + + for (const status of ['status-503', 'status-429', 'status-408'] as const) { + it(`retries OpenSession on a transient HTTP error (${status})`, async () => { + const { server, connection } = await setup([status, 'reply'], 0); + + const result = await send(connection, 'OpenSession', 1); + + expect(result).to.equal('OpenSession result'); + expect(server.requests).to.equal(2); + }); + } + + it('retries OpenSession after a request timeout, as the kernel', async () => { + const { server, connection } = await setup(['stall', 'reply'], 0); + + const result = await send(connection, 'OpenSession', 1); + + expect(result).to.equal('OpenSession result'); + expect(server.requests).to.equal(2); + }); + + it('fails an OpenSession that times out on every attempt with "Request timed out"', async () => { + const { server, connection } = await setup(['stall', 'stall', 'stall'], 0); + + const error = await expectRejection(send(connection, 'OpenSession', 1)); + + expect(error.message).to.equal('Request timed out'); + expect(server.requests).to.equal(3); + }); + + it('fails with "Request timed out" when the first timeout uses up retriesTimeout, as with the defaults', async () => { + const { server, connection } = await setup(['stall', 'reply'], 0, { retriesTimeout: 300 }); + + const error = await expectRejection(send(connection, 'OpenSession', 1)); + + expect(error.message).to.equal('Request timed out'); + expect(server.requests).to.equal(1); + }); + + it('does not retry ExecuteStatement after a request timeout', async () => { + const { server, connection } = await setup(['stall', 'reply'], 0); const error = await expectRejection(send(connection, 'ExecuteStatement', 1)); - expect(error).to.be.instanceOf(THTTPException); - expect(error.statusCode).to.equal(503); + expect(error.message).to.equal('Request timed out'); + expect(server.requests).to.equal(1); + }); + + it('retries OpenSession when the connection is refused', async () => { + const { server, agent, connection } = await setup(['reply'], 1); + + const result = await send(connection, 'OpenSession', 1); + + expect(result).to.equal('OpenSession result'); + expect(agent.connectAttempts).to.equal(2); expect(server.requests).to.equal(1); }); it('does not retry other non-idempotent operations when the connection is refused', async () => { const { server, agent, connection } = await setup(['reply'], 1); - const error = await expectRejection(send(connection, 'OpenSession', 1)); + const error = await expectRejection(send(connection, 'FetchResults', 1)); expect(error).to.be.instanceOf(FetchError); expect(error.code).to.equal('ECONNREFUSED'); @@ -208,4 +359,155 @@ describe('ThriftHttpConnection retries', () => { expect(result).to.equal('GetOperationStatus result'); expect(server.requests).to.equal(2); }); + + describe('once retries run out', () => { + function expectHttpError(error: any, status: number) { + expect(error).to.be.instanceOf(THTTPException); + expect(error.statusCode).to.equal(status); + expect(error.message).to.equal(`Received a response with a bad HTTP status code: ${status}: ${ERROR_DETAIL}`); + } + + // Requests that weren't retried before fail as they did then + for (const [method, action, status] of [ + ['OpenSession', 'status-503', 503], + ['OpenSession', 'status-429', 429], + ['OpenSession', 'status-501', 501], + ['GetOperationStatus', 'status-408', 408], + ['GetOperationStatus', 'status-501', 501], + ['ExecuteStatement', 'status-429', 429], + ['ExecuteStatement', 'status-503-retry-after', 503], + ] as const) { + it(`fails ${method} with the last HTTP error (${action})`, async () => { + const { server, connection } = await setup([action, action, action], 0); + + const error = await expectRejection(send(connection, method, 1)); + + expectHttpError(error, status); + expect(server.requests).to.equal(3); + }); + } + + it('fails OpenSession with the connection error', async () => { + const { server, agent, connection } = await setup([], Infinity); + + const error = await expectRejection(send(connection, 'OpenSession', 1)); + + expect(error).to.be.instanceOf(FetchError); + expect(error.code).to.equal('ECONNREFUSED'); + expect(agent.connectAttempts).to.equal(3); + expect(server.requests).to.equal(0); + }); + + it('fails OpenSession with the network error when the connection closes after the request', async () => { + const action = 'close-after-request'; + const { server, connection } = await setup([action, action, action], 0); + + const error = await expectRejection(send(connection, 'OpenSession', 1)); + + expect(error).to.be.instanceOf(FetchError); + expect(error.code).to.equal('ECONNRESET'); + expect(server.requests).to.equal(3); + }); + + // Retried before, so they keep failing with the RetryError + for (const [action, status] of [ + ['status-503', 503], + ['status-099', 99], + ] as const) { + it(`keeps the RetryError for an idempotent RPC on a ${status}`, async () => { + const { server, connection } = await setup([action, action, action], 0); + + const error = await expectRejection(send(connection, 'GetOperationStatus', 1)); + + expect(error).to.be.instanceOf(RetryError); + expect(error.errorCode).to.equal(RetryErrorCode.AttemptsExceeded); + expect(error.payload.response.status).to.equal(status); + expect(server.requests).to.equal(3); + }); + } + + it('keeps the RetryError for an idempotent RPC whose connection is refused', async () => { + const { agent, connection } = await setup([], Infinity); + + const error = await expectRejection(send(connection, 'GetOperationStatus', 1)); + + expect(error).to.be.instanceOf(RetryError); + expect(error.errorCode).to.equal(RetryErrorCode.AttemptsExceeded); + expect(error.payload.code).to.equal('ECONNREFUSED'); + expect(agent.connectAttempts).to.equal(3); + }); + }); +}); + +describe('ThriftHttpConnection with a certificate that fails verification', () => { + // A trusted self-signed `localhost` certificate with an unknown critical extension, generated per run so no + // private key is committed. OpenSSL rejects it as "unhandled critical extension", which Node reports as + // `UNSPECIFIED` because it has no name for that X509 error. + let cert = ''; + let key = ''; + + before(function () { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'dbsql-cert-')); + try { + execFileSync( + 'openssl', + [ + 'req', + '-x509', + '-newkey', + 'ec', + '-pkeyopt', + 'ec_paramgen_curve:prime256v1', + '-nodes', + '-days', + '1', + '-subj', + '/CN=localhost', + '-addext', + 'subjectAltName=DNS:localhost,IP:127.0.0.1', + '-addext', + '1.3.6.1.4.1.55555.1=critical,ASN1:UTF8String:x', + '-keyout', + path.join(dir, 'key.pem'), + '-out', + path.join(dir, 'cert.pem'), + ], + { stdio: 'ignore' }, + ); + cert = fs.readFileSync(path.join(dir, 'cert.pem'), 'utf8'); + key = fs.readFileSync(path.join(dir, 'key.pem'), 'utf8'); + } catch { + this.skip(); // openssl is not available + } finally { + fs.rmSync(dir, { recursive: true, force: true }); + } + }); + + it('fails OpenSession at once with the certificate error Node reports as UNSPECIFIED', async () => { + let connections = 0; + const server = https.createServer({ cert, key }, (req, res) => res.end()); + server.on('connection', () => { + connections += 1; + }); + await new Promise((resolve) => { + server.listen(0, '127.0.0.1', resolve); + }); + const context = new ClientContextStub({ retryDelayMin: 1, retryDelayMax: 2, retryMaxAttempts: 3 }); + const { port } = server.address() as AddressInfo; + const httpConnection = new HttpConnection({ host: 'localhost', port, path: '/', https: true, ca: cert }, context); + try { + const connection = await httpConnection.getThriftConnection(); + + const error = await expectRejection(send(connection, 'OpenSession', 1)); + + expect(error).to.be.instanceOf(FetchError); + expect(error.code).to.equal('UNSPECIFIED'); + expect(error.message).to.contain('unhandled critical extension'); + expect(connections).to.equal(1); + } finally { + (await httpConnection.getAgent())?.destroy(); + server.closeAllConnections(); + server.close(); + } + }); }); diff --git a/tests/unit/connection/connections/UnsentRequestRetryPolicy.test.ts b/tests/unit/connection/connections/UnsentRequestRetryPolicy.test.ts deleted file mode 100644 index ca01a9a7..00000000 --- a/tests/unit/connection/connections/UnsentRequestRetryPolicy.test.ts +++ /dev/null @@ -1,110 +0,0 @@ -import { expect, AssertionError } from 'chai'; -import sinon from 'sinon'; -import { Request, Response, FetchError } from 'node-fetch'; -import UnsentRequestRetryPolicy from '../../../../lib/connection/connections/UnsentRequestRetryPolicy'; -import RetryError, { RetryErrorCode } from '../../../../lib/errors/RetryError'; - -import ClientContextStub from '../../.stubs/ClientContextStub'; - -function systemError(code: string, reason = code): FetchError { - return new FetchError(`request to https://example.com failed, reason: ${reason}`, 'system', { - code, - } as unknown as NodeJS.ErrnoException); -} - -function okResponse() { - return { request: new Request('http://localhost'), response: new Response(undefined, { status: 200 }) }; -} - -function createPolicy(retryMaxAttempts = 5) { - return new UnsentRequestRetryPolicy(new ClientContextStub({ retryDelayMin: 1, retryDelayMax: 2, retryMaxAttempts })); -} - -async function expectRejection(promise: Promise): Promise { - try { - await promise; - } catch (error) { - if (error instanceof AssertionError) { - throw error; - } - return error; - } - return expect.fail('It should throw an error'); -} - -describe('UnsentRequestRetryPolicy', () => { - for (const code of ['ECONNREFUSED', 'ENOTFOUND', 'EAI_AGAIN']) { - it(`retries ${code}, which is raised before the request is sent`, async () => { - const operation = sinon.stub(); - operation.onCall(0).rejects(systemError(code)); - operation.onCall(1).resolves(okResponse()); - - const result = await createPolicy().invokeWithRetry(operation); - - expect(result.response.status).to.equal(200); - expect(operation.callCount).to.equal(2); - }); - } - - const sentRequestErrors: Array<[string, Error]> = [ - ['ECONNRESET (socket hang up)', systemError('ECONNRESET', 'socket hang up')], - ['EPIPE', systemError('EPIPE')], - ['ETIMEDOUT', systemError('ETIMEDOUT')], - ['EHOSTUNREACH', systemError('EHOSTUNREACH')], - ['a body timeout', new FetchError('Response timeout while trying to fetch', 'body-timeout')], - ['a premature close', new Error('Premature close')], - ]; - - for (const [label, error] of sentRequestErrors) { - it(`does not retry ${label}`, async () => { - const operation = sinon.stub().rejects(error); - - expect(await expectRejection(createPolicy().invokeWithRetry(operation))).to.equal(error); - expect(operation.callCount).to.equal(1); - }); - } - - for (const status of [429, 500, 503]) { - it(`does not retry an HTTP ${status} response`, async () => { - const details = { request: new Request('http://localhost'), response: new Response(undefined, { status }) }; - const policy = createPolicy(); - const operation = sinon.stub().resolves(details); - - expect(await policy.invokeWithRetry(operation)).to.equal(details); - expect(await policy.shouldRetry(details)).to.deep.equal({ shouldRetry: false }); - expect(operation.callCount).to.equal(1); - }); - } - - it('rethrows the original error once max attempts is reached', async () => { - const error = systemError('ECONNREFUSED'); - const operation = sinon.stub().rejects(error); - - expect(await expectRejection(createPolicy(3).invokeWithRetry(operation))).to.equal(error); - expect(operation.callCount).to.equal(3); - }); - - it('rethrows the original error once the retry timeout is exceeded', async () => { - const error = systemError('ECONNREFUSED'); - const policy = new UnsentRequestRetryPolicy( - new ClientContextStub({ retryDelayMin: 1, retryDelayMax: 2, retryMaxAttempts: 100, retriesTimeout: 0 }), - ); - const operation = sinon.stub().rejects(error); - - expect(await expectRejection(policy.invokeWithRetry(operation))).to.equal(error); - expect(operation.callCount).to.equal(1); - }); - - it('passes through a RetryError raised by the operation itself', async () => { - for (const error of [ - new RetryError(RetryErrorCode.AttemptsExceeded), - new RetryError(RetryErrorCode.TimeoutExceeded, new TypeError('unrelated')), - ]) { - const operation = sinon.stub().rejects(error); - - // eslint-disable-next-line no-await-in-loop - expect(await expectRejection(createPolicy().invokeWithRetry(operation))).to.equal(error); - expect(operation.callCount).to.equal(1); - } - }); -}); diff --git a/tests/unit/result/CloudFetchResultHandler.test.ts b/tests/unit/result/CloudFetchResultHandler.test.ts index 4498d177..a1dd524e 100644 --- a/tests/unit/result/CloudFetchResultHandler.test.ts +++ b/tests/unit/result/CloudFetchResultHandler.test.ts @@ -5,7 +5,7 @@ import { AddressInfo } from 'net'; import Int64 from 'node-int64'; import { compressFrameSync } from 'lz4-napi'; import { Request, Response } from 'node-fetch'; -import { ShouldRetryResult } from '../../../lib/connection/contracts/IRetryPolicy'; +import { RetryRequestClass, ShouldRetryResult } from '../../../lib/connection/contracts/IRetryPolicy'; import { HttpTransactionDetails } from '../../../lib/connection/contracts/IConnectionProvider'; import { LogLevel } from '../../../lib/contracts/IDBSQLLogger'; import CloudFetchResultHandler from '../../../lib/result/CloudFetchResultHandler'; @@ -748,9 +748,14 @@ describe('CloudFetchResultHandler', () => { // Real `HttpRetryPolicy` and node-fetch against a local server, so the deadline is exercised // across actual attempts, backoff sleeps and sockets. - function createHandler(fileLink: string, configOverrides: Partial) { + function createHandler(fileLink: string, configOverrides: Partial, agent?: http.Agent) { const context = new BaseClientContextStub({ cloudFetchConcurrentDownloads: 1, ...configOverrides }); - sinon.stub(context.connectionProvider, 'getRetryPolicy').callsFake(async () => new HttpRetryPolicy(context)); + sinon + .stub(context.connectionProvider, 'getRetryPolicy') + .callsFake(async (requestClass?: RetryRequestClass) => new HttpRetryPolicy(context, requestClass)); + if (agent) { + sinon.stub(context.connectionProvider, 'getAgent').resolves(agent); + } const rowSet: TRowSet = { startRowOffset: new Int64(0), rows: [], @@ -789,6 +794,61 @@ describe('CloudFetchResultHandler', () => { return { ...outcome, elapsedMs: Date.now() - startedAt }; } + // An agent whose every connection fails with `code` before any request is sent + function failingAgent(code: string, onLookup = () => {}) { + return new http.Agent({ + lookup: (hostname: string, options: unknown, callback: Function) => { + onLookup(); + callback(Object.assign(new Error(`${code} ${hostname}`), { code })); + }, + } as unknown as http.AgentOptions); + } + + // A certificate failure is thrown without retrying (`UNSPECIFIED`: an X509 error Node doesn't name) + for (const code of ['CERT_HAS_EXPIRED', 'UNSPECIFIED']) { + it(`should not retry or leak the presigned URL from a certificate failure (${code})`, async () => { + let lookups = 0; + const agent = failingAgent(code, () => { + lookups += 1; + }); + const handler = createHandler( + 'http://storage.example.invalid/chunk?sig=SECRET_SIGNATURE', + { retryDelayMin: 1, retryDelayMax: 2, retryMaxAttempts: 3 }, + agent, + ); + + const { error } = await fetchWithin(handler, 1500); + agent.destroy(); + + expect(lookups).to.be.equal(1); + expect(error.code).to.be.equal(code); + expect(error.message).to.be.equal(`CloudFetch result download failed: network error ${code}`); + expect(error.stack).to.not.contain('SECRET_SIGNATURE'); + }); + } + + it('should retry a download whose host does not resolve, as the kernel', async () => { + let lookups = 0; + const agent = failingAgent('ENOTFOUND', () => { + lookups += 1; + }); + const handler = createHandler( + 'http://storage.example.invalid/chunk?sig=SECRET_SIGNATURE', + { retryDelayMin: 1, retryDelayMax: 2, retryMaxAttempts: 3 }, + agent, + ); + + const { error } = await fetchWithin(handler, 1500); + agent.destroy(); + + expect(lookups).to.be.equal(3); + expect(error).to.be.instanceOf(RetryError); + expect(error.message).to.be.equal( + 'CloudFetch result download failed: Max retry count exceeded (last error: network error ENOTFOUND)', + ); + expect(error.stack).to.not.contain('SECRET_SIGNATURE'); + }); + it('should fail a download whose response headers never arrive with a timeout error', async () => { const closedRequests: Array> = []; const url = await startServer((req) => { diff --git a/tests/unit/telemetry/FeatureFlagCache.test.ts b/tests/unit/telemetry/FeatureFlagCache.test.ts index 7f1ea951..227854ad 100644 --- a/tests/unit/telemetry/FeatureFlagCache.test.ts +++ b/tests/unit/telemetry/FeatureFlagCache.test.ts @@ -16,8 +16,11 @@ import { expect } from 'chai'; import sinon from 'sinon'; +import http from 'http'; +import { AddressInfo } from 'net'; import { Response } from 'node-fetch'; import FeatureFlagCache from '../../../lib/FeatureFlagCache'; +import HttpRetryPolicy from '../../../lib/connection/connections/HttpRetryPolicy'; import ClientContextStub from '../.stubs/ClientContextStub'; import { LogLevel } from '../../../lib/contracts/IDBSQLLogger'; @@ -273,6 +276,60 @@ describe('FeatureFlagCache', () => { fetchStub.restore(); }); + + // connect() waits for the flags, so a failing or stalled endpoint must not hold it past the fetch deadline + const slowEndpoints: Array<[string, http.RequestListener]> = [ + [ + 'keeps failing and asks for long waits', + (req, res) => { + res.writeHead(503, { 'Retry-After': '60' }); + res.end(); + }, + ], + ['never responds', () => {}], + ]; + + for (const [label, handler] of slowEndpoints) { + it(`should fall back to defaults at the fetch deadline when the endpoint ${label}`, async () => { + clock.restore(); + let hits = 0; + const server = http.createServer((req, res) => { + hits += 1; + handler(req, res); + }); + await new Promise((resolve) => { + server.listen(0, '127.0.0.1', resolve); + }); + try { + // The default retry settings, with a real retry policy + const context = new ClientContextStub(); + sinon.stub(context.connectionProvider, 'getRetryPolicy').callsFake(async () => new HttpRetryPolicy(context)); + const cache = new FeatureFlagCache(context); + (cache as any).FETCH_DEADLINE_MS = 300; + // The flags URL is always https, so point the real fetch at the local endpoint + const { port } = server.address() as AddressInfo; + const fetchWithRetry = (cache as any).fetchWithRetry.bind(cache); + sinon + .stub(cache as any, 'fetchWithRetry') + .callsFake((url: unknown, init: unknown) => fetchWithRetry(`http://127.0.0.1:${port}/`, init)); + cache.getOrCreateContext('test-host'); + + const started = Date.now(); + expect(await cache.getBoolean('test-host', 'flag', true)).to.equal(true); + + expect(Date.now() - started).to.be.within(250, 1000); + expect(hits).to.equal(1); + } finally { + server.closeAllConnections(); + server.close(); + // Let node-fetch finish the aborted request (it clears its timer) before faking timers again + await new Promise((resolve) => { + setTimeout(resolve, 10); + }); + clock = sinon.useFakeTimers(); + } + }); + } }); describe('Integration scenarios', () => {