Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
- Honor the client `useCloudFetch: false` default on the kernel backend so complete results can be fetched without cloud-storage downloads. (#574)
- Thrift HTTP errors (`THTTPException`) now append server detail from the error headers (at most 500 characters). Kernel-backend errors now carry a non-enumerable `cause` summarizing the kernel error code and message (also at most 500 characters). Reyden fallback preserves KP001 after any existing kernel cause. (#467)
- Add `Symbol.asyncDispose` to clients, sessions, and operations, delegating to `close()`. Use `asyncDisposable()` with interface-typed results in TypeScript; existing interfaces and factory return types remain unchanged. (#331)
- Thrift `ExecuteStatement` requests are now retried when the connection fails before the request is sent (`ECONNREFUSED`, `ENOTFOUND`, `EAI_AGAIN`), within the existing `retryMaxAttempts` / `retriesTimeout` budget; once retries run out, the original connection error is thrown. Failures after the request may have reached the server are still not retried, so a statement can't run twice. (#458)
- CloudFetch retry-exhaustion errors now identify the download and the last failure: the `RetryError` message reads `CloudFetch result download failed: <original message> (last error: <cause>)`. `<cause>` is `HTTP <status>: <statusText>` (or `HTTP <status>` when the status text is empty) for HTTP failures, and `network error <code>`, `network error (<type>)` (e.g. `aborted`, when there is no `code`) or a bare `network error` for network failures (the raw network-error message, which can contain the presigned URL, is not copied). If no cause can be determined, the ` (last error: …)` suffix is omitted. The `RetryError` instance, `errorCode` and `payload` (including the node-fetch `Request` / `Response`) are unchanged. (#471)
- Report untyped NULL columns as STRING in kernel result metadata, matching Thrift. (#470)
- `IOperation.close()` no longer throws when the close request fails after the operation has finished (for example, once its results were read, or when the execute response already reported completion): the failure is logged as a warning and reported in the returned status's `info`, and the operation is still closed locally. Closing an operation that hasn't finished still throws on failure. (#480)
Expand Down
2 changes: 2 additions & 0 deletions lib/connection/connections/HttpConnection.ts
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ import HiveDriverError from '../../errors/HiveDriverError';
import ThriftHttpConnection from './ThriftHttpConnection';
import IRetryPolicy from '../contracts/IRetryPolicy';
import HttpRetryPolicy from './HttpRetryPolicy';
import UnsentRequestRetryPolicy from './UnsentRequestRetryPolicy';

// Set by the driver, auth provider or Thrift transport. node-fetch merges
// same-named headers case-insensitively into one value, so a caller copy
Expand Down Expand Up @@ -184,6 +185,7 @@ export default class HttpConnection implements IConnectionProvider {
transport: thrift.TBufferedTransport,
protocol: thrift.TBinaryProtocol,
getRetryPolicy: () => this.getRetryPolicy(),
getUnsentRequestRetryPolicy: async () => new UnsentRequestRetryPolicy(this.context),
},
{
agent,
Expand Down
11 changes: 11 additions & 0 deletions lib/connection/connections/ThriftHttpConnection.ts
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,7 @@ interface ThriftHttpConnectionOptions {
transport?: TTransportType;
protocol?: TProtocolConstructor;
getRetryPolicy(): Promise<IRetryPolicy<HttpTransactionDetails>>;
getUnsentRequestRetryPolicy?(): Promise<IRetryPolicy<HttpTransactionDetails>>;
}

// This type describes a shape of internals of Thrift client object.
Expand Down Expand Up @@ -92,6 +93,9 @@ const retryableThriftMethods = new Set([
'GetCrossReference',
]);

// Not safe to repeat once the server may have the request, so retried only if it was never sent
const unsentRequestRetryableThriftMethods = new Set(['ExecuteStatement']);

export default class ThriftHttpConnection extends EventEmitter {
private readonly url: string;

Expand Down Expand Up @@ -122,6 +126,13 @@ export default class ThriftHttpConnection extends EventEmitter {
if (thriftMethodName && retryableThriftMethods.has(thriftMethodName)) {
return this.options.getRetryPolicy();
}
if (
thriftMethodName &&
unsentRequestRetryableThriftMethods.has(thriftMethodName) &&
this.options.getUnsentRequestRetryPolicy
) {
return this.options.getUnsentRequestRetryPolicy();
}
// Don't retry everything that is not explicitly allowed to retry
return new NullRetryPolicy();
}
Expand Down
32 changes: 32 additions & 0 deletions lib/connection/connections/UnsentRequestRetryPolicy.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
import HttpRetryPolicy from './HttpRetryPolicy';
import { HttpTransactionDetails } from '../contracts/IConnectionProvider';
import { RetryableOperation } from '../contracts/IRetryPolicy';
import RetryError from '../../errors/RetryError';

// Raised before a connection exists, so no request bytes were written. Failures on a connected
// socket (ECONNRESET, EPIPE, timeouts, ...) are excluded: the server may already have the request.
const UNSENT_REQUEST_ERROR_CODES = new Set(['ECONNREFUSED', 'ENOTFOUND', 'EAI_AGAIN']);

// For operations that are not safe to repeat: retries only requests that never left the client.
export default class UnsentRequestRetryPolicy extends HttpRetryPolicy {
public async invokeWithRetry(operation: RetryableOperation<HttpTransactionDetails>): Promise<HttpTransactionDetails> {
try {
return await super.invokeWithRetry(operation);
} catch (error) {
// When retries run out, surface the connection error itself rather than a RetryError
if (error instanceof RetryError && this.isRetryableNetworkError(error.payload)) {
throw error.payload;
}
throw error;
}
}

protected isRetryable(): boolean {
return false;
}

protected isRetryableNetworkError(error: unknown): boolean {
const code = (error as { code?: unknown } | undefined)?.code;
return typeof code === 'string' && UNSENT_REQUEST_ERROR_CODES.has(code);
}
}
211 changes: 211 additions & 0 deletions tests/unit/connection/connections/ThriftHttpConnectionRetry.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,211 @@
import http from 'http';
import net, { AddressInfo } from 'net';
import { expect } from 'chai';
import { FetchError } from 'node-fetch';
import HttpConnection from '../../../../lib/connection/connections/HttpConnection';
import ThriftHttpConnection, { THTTPException } from '../../../../lib/connection/connections/ThriftHttpConnection';
import ClientContextStub from '../../.stubs/ClientContextStub';

// Header-only Thrift binary messages: enough for method-name detection and for the fake `recv_*` below.
function thriftMessage(name: string, type: number, seqId: number): Buffer {
const nameBytes = Buffer.from(name);
const buffer = Buffer.alloc(12 + nameBytes.length);
buffer.writeUInt32BE((0x80010000 | type) >>> 0, 0);
buffer.writeInt32BE(nameBytes.length, 4);
nameBytes.copy(buffer, 8);
buffer.writeInt32BE(seqId, 8 + nameBytes.length);
return buffer;
}

const CALL = 1;
const REPLY = 2;

type ServerAction = 'reply' | 'close-after-request' | 'status-503';

interface TestServer {
port: number;
requests: number;
close(): void;
}

async function startServer(actions: ServerAction[]): Promise<TestServer> {
const state = { requests: 0 };
const server = http.createServer((req, res) => {
const chunks: Buffer[] = [];
req.on('data', (chunk: Buffer) => chunks.push(chunk));
req.on('end', () => {
const action = actions[state.requests] ?? 'reply';
state.requests += 1;
if (action === 'close-after-request') {
// The full request has been received; drop the connection without a response
req.socket.destroy();
} else if (action === 'status-503') {
res.writeHead(503);
res.end();
} else {
const body = Buffer.concat(chunks);
const nameLength = body.readInt32BE(4);
const name = body.toString('utf8', 8, 8 + nameLength);
res.writeHead(200, { 'Content-Type': 'application/x-thrift' });
res.end(thriftMessage(name, REPLY, body.readInt32BE(8 + nameLength)));
}
});
});
await new Promise<void>((resolve) => {
server.listen(0, '127.0.0.1', resolve);
});
return {
port: (server.address() as AddressInfo).port,
get requests() {
return state.requests;
},
close() {
server.closeAllConnections();
server.close();
},
};
}

async function getClosedPort(): Promise<number> {
const server = net.createServer();
await new Promise<void>((resolve) => {
server.listen(0, '127.0.0.1', resolve);
});
const { port } = server.address() as AddressInfo;
await new Promise<void>((resolve) => {
server.close(() => resolve());
});
return port;
}

// Keep-alive agent whose first `refusals` connection attempts go to a closed port (ECONNREFUSED)
class RefusingAgent extends http.Agent {
public connectAttempts = 0;

constructor(private readonly refusals: number, private readonly closedPort: number) {
super({ keepAlive: true });
}

createConnection(options: net.NetConnectOpts): net.Socket {
this.connectAttempts += 1;
if (this.connectAttempts <= this.refusals) {
return net.createConnection({ host: '127.0.0.1', port: this.closedPort });
}
return net.createConnection(options);
}
}

async function createThriftConnection(port: number, agent: http.Agent): Promise<ThriftHttpConnection> {
const context = new ClientContextStub({ retryDelayMin: 1, retryDelayMax: 2, retryMaxAttempts: 3 });
const httpConnection = new HttpConnection({ host: '127.0.0.1', port, path: '/' }, context);
httpConnection['agent'] = agent;
return httpConnection.getThriftConnection();
}

function send(connection: ThriftHttpConnection, method: string, seqId: number): Promise<unknown> {
return new Promise((resolve, reject) => {
const reqs: Record<number, (error: unknown, result?: unknown) => void> = {
[seqId]: (error, result) => (error ? reject(error) : resolve(result)),
};
connection.client = {
_reqs: reqs,
[`recv_${method}`](this: { _reqs: typeof reqs }, input: unknown, mtype: unknown, rseqid: number) {
const callback = this._reqs[rseqid];
delete this._reqs[rseqid];
callback(null, `${method} result`);
},
} as any;
connection.write(thriftMessage(method, CALL, seqId), seqId);
});
}

async function expectRejection(promise: Promise<unknown>): Promise<any> {
try {
await promise;
} catch (error) {
return error;
}
return expect.fail('It should throw an error');
}

describe('ThriftHttpConnection retries', () => {
const cleanup: Array<() => void> = [];

afterEach(() => {
cleanup.splice(0).forEach((fn) => fn());
});

async function setup(actions: ServerAction[], refusals: number) {
const closedPort = await getClosedPort();
const server = await startServer(actions);
const agent = new RefusingAgent(refusals, closedPort);
cleanup.push(
() => server.close(),
() => agent.destroy(),
);
const connection = await createThriftConnection(server.port, agent);
return { server, agent, connection };
}

it('retries ExecuteStatement when the connection is refused before the request is sent', async () => {
const { server, agent, connection } = await setup(['reply'], 1);

const result = await send(connection, 'ExecuteStatement', 1);

expect(result).to.equal('ExecuteStatement result');
expect(agent.connectAttempts).to.equal(2);
expect(server.requests).to.equal(1);
});

it('surfaces the original error once retries for a refused ExecuteStatement are exhausted', async () => {
const { server, agent, connection } = await setup([], Infinity);

const error = await expectRejection(send(connection, 'ExecuteStatement', 1));

expect(error).to.be.instanceOf(FetchError);
expect(error.code).to.equal('ECONNREFUSED');
expect(agent.connectAttempts).to.equal(3);
expect(server.requests).to.equal(0);
});

it('does not retry ExecuteStatement when the connection closes after the request was sent', async () => {
const { server, connection } = await setup(['reply', 'close-after-request'], 0);

await send(connection, 'ExecuteStatement', 1);
const error = await expectRejection(send(connection, 'ExecuteStatement', 2));

expect(error).to.be.instanceOf(FetchError);
expect(error.code).to.equal('ECONNRESET');
expect(server.requests).to.equal(2);
});

it('does not retry ExecuteStatement on an HTTP error status', async () => {
const { server, connection } = await setup(['status-503'], 0);

const error = await expectRejection(send(connection, 'ExecuteStatement', 1));

expect(error).to.be.instanceOf(THTTPException);
expect(error.statusCode).to.equal(503);
expect(server.requests).to.equal(1);
});

it('does not retry other non-idempotent operations when the connection is refused', async () => {
const { server, agent, connection } = await setup(['reply'], 1);

const error = await expectRejection(send(connection, 'OpenSession', 1));

expect(error).to.be.instanceOf(FetchError);
expect(error.code).to.equal('ECONNREFUSED');
expect(agent.connectAttempts).to.equal(1);
expect(server.requests).to.equal(0);
});

it('still retries idempotent operations when the connection closes after the request was sent', async () => {
const { server, connection } = await setup(['close-after-request', 'reply'], 0);

const result = await send(connection, 'GetOperationStatus', 1);

expect(result).to.equal('GetOperationStatus result');
expect(server.requests).to.equal(2);
});
});
Loading
Loading