Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion KERNEL_REV
Original file line number Diff line number Diff line change
@@ -1 +1 @@
0b5d0edfed32c0745d5abc2468fc3070e2eb2a17
dac32e2bf44468bdab06747dad4f289d1c0a18bd
15 changes: 15 additions & 0 deletions lib/kernel/KernelOperationBackend.ts
Original file line number Diff line number Diff line change
Expand Up @@ -225,6 +225,10 @@ export default class KernelOperationBackend implements IOperationBackend {
// synchronous terminal-state accessor, so the observed success is retained.
private asyncSucceeded = false;

// Set by close() and kept when its RPC fails (which resets `isClosed`): the
// kernel closes the result handle either way.
private closeRequested = false;

constructor({
asyncStatement,
statement,
Expand Down Expand Up @@ -464,6 +468,7 @@ export default class KernelOperationBackend implements IOperationBackend {
}

public async close(): Promise<Status> {
this.closeRequested = true;
return kernelClose(this.lifecycle, this.lifecycleHandle, this.context, this._id);
}

Expand Down Expand Up @@ -736,9 +741,19 @@ export default class KernelOperationBackend implements IOperationBackend {
/**
* A failed fetch leaves the napi stream in an unspecified state ("call close()
* and discard"), so close the statement, then return the decoded error to throw.
* A fetch that a local close()/cancel() interrupted (the kernel empties the
* result handle on close) reports that state instead.
*/
private async fetchFailure(err: unknown): Promise<Error> {
// Not `isClosed`: cleanup closes (this one, or a concurrent fetch's) set it too.
const interrupted = this.lifecycle.isCancelled || this.closeRequested;
await this.bestEffortClose('fetch-error');
if (interrupted) {
// Not failIfNotActive(): a failed cleanup close leaves `isClosed` false.
return new OperationStateError(
this.lifecycle.isCancelled ? OperationStateErrorCode.Canceled : OperationStateErrorCode.Closed,
);
}
return decodeNapiKernelError(err);
}

Expand Down
6 changes: 6 additions & 0 deletions native/kernel/index.d.ts

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

99 changes: 99 additions & 0 deletions tests/unit/kernel/operation-lifecycle.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -516,6 +516,105 @@ describe('DBSQLOperation over KernelOperationBackend — close after the result
});
});

describe('KernelOperationBackend — close()/cancel() during an in-flight fetch', () => {
function inflightFetch(
fetch: 'fetchChunk' | 'hasMore',
close: sinon.SinonStub,
kernelError = { code: 'InvalidStatementHandle', message: 'AsyncResultHandle already closed' },
) {
const ipcBytes = Buffer.from(tableToIPC(tableFromArrays({ x: Int32Array.from([1]) }), 'stream'));
let interrupted: () => void = () => {};
const interruptDone = new Promise<void>((resolve) => {
interrupted = resolve;
});
const statement = {
cancel: sinon.stub().resolves(),
close,
schema: () => ({ ipcBytes }),
// The kernel empties the handle's stream on close(), so the pending fetch fails.
fetchNextBatch: async () => {
await interruptDone;
throw new Error(`__databricks_error__:${JSON.stringify(kernelError)}`);
},
};
const backend = new KernelOperationBackend({ statement, context: new ClientContextStub(), id: 'kernel-op' });
const inflight = fetch === 'fetchChunk' ? backend.fetchChunk({ limit: 10 }) : backend.hasMore();
return { statement, backend, inflight, interrupted };
}

async function rejectionOf(promise: Promise<unknown>): Promise<unknown> {
try {
await promise;
} catch (err) {
return err;
}
return undefined;
}

const cases = [
{ action: 'close', code: OperationStateErrorCode.Closed },
{ action: 'cancel', code: OperationStateErrorCode.Canceled },
] as const;
for (const { action, code } of cases) {
for (const fetch of ['fetchChunk', 'hasMore'] as const) {
it(`${fetch}() reports OperationStateError(${code}) after ${action}() and closes the statement once`, async () => {
const { statement, backend, inflight, interrupted } = inflightFetch(fetch, sinon.stub().resolves());

await (action === 'close' ? backend.close() : backend.cancel());
interrupted();

const thrown = await rejectionOf(inflight);
expect(thrown).to.be.instanceOf(OperationStateError);
expect((thrown as OperationStateError).errorCode).to.equal(code);
// A cancel leaves the statement for the fetch-failure cleanup to close.
expect(statement.close.calledOnce).to.equal(true);
});
}
}

// The kernel empties the handle even when the close RPC fails (async_statement.rs:398-414 at
// dac32e2); a retried close() then resolves (391-393). Also cover a retry that fails again.
for (const retry of ['resolves', 'rejects'] as const) {
for (const fetch of ['fetchChunk', 'hasMore'] as const) {
it(`${fetch}() reports OperationStateError(${OperationStateErrorCode.Closed}) after a failed close() whose retry ${retry}`, async () => {
const close = sinon
.stub()
.rejects(new Error(`__databricks_error__:${JSON.stringify({ code: 'NotFound', message: 'HTTP 404' })}`));
if (retry === 'resolves') {
close.onSecondCall().resolves();
}
const { statement, backend, inflight, interrupted } = inflightFetch(fetch, close);

expect(await rejectionOf(backend.close())).to.be.instanceOf(Error);
interrupted();

const thrown = await rejectionOf(inflight);
expect(thrown).to.be.instanceOf(OperationStateError);
expect((thrown as OperationStateError).errorCode).to.equal(OperationStateErrorCode.Closed);
// The failed close() and one fetch-failure cleanup close.
expect(statement.close.calledTwice).to.equal(true);
});
}
}

it('reports the fetch error to concurrent fetches that fail without a close() or cancel()', async () => {
const { statement, backend, inflight, interrupted } = inflightFetch('fetchChunk', sinon.stub().resolves(), {
code: 'NetworkError',
message: 'connection reset',
});
const second = backend.hasMore();
interrupted();

// The first failure's cleanup close sets `isClosed` before the second failure is handled.
for (const thrown of await Promise.all([rejectionOf(inflight), rejectionOf(second)])) {
expect(thrown).to.be.instanceOf(HiveDriverError);
expect(thrown).not.to.be.instanceOf(OperationStateError);
expect((thrown as Error).message).to.equal('connection reset');
}
expect(statement.close.calledOnce).to.equal(true);
});
});

describe('DBSQLSession over KernelSessionBackend — close right after execution', () => {
const closeFailure = () =>
new Error(`__databricks_error__:${JSON.stringify({ code: 'NotFound', message: 'HTTP 404 Not Found' })}`);
Expand Down
Loading