diff --git a/KERNEL_REV b/KERNEL_REV index de1b52ca..18337b65 100644 --- a/KERNEL_REV +++ b/KERNEL_REV @@ -1 +1 @@ -0b5d0edfed32c0745d5abc2468fc3070e2eb2a17 +dac32e2bf44468bdab06747dad4f289d1c0a18bd diff --git a/lib/kernel/KernelOperationBackend.ts b/lib/kernel/KernelOperationBackend.ts index 81a11142..81647ece 100644 --- a/lib/kernel/KernelOperationBackend.ts +++ b/lib/kernel/KernelOperationBackend.ts @@ -225,6 +225,10 @@ export default class KernelOperationBackend implements IOperationBackend { // synchronous terminal-state accessor, so the observed success is retained. private asyncSucceeded = false; + // Set by close() and kept when its RPC fails (which resets `isClosed`): the + // kernel closes the result handle either way. + private closeRequested = false; + constructor({ asyncStatement, statement, @@ -464,6 +468,7 @@ export default class KernelOperationBackend implements IOperationBackend { } public async close(): Promise { + this.closeRequested = true; return kernelClose(this.lifecycle, this.lifecycleHandle, this.context, this._id); } @@ -736,9 +741,19 @@ export default class KernelOperationBackend implements IOperationBackend { /** * A failed fetch leaves the napi stream in an unspecified state ("call close() * and discard"), so close the statement, then return the decoded error to throw. + * A fetch that a local close()/cancel() interrupted (the kernel empties the + * result handle on close) reports that state instead. */ private async fetchFailure(err: unknown): Promise { + // Not `isClosed`: cleanup closes (this one, or a concurrent fetch's) set it too. + const interrupted = this.lifecycle.isCancelled || this.closeRequested; await this.bestEffortClose('fetch-error'); + if (interrupted) { + // Not failIfNotActive(): a failed cleanup close leaves `isClosed` false. + return new OperationStateError( + this.lifecycle.isCancelled ? OperationStateErrorCode.Canceled : OperationStateErrorCode.Closed, + ); + } return decodeNapiKernelError(err); } diff --git a/native/kernel/index.d.ts b/native/kernel/index.d.ts index 1f1d3581..87640ca0 100644 --- a/native/kernel/index.d.ts +++ b/native/kernel/index.d.ts @@ -31,6 +31,8 @@ export declare class AsyncResultHandle { * message), suitable for handing to `apache-arrow`'s * `RecordBatchReader`. Byte-identical to the sync * `Statement.fetchNextBatch()` payload for the same query. + * Rejects with `KernelError(InvalidStatementHandle)` after the + * parent `AsyncStatement` is closed (see `AsyncStatement.close()`). */ fetchNextBatch(): Promise /** @@ -146,6 +148,10 @@ export declare class AsyncStatement { * closed-handle short-circuit and returns `Ok(())` without * re-attempting the wire call. The kernel's own `Drop` * fire-and-forget retry runs once in the background. + * + * Also invalidates the handles `awaitResult()` returned: their + * `fetchNextBatch()` then rejects with `InvalidStatementHandle`, + * unless a fetch was in progress when `close()` ran. */ close(): Promise } diff --git a/tests/unit/kernel/operation-lifecycle.test.ts b/tests/unit/kernel/operation-lifecycle.test.ts index 91c71431..290d343e 100644 --- a/tests/unit/kernel/operation-lifecycle.test.ts +++ b/tests/unit/kernel/operation-lifecycle.test.ts @@ -516,6 +516,105 @@ describe('DBSQLOperation over KernelOperationBackend — close after the result }); }); +describe('KernelOperationBackend — close()/cancel() during an in-flight fetch', () => { + function inflightFetch( + fetch: 'fetchChunk' | 'hasMore', + close: sinon.SinonStub, + kernelError = { code: 'InvalidStatementHandle', message: 'AsyncResultHandle already closed' }, + ) { + const ipcBytes = Buffer.from(tableToIPC(tableFromArrays({ x: Int32Array.from([1]) }), 'stream')); + let interrupted: () => void = () => {}; + const interruptDone = new Promise((resolve) => { + interrupted = resolve; + }); + const statement = { + cancel: sinon.stub().resolves(), + close, + schema: () => ({ ipcBytes }), + // The kernel empties the handle's stream on close(), so the pending fetch fails. + fetchNextBatch: async () => { + await interruptDone; + throw new Error(`__databricks_error__:${JSON.stringify(kernelError)}`); + }, + }; + const backend = new KernelOperationBackend({ statement, context: new ClientContextStub(), id: 'kernel-op' }); + const inflight = fetch === 'fetchChunk' ? backend.fetchChunk({ limit: 10 }) : backend.hasMore(); + return { statement, backend, inflight, interrupted }; + } + + async function rejectionOf(promise: Promise): Promise { + try { + await promise; + } catch (err) { + return err; + } + return undefined; + } + + const cases = [ + { action: 'close', code: OperationStateErrorCode.Closed }, + { action: 'cancel', code: OperationStateErrorCode.Canceled }, + ] as const; + for (const { action, code } of cases) { + for (const fetch of ['fetchChunk', 'hasMore'] as const) { + it(`${fetch}() reports OperationStateError(${code}) after ${action}() and closes the statement once`, async () => { + const { statement, backend, inflight, interrupted } = inflightFetch(fetch, sinon.stub().resolves()); + + await (action === 'close' ? backend.close() : backend.cancel()); + interrupted(); + + const thrown = await rejectionOf(inflight); + expect(thrown).to.be.instanceOf(OperationStateError); + expect((thrown as OperationStateError).errorCode).to.equal(code); + // A cancel leaves the statement for the fetch-failure cleanup to close. + expect(statement.close.calledOnce).to.equal(true); + }); + } + } + + // The kernel empties the handle even when the close RPC fails (async_statement.rs:398-414 at + // dac32e2); a retried close() then resolves (391-393). Also cover a retry that fails again. + for (const retry of ['resolves', 'rejects'] as const) { + for (const fetch of ['fetchChunk', 'hasMore'] as const) { + it(`${fetch}() reports OperationStateError(${OperationStateErrorCode.Closed}) after a failed close() whose retry ${retry}`, async () => { + const close = sinon + .stub() + .rejects(new Error(`__databricks_error__:${JSON.stringify({ code: 'NotFound', message: 'HTTP 404' })}`)); + if (retry === 'resolves') { + close.onSecondCall().resolves(); + } + const { statement, backend, inflight, interrupted } = inflightFetch(fetch, close); + + expect(await rejectionOf(backend.close())).to.be.instanceOf(Error); + interrupted(); + + const thrown = await rejectionOf(inflight); + expect(thrown).to.be.instanceOf(OperationStateError); + expect((thrown as OperationStateError).errorCode).to.equal(OperationStateErrorCode.Closed); + // The failed close() and one fetch-failure cleanup close. + expect(statement.close.calledTwice).to.equal(true); + }); + } + } + + it('reports the fetch error to concurrent fetches that fail without a close() or cancel()', async () => { + const { statement, backend, inflight, interrupted } = inflightFetch('fetchChunk', sinon.stub().resolves(), { + code: 'NetworkError', + message: 'connection reset', + }); + const second = backend.hasMore(); + interrupted(); + + // The first failure's cleanup close sets `isClosed` before the second failure is handled. + for (const thrown of await Promise.all([rejectionOf(inflight), rejectionOf(second)])) { + expect(thrown).to.be.instanceOf(HiveDriverError); + expect(thrown).not.to.be.instanceOf(OperationStateError); + expect((thrown as Error).message).to.equal('connection reset'); + } + expect(statement.close.calledOnce).to.equal(true); + }); +}); + describe('DBSQLSession over KernelSessionBackend — close right after execution', () => { const closeFailure = () => new Error(`__databricks_error__:${JSON.stringify({ code: 'NotFound', message: 'HTTP 404 Not Found' })}`);