From a5496cbb3424d006af75e4a5f6543b7c8e63b742 Mon Sep 17 00:00:00 2001 From: Vu Anh Phung Date: Mon, 16 Nov 2026 06:44:16 +0000 Subject: [PATCH 1/5] fix: update kernel pin to release closed async statements Pin the kernel to dac32e2 on main. It includes databricks/databricks-sql-kernel#462: AsyncStatement.close() now releases its canceller and the result streams of its handles, so a closed async statement no longer keeps the session, and the host's shared telemetry client, alive until GC. It also picks up #451, #473 and #454. The regenerated native/kernel/index.d.ts only adds #462's doc comments. Signed-off-by: Vu Anh Phung Co-authored-by: Isaac --- KERNEL_REV | 2 +- native/kernel/index.d.ts | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/KERNEL_REV b/KERNEL_REV index de1b52ca..18337b65 100644 --- a/KERNEL_REV +++ b/KERNEL_REV @@ -1 +1 @@ -0b5d0edfed32c0745d5abc2468fc3070e2eb2a17 +dac32e2bf44468bdab06747dad4f289d1c0a18bd diff --git a/native/kernel/index.d.ts b/native/kernel/index.d.ts index 1f1d3581..87640ca0 100644 --- a/native/kernel/index.d.ts +++ b/native/kernel/index.d.ts @@ -31,6 +31,8 @@ export declare class AsyncResultHandle { * message), suitable for handing to `apache-arrow`'s * `RecordBatchReader`. Byte-identical to the sync * `Statement.fetchNextBatch()` payload for the same query. + * Rejects with `KernelError(InvalidStatementHandle)` after the + * parent `AsyncStatement` is closed (see `AsyncStatement.close()`). */ fetchNextBatch(): Promise /** @@ -146,6 +148,10 @@ export declare class AsyncStatement { * closed-handle short-circuit and returns `Ok(())` without * re-attempting the wire call. The kernel's own `Drop` * fire-and-forget retry runs once in the background. + * + * Also invalidates the handles `awaitResult()` returned: their + * `fetchNextBatch()` then rejects with `InvalidStatementHandle`, + * unless a fetch was in progress when `close()` ran. */ close(): Promise } From e2577eef1a40f7a13f17ba9bddbf15332b9de08b Mon Sep 17 00:00:00 2001 From: Vu Anh Phung Date: Mon, 16 Nov 2026 07:21:17 +0000 Subject: [PATCH 2/5] fix(kernel): report a fetch interrupted by close() as OperationStateError With the new kernel pin, AsyncStatement.close() empties the result handle, so a fetchChunk() or hasMore() in flight on SEA rejects with the kernel's InvalidStatementHandle ("AsyncResultHandle already closed"), surfaced as a generic HiveDriverError. Check the lifecycle first in their catch blocks so a local close or cancel still reports OperationStateError(Closed/Canceled). Signed-off-by: Vu Anh Phung Co-authored-by: Isaac --- lib/kernel/KernelOperationBackend.ts | 4 ++ tests/unit/kernel/operation-lifecycle.test.ts | 40 +++++++++++++++++++ 2 files changed, 44 insertions(+) diff --git a/lib/kernel/KernelOperationBackend.ts b/lib/kernel/KernelOperationBackend.ts index 81a11142..aa0d2e79 100644 --- a/lib/kernel/KernelOperationBackend.ts +++ b/lib/kernel/KernelOperationBackend.ts @@ -330,6 +330,9 @@ export default class KernelOperationBackend implements IOperationBackend { } return await slicer.fetchNext({ limit, disableBuffering }); } catch (err) { + // A local close()/cancel() can invalidate the kernel result handle + // mid-fetch; report the lifecycle state, not the handle error. + failIfNotActive(this.lifecycle); throw await this.fetchFailure(err); } } @@ -341,6 +344,7 @@ export default class KernelOperationBackend implements IOperationBackend { const slicer = await this.getResultSlicer(); return await slicer.hasMore(); } catch (err) { + failIfNotActive(this.lifecycle); throw await this.fetchFailure(err); } } diff --git a/tests/unit/kernel/operation-lifecycle.test.ts b/tests/unit/kernel/operation-lifecycle.test.ts index 91c71431..67e13b4d 100644 --- a/tests/unit/kernel/operation-lifecycle.test.ts +++ b/tests/unit/kernel/operation-lifecycle.test.ts @@ -516,6 +516,46 @@ describe('DBSQLOperation over KernelOperationBackend — close after the result }); }); +describe('KernelOperationBackend — close() during an in-flight fetch', () => { + for (const fetch of ['fetchChunk', 'hasMore'] as const) { + it(`${fetch}() reports OperationStateError(Closed) when close() invalidates the result handle`, async () => { + const ipcBytes = Buffer.from(tableToIPC(tableFromArrays({ x: Int32Array.from([1]) }), 'stream')); + let closed: () => void = () => {}; + const closeDone = new Promise((resolve) => { + closed = resolve; + }); + const statement = { + cancel: sinon.stub().resolves(), + close: sinon.stub().resolves(), + schema: () => ({ ipcBytes }), + // The kernel empties the handle's stream on close(), so the pending fetch fails. + fetchNextBatch: async () => { + await closeDone; + const payload = JSON.stringify({ + code: 'InvalidStatementHandle', + message: 'AsyncResultHandle already closed', + }); + throw new Error(`__databricks_error__:${payload}`); + }, + }; + const backend = new KernelOperationBackend({ statement, context: new ClientContextStub(), id: 'kernel-op' }); + + const inflight = fetch === 'fetchChunk' ? backend.fetchChunk({ limit: 10 }) : backend.hasMore(); + await backend.close(); + closed(); + + let thrown: unknown; + try { + await inflight; + } catch (err) { + thrown = err; + } + expect(thrown).to.be.instanceOf(OperationStateError); + expect((thrown as OperationStateError).errorCode).to.equal(OperationStateErrorCode.Closed); + }); + } +}); + describe('DBSQLSession over KernelSessionBackend — close right after execution', () => { const closeFailure = () => new Error(`__databricks_error__:${JSON.stringify({ code: 'NotFound', message: 'HTTP 404 Not Found' })}`); From d559569c0a5407177630685ff0a855bf3145fa44 Mon Sep 17 00:00:00 2001 From: Vu Anh Phung Date: Mon, 16 Nov 2026 07:40:46 +0000 Subject: [PATCH 3/5] fix(kernel): keep the cleanup close when a cancel interrupts a fetch The guard in the fetchChunk() and hasMore() catch blocks threw before fetchFailure() ran, so a fetch interrupted by cancel() skipped the cleanup close. After a cancel, nothing else closes the statement, so it and its session stayed alive until GC. Move the check into fetchFailure(): record whether a local close or cancel happened, run the cleanup close, then report the lifecycle state. The flags are read first because the cleanup close sets isClosed. Signed-off-by: Vu Anh Phung Co-authored-by: Isaac --- lib/kernel/KernelOperationBackend.ts | 11 ++- tests/unit/kernel/operation-lifecycle.test.ts | 80 ++++++++++--------- 2 files changed, 51 insertions(+), 40 deletions(-) diff --git a/lib/kernel/KernelOperationBackend.ts b/lib/kernel/KernelOperationBackend.ts index aa0d2e79..accd8948 100644 --- a/lib/kernel/KernelOperationBackend.ts +++ b/lib/kernel/KernelOperationBackend.ts @@ -330,9 +330,6 @@ export default class KernelOperationBackend implements IOperationBackend { } return await slicer.fetchNext({ limit, disableBuffering }); } catch (err) { - // A local close()/cancel() can invalidate the kernel result handle - // mid-fetch; report the lifecycle state, not the handle error. - failIfNotActive(this.lifecycle); throw await this.fetchFailure(err); } } @@ -344,7 +341,6 @@ export default class KernelOperationBackend implements IOperationBackend { const slicer = await this.getResultSlicer(); return await slicer.hasMore(); } catch (err) { - failIfNotActive(this.lifecycle); throw await this.fetchFailure(err); } } @@ -740,9 +736,16 @@ export default class KernelOperationBackend implements IOperationBackend { /** * A failed fetch leaves the napi stream in an unspecified state ("call close() * and discard"), so close the statement, then return the decoded error to throw. + * A fetch that a local close()/cancel() interrupted (the kernel empties the + * result handle on close) reports that state instead. */ private async fetchFailure(err: unknown): Promise { + // Read before the cleanup close, which sets `isClosed` itself. + const interrupted = this.lifecycle.isCancelled || this.lifecycle.isClosed; await this.bestEffortClose('fetch-error'); + if (interrupted) { + failIfNotActive(this.lifecycle); + } return decodeNapiKernelError(err); } diff --git a/tests/unit/kernel/operation-lifecycle.test.ts b/tests/unit/kernel/operation-lifecycle.test.ts index 67e13b4d..64e1efb8 100644 --- a/tests/unit/kernel/operation-lifecycle.test.ts +++ b/tests/unit/kernel/operation-lifecycle.test.ts @@ -516,43 +516,51 @@ describe('DBSQLOperation over KernelOperationBackend — close after the result }); }); -describe('KernelOperationBackend — close() during an in-flight fetch', () => { - for (const fetch of ['fetchChunk', 'hasMore'] as const) { - it(`${fetch}() reports OperationStateError(Closed) when close() invalidates the result handle`, async () => { - const ipcBytes = Buffer.from(tableToIPC(tableFromArrays({ x: Int32Array.from([1]) }), 'stream')); - let closed: () => void = () => {}; - const closeDone = new Promise((resolve) => { - closed = resolve; +describe('KernelOperationBackend — close()/cancel() during an in-flight fetch', () => { + const cases = [ + { action: 'close', code: OperationStateErrorCode.Closed }, + { action: 'cancel', code: OperationStateErrorCode.Canceled }, + ] as const; + for (const { action, code } of cases) { + for (const fetch of ['fetchChunk', 'hasMore'] as const) { + it(`${fetch}() reports OperationStateError(${code}) after ${action}() and closes the statement once`, async () => { + const ipcBytes = Buffer.from(tableToIPC(tableFromArrays({ x: Int32Array.from([1]) }), 'stream')); + let interrupted: () => void = () => {}; + const interruptDone = new Promise((resolve) => { + interrupted = resolve; + }); + const statement = { + cancel: sinon.stub().resolves(), + close: sinon.stub().resolves(), + schema: () => ({ ipcBytes }), + // The kernel empties the handle's stream on close(), so the pending fetch fails. + fetchNextBatch: async () => { + await interruptDone; + const payload = JSON.stringify({ + code: 'InvalidStatementHandle', + message: 'AsyncResultHandle already closed', + }); + throw new Error(`__databricks_error__:${payload}`); + }, + }; + const backend = new KernelOperationBackend({ statement, context: new ClientContextStub(), id: 'kernel-op' }); + + const inflight = fetch === 'fetchChunk' ? backend.fetchChunk({ limit: 10 }) : backend.hasMore(); + await (action === 'close' ? backend.close() : backend.cancel()); + interrupted(); + + let thrown: unknown; + try { + await inflight; + } catch (err) { + thrown = err; + } + expect(thrown).to.be.instanceOf(OperationStateError); + expect((thrown as OperationStateError).errorCode).to.equal(code); + // A cancel leaves the statement for the fetch-failure cleanup to close. + expect(statement.close.calledOnce).to.equal(true); }); - const statement = { - cancel: sinon.stub().resolves(), - close: sinon.stub().resolves(), - schema: () => ({ ipcBytes }), - // The kernel empties the handle's stream on close(), so the pending fetch fails. - fetchNextBatch: async () => { - await closeDone; - const payload = JSON.stringify({ - code: 'InvalidStatementHandle', - message: 'AsyncResultHandle already closed', - }); - throw new Error(`__databricks_error__:${payload}`); - }, - }; - const backend = new KernelOperationBackend({ statement, context: new ClientContextStub(), id: 'kernel-op' }); - - const inflight = fetch === 'fetchChunk' ? backend.fetchChunk({ limit: 10 }) : backend.hasMore(); - await backend.close(); - closed(); - - let thrown: unknown; - try { - await inflight; - } catch (err) { - thrown = err; - } - expect(thrown).to.be.instanceOf(OperationStateError); - expect((thrown as OperationStateError).errorCode).to.equal(OperationStateErrorCode.Closed); - }); + } } }); From dc64db0875d849070dadda7ab136a37a4a09a467 Mon Sep 17 00:00:00 2001 From: Vu Anh Phung Date: Tue, 17 Nov 2026 04:44:33 +0000 Subject: [PATCH 4/5] fix(kernel): keep close() intent when the close RPC fails With the new kernel pin, AsyncStatement.close() empties the result handle even when its close RPC fails, but kernelClose rolls isClosed back. A fetch that then hits the emptied handle reported the kernel's HiveDriverError "AsyncResultHandle already closed" instead of OperationStateError(Closed). On the old pin it kept returning rows. Record the close() request in a flag that the failure doesn't reset and treat it in fetchFailure() like a local close. Cleanup closes don't set it. fetchFailure() now builds the Canceled/Closed error itself, so the result doesn't depend on its own cleanup close succeeding. Signed-off-by: Vu Anh Phung Co-authored-by: Isaac --- lib/kernel/KernelOperationBackend.ts | 12 ++- tests/unit/kernel/operation-lifecycle.test.ts | 90 +++++++++++++------ 2 files changed, 72 insertions(+), 30 deletions(-) diff --git a/lib/kernel/KernelOperationBackend.ts b/lib/kernel/KernelOperationBackend.ts index accd8948..ac9fcfb5 100644 --- a/lib/kernel/KernelOperationBackend.ts +++ b/lib/kernel/KernelOperationBackend.ts @@ -225,6 +225,10 @@ export default class KernelOperationBackend implements IOperationBackend { // synchronous terminal-state accessor, so the observed success is retained. private asyncSucceeded = false; + // Set by close() and kept when its RPC fails (which resets `isClosed`): the + // kernel closes the result handle either way. + private closeRequested = false; + constructor({ asyncStatement, statement, @@ -464,6 +468,7 @@ export default class KernelOperationBackend implements IOperationBackend { } public async close(): Promise { + this.closeRequested = true; return kernelClose(this.lifecycle, this.lifecycleHandle, this.context, this._id); } @@ -741,10 +746,13 @@ export default class KernelOperationBackend implements IOperationBackend { */ private async fetchFailure(err: unknown): Promise { // Read before the cleanup close, which sets `isClosed` itself. - const interrupted = this.lifecycle.isCancelled || this.lifecycle.isClosed; + const interrupted = this.lifecycle.isCancelled || this.lifecycle.isClosed || this.closeRequested; await this.bestEffortClose('fetch-error'); if (interrupted) { - failIfNotActive(this.lifecycle); + // Not failIfNotActive(): a failed cleanup close leaves `isClosed` false. + return new OperationStateError( + this.lifecycle.isCancelled ? OperationStateErrorCode.Canceled : OperationStateErrorCode.Closed, + ); } return decodeNapiKernelError(err); } diff --git a/tests/unit/kernel/operation-lifecycle.test.ts b/tests/unit/kernel/operation-lifecycle.test.ts index 64e1efb8..2218d72b 100644 --- a/tests/unit/kernel/operation-lifecycle.test.ts +++ b/tests/unit/kernel/operation-lifecycle.test.ts @@ -517,6 +517,40 @@ describe('DBSQLOperation over KernelOperationBackend — close after the result }); describe('KernelOperationBackend — close()/cancel() during an in-flight fetch', () => { + function inflightFetch(fetch: 'fetchChunk' | 'hasMore', close: sinon.SinonStub) { + const ipcBytes = Buffer.from(tableToIPC(tableFromArrays({ x: Int32Array.from([1]) }), 'stream')); + let interrupted: () => void = () => {}; + const interruptDone = new Promise((resolve) => { + interrupted = resolve; + }); + const statement = { + cancel: sinon.stub().resolves(), + close, + schema: () => ({ ipcBytes }), + // The kernel empties the handle's stream on close(), so the pending fetch fails. + fetchNextBatch: async () => { + await interruptDone; + const payload = JSON.stringify({ + code: 'InvalidStatementHandle', + message: 'AsyncResultHandle already closed', + }); + throw new Error(`__databricks_error__:${payload}`); + }, + }; + const backend = new KernelOperationBackend({ statement, context: new ClientContextStub(), id: 'kernel-op' }); + const inflight = fetch === 'fetchChunk' ? backend.fetchChunk({ limit: 10 }) : backend.hasMore(); + return { statement, backend, inflight, interrupted }; + } + + async function rejectionOf(promise: Promise): Promise { + try { + await promise; + } catch (err) { + return err; + } + return undefined; + } + const cases = [ { action: 'close', code: OperationStateErrorCode.Closed }, { action: 'cancel', code: OperationStateErrorCode.Canceled }, @@ -524,37 +558,12 @@ describe('KernelOperationBackend — close()/cancel() during an in-flight fetch' for (const { action, code } of cases) { for (const fetch of ['fetchChunk', 'hasMore'] as const) { it(`${fetch}() reports OperationStateError(${code}) after ${action}() and closes the statement once`, async () => { - const ipcBytes = Buffer.from(tableToIPC(tableFromArrays({ x: Int32Array.from([1]) }), 'stream')); - let interrupted: () => void = () => {}; - const interruptDone = new Promise((resolve) => { - interrupted = resolve; - }); - const statement = { - cancel: sinon.stub().resolves(), - close: sinon.stub().resolves(), - schema: () => ({ ipcBytes }), - // The kernel empties the handle's stream on close(), so the pending fetch fails. - fetchNextBatch: async () => { - await interruptDone; - const payload = JSON.stringify({ - code: 'InvalidStatementHandle', - message: 'AsyncResultHandle already closed', - }); - throw new Error(`__databricks_error__:${payload}`); - }, - }; - const backend = new KernelOperationBackend({ statement, context: new ClientContextStub(), id: 'kernel-op' }); - - const inflight = fetch === 'fetchChunk' ? backend.fetchChunk({ limit: 10 }) : backend.hasMore(); + const { statement, backend, inflight, interrupted } = inflightFetch(fetch, sinon.stub().resolves()); + await (action === 'close' ? backend.close() : backend.cancel()); interrupted(); - let thrown: unknown; - try { - await inflight; - } catch (err) { - thrown = err; - } + const thrown = await rejectionOf(inflight); expect(thrown).to.be.instanceOf(OperationStateError); expect((thrown as OperationStateError).errorCode).to.equal(code); // A cancel leaves the statement for the fetch-failure cleanup to close. @@ -562,6 +571,31 @@ describe('KernelOperationBackend — close()/cancel() during an in-flight fetch' }); } } + + // The kernel empties the handle even when the close RPC fails (async_statement.rs:398-414 at + // dac32e2); a retried close() then resolves (391-393). Also cover a retry that fails again. + for (const retry of ['resolves', 'rejects'] as const) { + for (const fetch of ['fetchChunk', 'hasMore'] as const) { + it(`${fetch}() reports OperationStateError(${OperationStateErrorCode.Closed}) after a failed close() whose retry ${retry}`, async () => { + const close = sinon + .stub() + .rejects(new Error(`__databricks_error__:${JSON.stringify({ code: 'NotFound', message: 'HTTP 404' })}`)); + if (retry === 'resolves') { + close.onSecondCall().resolves(); + } + const { statement, backend, inflight, interrupted } = inflightFetch(fetch, close); + + expect(await rejectionOf(backend.close())).to.be.instanceOf(Error); + interrupted(); + + const thrown = await rejectionOf(inflight); + expect(thrown).to.be.instanceOf(OperationStateError); + expect((thrown as OperationStateError).errorCode).to.equal(OperationStateErrorCode.Closed); + // The failed close() and one fetch-failure cleanup close. + expect(statement.close.calledTwice).to.equal(true); + }); + } + } }); describe('DBSQLSession over KernelSessionBackend — close right after execution', () => { From 87b7dfbea513b96a3be60bf36a3f4fcc0356916c Mon Sep 17 00:00:00 2001 From: Vu Anh Phung Date: Tue, 17 Nov 2026 04:57:56 +0000 Subject: [PATCH 5/5] fix(kernel): don't report a cleanup close as a client close fetchFailure() counted isClosed as a local close, but its own cleanup close sets that flag before its first await. When two fetches failed on the same error, the second reported OperationStateError(Closed) instead of the error. closeRequested now covers every client close(), so check only it and isCancelled. Signed-off-by: Vu Anh Phung Co-authored-by: Isaac --- lib/kernel/KernelOperationBackend.ts | 4 +-- tests/unit/kernel/operation-lifecycle.test.ts | 29 +++++++++++++++---- 2 files changed, 25 insertions(+), 8 deletions(-) diff --git a/lib/kernel/KernelOperationBackend.ts b/lib/kernel/KernelOperationBackend.ts index ac9fcfb5..81647ece 100644 --- a/lib/kernel/KernelOperationBackend.ts +++ b/lib/kernel/KernelOperationBackend.ts @@ -745,8 +745,8 @@ export default class KernelOperationBackend implements IOperationBackend { * result handle on close) reports that state instead. */ private async fetchFailure(err: unknown): Promise { - // Read before the cleanup close, which sets `isClosed` itself. - const interrupted = this.lifecycle.isCancelled || this.lifecycle.isClosed || this.closeRequested; + // Not `isClosed`: cleanup closes (this one, or a concurrent fetch's) set it too. + const interrupted = this.lifecycle.isCancelled || this.closeRequested; await this.bestEffortClose('fetch-error'); if (interrupted) { // Not failIfNotActive(): a failed cleanup close leaves `isClosed` false. diff --git a/tests/unit/kernel/operation-lifecycle.test.ts b/tests/unit/kernel/operation-lifecycle.test.ts index 2218d72b..290d343e 100644 --- a/tests/unit/kernel/operation-lifecycle.test.ts +++ b/tests/unit/kernel/operation-lifecycle.test.ts @@ -517,7 +517,11 @@ describe('DBSQLOperation over KernelOperationBackend — close after the result }); describe('KernelOperationBackend — close()/cancel() during an in-flight fetch', () => { - function inflightFetch(fetch: 'fetchChunk' | 'hasMore', close: sinon.SinonStub) { + function inflightFetch( + fetch: 'fetchChunk' | 'hasMore', + close: sinon.SinonStub, + kernelError = { code: 'InvalidStatementHandle', message: 'AsyncResultHandle already closed' }, + ) { const ipcBytes = Buffer.from(tableToIPC(tableFromArrays({ x: Int32Array.from([1]) }), 'stream')); let interrupted: () => void = () => {}; const interruptDone = new Promise((resolve) => { @@ -530,11 +534,7 @@ describe('KernelOperationBackend — close()/cancel() during an in-flight fetch' // The kernel empties the handle's stream on close(), so the pending fetch fails. fetchNextBatch: async () => { await interruptDone; - const payload = JSON.stringify({ - code: 'InvalidStatementHandle', - message: 'AsyncResultHandle already closed', - }); - throw new Error(`__databricks_error__:${payload}`); + throw new Error(`__databricks_error__:${JSON.stringify(kernelError)}`); }, }; const backend = new KernelOperationBackend({ statement, context: new ClientContextStub(), id: 'kernel-op' }); @@ -596,6 +596,23 @@ describe('KernelOperationBackend — close()/cancel() during an in-flight fetch' }); } } + + it('reports the fetch error to concurrent fetches that fail without a close() or cancel()', async () => { + const { statement, backend, inflight, interrupted } = inflightFetch('fetchChunk', sinon.stub().resolves(), { + code: 'NetworkError', + message: 'connection reset', + }); + const second = backend.hasMore(); + interrupted(); + + // The first failure's cleanup close sets `isClosed` before the second failure is handled. + for (const thrown of await Promise.all([rejectionOf(inflight), rejectionOf(second)])) { + expect(thrown).to.be.instanceOf(HiveDriverError); + expect(thrown).not.to.be.instanceOf(OperationStateError); + expect((thrown as Error).message).to.equal('connection reset'); + } + expect(statement.close.calledOnce).to.equal(true); + }); }); describe('DBSQLSession over KernelSessionBackend — close right after execution', () => {