From 885f3c53917d0eeffc7c0d64491bab150b8c6014 Mon Sep 17 00:00:00 2001 From: Vu Anh Phung Date: Wed, 18 Nov 2026 08:19:54 +0000 Subject: [PATCH 1/9] fix(kernel): preserve rows across overlapping result fetches Share result pipeline initialization and provider priming, and serialize kernel cursor consumption through the converter and slicer. Keep Thrift interleaving and the public API unchanged. Signed-off-by: Vu Anh Phung Co-authored-by: Isaac --- CHANGELOG.md | 1 + lib/kernel/KernelOperationBackend.ts | 47 +++++--- lib/kernel/KernelResultsProvider.ts | 13 ++- tests/e2e/concurrent_fetch.test.ts | 39 +++++++ tests/unit/kernel/concurrent-fetch.test.ts | 122 +++++++++++++++++++++ 5 files changed, 208 insertions(+), 14 deletions(-) create mode 100644 tests/e2e/concurrent_fetch.test.ts create mode 100644 tests/unit/kernel/concurrent-fetch.test.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 938983a3..d3ea3583 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ## Unreleased +- Kernel backend: overlapping `fetchAll()`, `fetchChunk()` and `hasMoreRows()` calls on one operation no longer lose rows. Result consumption is serialized per operation without rejecting concurrent callers. - The Thrift token federation exchange (`enableTokenFederation`) now goes through the connection's agent, so `proxy`, `checkServerCertificate`, `customCaCert` and `clientCert`/`clientKey` apply to it. Previously it connected directly with Node's default agent, so behind a required proxy the exchange failed and the driver fell back to the original token. (#611) - Kernel backend: a failed status poll while waiting for a statement (a `runAsync: true` statement, or a default execution that outlives the server's inline wait) and a failed result fetch in `hasMoreRows()` now throw the mapped driver error, such as `HiveDriverError: HTTP 500: Internal Server Error` once retries on HTTP 5xx run out, instead of a raw `Error` whose message is the internal `__databricks_error__:{...}` envelope. As in `fetchChunk()`, a failed fetch in `hasMoreRows()` also closes the statement. (#608) - Honor `cloudFetchConcurrentDownloads` when opening kernel sessions, with a default window of 16 and explicit chunk-setting overrides. Values other than positive safe integers raise `HiveDriverError` at session open. diff --git a/lib/kernel/KernelOperationBackend.ts b/lib/kernel/KernelOperationBackend.ts index 81647ece..46b9d148 100644 --- a/lib/kernel/KernelOperationBackend.ts +++ b/lib/kernel/KernelOperationBackend.ts @@ -207,7 +207,9 @@ export default class KernelOperationBackend implements IOperationBackend { private readonly rowMode?: RowMode; - private resultSlicer?: ResultSlicer; + private resultSlicer?: Promise>; + + private resultConsumption: Promise = Promise.resolve(); private resultsProvider?: KernelResultsProvider; @@ -328,11 +330,13 @@ export default class KernelOperationBackend implements IOperationBackend { return []; } try { - const slicer = await this.getResultSlicer(); - if (isClosed?.()) { - return []; - } - return await slicer.fetchNext({ limit, disableBuffering }); + return await this.withResultLock(async () => { + const slicer = await this.getResultSlicer(); + if (isClosed?.()) { + return []; + } + return slicer.fetchNext({ limit, disableBuffering }); + }); } catch (err) { throw await this.fetchFailure(err); } @@ -342,8 +346,10 @@ export default class KernelOperationBackend implements IOperationBackend { failIfNotActive(this.lifecycle); try { // `hasMore()` prefetches the next batch, so it fails like `fetchChunk()`. - const slicer = await this.getResultSlicer(); - return await slicer.hasMore(); + return await this.withResultLock(async () => { + const slicer = await this.getResultSlicer(); + return slicer.hasMore(); + }); } catch (err) { throw await this.fetchFailure(err); } @@ -838,10 +844,26 @@ export default class KernelOperationBackend implements IOperationBackend { return this.fetchHandlePromise; } - private async getResultSlicer(): Promise> { - if (this.resultSlicer) { - return this.resultSlicer; + private withResultLock(consume: () => Promise): Promise { + const result = this.resultConsumption.then(consume); + this.resultConsumption = result.then( + () => undefined, + () => undefined, + ); + return result; + } + + private getResultSlicer(): Promise> { + if (!this.resultSlicer) { + this.resultSlicer = this.createResultSlicer().catch((error) => { + this.resultSlicer = undefined; + throw error; + }); } + return this.resultSlicer; + } + + private async createResultSlicer(): Promise> { const metadata = await this.getResultMetadata(); const handle = await this.getFetchHandle(); // KernelResultsProvider consumes only `fetchNextBatch`; both the async result @@ -856,7 +878,6 @@ export default class KernelOperationBackend implements IOperationBackend { disableRowMaterialization: this.context.getConfig().disableRowMaterialization ?? false, rowMode: this.rowMode, }); - this.resultSlicer = new ResultSlicer(this.context, converter); - return this.resultSlicer; + return new ResultSlicer(this.context, converter); } } diff --git a/lib/kernel/KernelResultsProvider.ts b/lib/kernel/KernelResultsProvider.ts index b84fdca4..a48ecddc 100644 --- a/lib/kernel/KernelResultsProvider.ts +++ b/lib/kernel/KernelResultsProvider.ts @@ -54,6 +54,8 @@ export default class KernelResultsProvider implements IResultsProvider; + // Set once the kernel returns `null` from `fetchNextBatch()`. private exhausted = false; @@ -88,7 +90,16 @@ export default class KernelResultsProvider implements IResultsProvider { + private prime(): Promise { + if (!this.priming) { + this.priming = this.primeInternal().finally(() => { + this.priming = undefined; + }); + } + return this.priming; + } + + private async primeInternal(): Promise { // Loop rather than self-recurse: a long run of empty batches drains // iteratively toward the null sentinel without building a deep promise // chain. diff --git a/tests/e2e/concurrent_fetch.test.ts b/tests/e2e/concurrent_fetch.test.ts new file mode 100644 index 00000000..920d9fa1 --- /dev/null +++ b/tests/e2e/concurrent_fetch.test.ts @@ -0,0 +1,39 @@ +import { expect } from 'chai'; +import { DBSQLClient } from '../../lib'; +import config from './utils/config'; + +describe('Concurrent result fetching', () => { + for (const useKernel of [false, true]) { + for (const options of [{}, { maxRows: 17 }, { maxRows: 17, disableBuffering: true }]) { + it(`returns each id once across overlapping fetchAll calls (${useKernel ? 'kernel' : 'thrift'}, ${JSON.stringify( + options, + )})`, async () => { + const client = new DBSQLClient(); + try { + await client.connect({ host: config.host, path: config.path, token: config.token, useKernel } as any); + const session = await client.openSession(); + try { + const rowCount = options.maxRows ? 1000 : 100000; + const operation = await session.executeStatement( + `SELECT id FROM range(0, ${rowCount}) AS T(id) ORDER BY id`, + ); + try { + const chunks = await Promise.all([operation.fetchAll(options), operation.fetchAll(options)]); + const ids = chunks.flat().map((row: any) => Number(row.id)); + expect(ids.sort((left, right) => left - right)).to.deep.equal( + Array.from({ length: rowCount }, (unused, index) => index), + ); + expect(await operation.hasMoreRows()).to.equal(false); + } finally { + await operation.close(); + } + } finally { + await session.close(); + } + } finally { + await client.close(); + } + }); + } + } +}); diff --git a/tests/unit/kernel/concurrent-fetch.test.ts b/tests/unit/kernel/concurrent-fetch.test.ts new file mode 100644 index 00000000..0f086cd7 --- /dev/null +++ b/tests/unit/kernel/concurrent-fetch.test.ts @@ -0,0 +1,122 @@ +import { expect } from 'chai'; +import sinon from 'sinon'; +import { tableFromArrays, tableToIPC } from 'apache-arrow'; +import KernelOperationBackend from '../../../lib/kernel/KernelOperationBackend'; +import KernelResultsProvider from '../../../lib/kernel/KernelResultsProvider'; +import ArrowResultConverter from '../../../lib/result/ArrowResultConverter'; +import ResultSlicer from '../../../lib/result/ResultSlicer'; +import { TTypeId } from '../../../thrift/TCLIService_types'; +import ClientContextStub from '../.stubs/ClientContextStub'; +import ResultsProviderStub from '../.stubs/ResultsProviderStub'; + +function createStatement() { + const batches = [ + [0, 1, 2], + [3, 4, 5], + [6, 7, 8], + ].map((ids) => ({ + ipcBytes: Buffer.from(tableToIPC(tableFromArrays({ id: Int32Array.from(ids) }))), + })); + return { + statementId: 'concurrent-fetch', + schema: () => batches[0], + fetchNextBatch: sinon.stub().callsFake(async () => batches.shift() ?? null), + close: sinon.stub().resolves(), + cancel: sinon.stub().resolves(), + }; +} + +describe('Kernel concurrent result consumption', () => { + it('shares one provider prime between concurrent hasMore calls', async () => { + const statement = createStatement(); + const provider = new KernelResultsProvider(statement); + expect(await Promise.all([provider.hasMore(), provider.hasMore()])).to.deep.equal([true, true]); + expect(statement.fetchNextBatch.callCount).to.equal(1); + let rows = 0; + while (await provider.hasMore()) { + rows += (await provider.fetchNext({ limit: 100 })).rowCount; + } + expect(rows).to.equal(9); + }); + + it('releases the provider prime after a rejection', async () => { + const statement = createStatement(); + const failure = new Error('fetch failed'); + statement.fetchNextBatch.onFirstCall().rejects(failure); + const provider = new KernelResultsProvider(statement); + const outcomes = await Promise.allSettled([provider.hasMore(), provider.hasMore()]); + expect(outcomes).to.deep.equal([ + { status: 'rejected', reason: failure }, + { status: 'rejected', reason: failure }, + ]); + expect(await provider.hasMore()).to.equal(true); + expect((await provider.fetchNext({ limit: 100 })).rowCount).to.equal(3); + }); + + it('constructs one result pipeline for concurrent first readers', async () => { + const backend = new KernelOperationBackend({ statement: createStatement(), context: new ClientContextStub() }); + const [first, second] = await Promise.all([backend['getResultSlicer'](), backend['getResultSlicer']()]); + expect(first).to.equal(second); + }); + + it('serializes Arrow converter prefetch on an initialized pipeline', async () => { + const context = new ClientContextStub(); + const backend = new KernelOperationBackend({ statement: createStatement(), context }); + const batches = [ + [0, 1], + [2, 3], + [4, 5], + ].map((ids) => ({ + batches: [Buffer.from(tableToIPC(tableFromArrays({ id: Int32Array.from(ids) })))], + rowCount: ids.length, + })); + const source = new ResultsProviderStub(batches, { batches: [], rowCount: 0 }); + const converter = new ArrowResultConverter(context, source, { + schema: { + columns: [ + { columnName: 'id', position: 1, typeDesc: { types: [{ primitiveEntry: { type: TTypeId.INT_TYPE } }] } }, + ], + }, + }); + sinon.stub(backend as any, 'getResultSlicer').resolves(new ResultSlicer(context, converter)); + const chunks = await Promise.all([ + backend.fetchChunk({ limit: 100, disableBuffering: true }), + backend.fetchChunk({ limit: 100, disableBuffering: true }), + ]); + while (await backend.hasMore()) { + chunks.push(await backend.fetchChunk({ limit: 100, disableBuffering: true })); + } + expect(chunks.flat().map((row: any) => row.id)).to.deep.equal([0, 1, 2, 3, 4, 5]); + }); + + for (const initialized of [false, true]) { + for (const limit of [1, 4, 100]) { + it(`returns an exact union with overlapping chunks (initialized=${initialized}, limit=${limit})`, async () => { + const backend = new KernelOperationBackend({ statement: createStatement(), context: new ClientContextStub() }); + if (initialized) { + await backend.hasMore(); + } + const chunks = await Promise.all([backend.fetchChunk({ limit }), backend.fetchChunk({ limit })]); + while (await backend.hasMore()) { + chunks.push(await backend.fetchChunk({ limit })); + } + const ids = chunks.flat().map((row: any) => row.id); + expect(ids.sort((left, right) => left - right)).to.deep.equal([0, 1, 2, 3, 4, 5, 6, 7, 8]); + }); + } + } + + it('does not lose prefetched rows when hasMore overlaps unbuffered reads', async () => { + const backend = new KernelOperationBackend({ statement: createStatement(), context: new ClientContextStub() }); + const [, first, second] = await Promise.all([ + backend.hasMore(), + backend.fetchChunk({ limit: 1, disableBuffering: true }), + backend.fetchChunk({ limit: 1, disableBuffering: true }), + ]); + const chunks = [first, second]; + while (await backend.hasMore()) { + chunks.push(await backend.fetchChunk({ limit: 1, disableBuffering: true })); + } + expect(chunks.flat().map((row: any) => row.id)).to.deep.equal([0, 1, 2, 3, 4, 5, 6, 7, 8]); + }); +}); From 939755070a8110ea236138e9e264ba0b304cf772 Mon Sep 17 00:00:00 2001 From: Vu Anh Phung Date: Wed, 18 Nov 2026 09:22:11 +0000 Subject: [PATCH 2/9] fix(kernel): apply overlap review deltas Keep provider priming unchanged behind the backend lock. Check queued readers' lifecycle and cover high-entropy kernel CloudFetch overlap, while documenting the unchanged Thrift CloudFetch gap. Signed-off-by: Vu Anh Phung Co-authored-by: Isaac --- CHANGELOG.md | 2 +- lib/kernel/KernelOperationBackend.ts | 2 + lib/kernel/KernelResultsProvider.ts | 13 +---- tests/e2e/concurrent_fetch.test.ts | 39 ++++++++++++--- tests/unit/kernel/concurrent-fetch.test.ts | 50 +++++++++---------- tests/unit/kernel/operation-lifecycle.test.ts | 14 +++--- 6 files changed, 65 insertions(+), 55 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d3ea3583..73ff4782 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,7 @@ ## Unreleased -- Kernel backend: overlapping `fetchAll()`, `fetchChunk()` and `hasMoreRows()` calls on one operation no longer lose rows. Result consumption is serialized per operation without rejecting concurrent callers. +- Kernel backend only: serialize overlapping `fetchAll()`, `fetchChunk()` and `hasMoreRows()` calls per operation so inline Arrow and CloudFetch results do not lose rows; queued readers honor close/cancel state. Thrift is unchanged: inline/single-page overlap is exact, but concurrent CloudFetch consumption remains a known gap. - The Thrift token federation exchange (`enableTokenFederation`) now goes through the connection's agent, so `proxy`, `checkServerCertificate`, `customCaCert` and `clientCert`/`clientKey` apply to it. Previously it connected directly with Node's default agent, so behind a required proxy the exchange failed and the driver fell back to the original token. (#611) - Kernel backend: a failed status poll while waiting for a statement (a `runAsync: true` statement, or a default execution that outlives the server's inline wait) and a failed result fetch in `hasMoreRows()` now throw the mapped driver error, such as `HiveDriverError: HTTP 500: Internal Server Error` once retries on HTTP 5xx run out, instead of a raw `Error` whose message is the internal `__databricks_error__:{...}` envelope. As in `fetchChunk()`, a failed fetch in `hasMoreRows()` also closes the statement. (#608) - Honor `cloudFetchConcurrentDownloads` when opening kernel sessions, with a default window of 16 and explicit chunk-setting overrides. Values other than positive safe integers raise `HiveDriverError` at session open. diff --git a/lib/kernel/KernelOperationBackend.ts b/lib/kernel/KernelOperationBackend.ts index 46b9d148..a40e25bb 100644 --- a/lib/kernel/KernelOperationBackend.ts +++ b/lib/kernel/KernelOperationBackend.ts @@ -331,6 +331,7 @@ export default class KernelOperationBackend implements IOperationBackend { } try { return await this.withResultLock(async () => { + failIfNotActive(this.lifecycle); const slicer = await this.getResultSlicer(); if (isClosed?.()) { return []; @@ -347,6 +348,7 @@ export default class KernelOperationBackend implements IOperationBackend { try { // `hasMore()` prefetches the next batch, so it fails like `fetchChunk()`. return await this.withResultLock(async () => { + failIfNotActive(this.lifecycle); const slicer = await this.getResultSlicer(); return slicer.hasMore(); }); diff --git a/lib/kernel/KernelResultsProvider.ts b/lib/kernel/KernelResultsProvider.ts index a48ecddc..b84fdca4 100644 --- a/lib/kernel/KernelResultsProvider.ts +++ b/lib/kernel/KernelResultsProvider.ts @@ -54,8 +54,6 @@ export default class KernelResultsProvider implements IResultsProvider; - // Set once the kernel returns `null` from `fetchNextBatch()`. private exhausted = false; @@ -90,16 +88,7 @@ export default class KernelResultsProvider implements IResultsProvider { - if (!this.priming) { - this.priming = this.primeInternal().finally(() => { - this.priming = undefined; - }); - } - return this.priming; - } - - private async primeInternal(): Promise { + private async prime(): Promise { // Loop rather than self-recurse: a long run of empty batches drains // iteratively toward the null sentinel without building a deep promise // chain. diff --git a/tests/e2e/concurrent_fetch.test.ts b/tests/e2e/concurrent_fetch.test.ts index 920d9fa1..fdb0665c 100644 --- a/tests/e2e/concurrent_fetch.test.ts +++ b/tests/e2e/concurrent_fetch.test.ts @@ -1,22 +1,42 @@ import { expect } from 'chai'; import { DBSQLClient } from '../../lib'; +import { LogLevel } from '../../lib/contracts/IDBSQLLogger'; import config from './utils/config'; describe('Concurrent result fetching', () => { for (const useKernel of [false, true]) { - for (const options of [{}, { maxRows: 17 }, { maxRows: 17, disableBuffering: true }]) { - it(`returns each id once across overlapping fetchAll calls (${useKernel ? 'kernel' : 'thrift'}, ${JSON.stringify( - options, - )})`, async () => { - const client = new DBSQLClient(); + for (const { cloudFetch, options } of [ + { cloudFetch: false, options: {} }, + { cloudFetch: false, options: { maxRows: 17 } }, + { cloudFetch: true, options: {} }, + ]) { + const knownThriftGap = cloudFetch && !useKernel; + const test = knownThriftGap ? it.skip : it; + const delivery = cloudFetch ? 'CloudFetch' : `inline, ${JSON.stringify(options)}`; + test(`returns each id once across overlapping fetchAll calls (${useKernel ? 'kernel' : 'thrift'}, ${delivery}${ + knownThriftGap ? '; known Thrift gap' : '' + })`, async () => { + const messages: string[] = []; + const client = new DBSQLClient({ + logger: { + log(level, message) { + if (level === LogLevel.info) messages.push(message); + }, + }, + }); + client.getConfig().useCloudFetch = true; try { await client.connect({ host: config.host, path: config.path, token: config.token, useKernel } as any); const session = await client.openSession(); try { const rowCount = options.maxRows ? 1000 : 100000; - const operation = await session.executeStatement( - `SELECT id FROM range(0, ${rowCount}) AS T(id) ORDER BY id`, - ); + const query = cloudFetch + ? `SELECT id, concat( + sha2(CAST(id AS STRING), 512), sha2(CAST(id + 1 AS STRING), 512), + sha2(CAST(id + 2 AS STRING), 512), sha2(CAST(id + 3 AS STRING), 512) + ) AS payload FROM range(0, 100000) AS T(id) ORDER BY id` + : `SELECT id FROM range(0, ${rowCount}) AS T(id) ORDER BY id`; + const operation = await session.executeStatement(query); try { const chunks = await Promise.all([operation.fetchAll(options), operation.fetchAll(options)]); const ids = chunks.flat().map((row: any) => Number(row.id)); @@ -24,6 +44,9 @@ describe('Concurrent result fetching', () => { Array.from({ length: rowCount }, (unused, index) => index), ); expect(await operation.hasMoreRows()).to.equal(false); + if (cloudFetch) { + expect(messages.some((message) => message.includes('Using CloudFetch reader:'))).to.equal(true); + } } finally { await operation.close(); } diff --git a/tests/unit/kernel/concurrent-fetch.test.ts b/tests/unit/kernel/concurrent-fetch.test.ts index 0f086cd7..ee279311 100644 --- a/tests/unit/kernel/concurrent-fetch.test.ts +++ b/tests/unit/kernel/concurrent-fetch.test.ts @@ -2,7 +2,7 @@ import { expect } from 'chai'; import sinon from 'sinon'; import { tableFromArrays, tableToIPC } from 'apache-arrow'; import KernelOperationBackend from '../../../lib/kernel/KernelOperationBackend'; -import KernelResultsProvider from '../../../lib/kernel/KernelResultsProvider'; +import OperationStateError, { OperationStateErrorCode } from '../../../lib/errors/OperationStateError'; import ArrowResultConverter from '../../../lib/result/ArrowResultConverter'; import ResultSlicer from '../../../lib/result/ResultSlicer'; import { TTypeId } from '../../../thrift/TCLIService_types'; @@ -27,38 +27,34 @@ function createStatement() { } describe('Kernel concurrent result consumption', () => { - it('shares one provider prime between concurrent hasMore calls', async () => { - const statement = createStatement(); - const provider = new KernelResultsProvider(statement); - expect(await Promise.all([provider.hasMore(), provider.hasMore()])).to.deep.equal([true, true]); - expect(statement.fetchNextBatch.callCount).to.equal(1); - let rows = 0; - while (await provider.hasMore()) { - rows += (await provider.fetchNext({ limit: 100 })).rowCount; - } - expect(rows).to.equal(9); - }); - - it('releases the provider prime after a rejection', async () => { - const statement = createStatement(); - const failure = new Error('fetch failed'); - statement.fetchNextBatch.onFirstCall().rejects(failure); - const provider = new KernelResultsProvider(statement); - const outcomes = await Promise.allSettled([provider.hasMore(), provider.hasMore()]); - expect(outcomes).to.deep.equal([ - { status: 'rejected', reason: failure }, - { status: 'rejected', reason: failure }, - ]); - expect(await provider.hasMore()).to.equal(true); - expect((await provider.fetchNext({ limit: 100 })).rowCount).to.equal(3); - }); - it('constructs one result pipeline for concurrent first readers', async () => { const backend = new KernelOperationBackend({ statement: createStatement(), context: new ClientContextStub() }); const [first, second] = await Promise.all([backend['getResultSlicer'](), backend['getResultSlicer']()]); expect(first).to.equal(second); }); + for (const fetch of ['fetchChunk', 'hasMore'] as const) { + it(`rejects a queued ${fetch} after close without touching the cursor`, async () => { + const statement = createStatement(); + const backend = new KernelOperationBackend({ statement, context: new ClientContextStub() }); + await backend['getResultSlicer'](); + let release!: () => void; + backend['resultConsumption'] = new Promise((resolve) => { + release = resolve; + }); + const pending = fetch === 'fetchChunk' ? backend.fetchChunk({ limit: 100 }) : backend.hasMore(); + await backend.close(); + release(); + const [outcome] = await Promise.allSettled([pending]); + expect(outcome.status).to.equal('rejected'); + if (outcome.status === 'rejected') { + expect(outcome.reason).to.be.instanceOf(OperationStateError); + expect(outcome.reason.errorCode).to.equal(OperationStateErrorCode.Closed); + } + expect(statement.fetchNextBatch.called).to.equal(false); + }); + } + it('serializes Arrow converter prefetch on an initialized pipeline', async () => { const context = new ClientContextStub(); const backend = new KernelOperationBackend({ statement: createStatement(), context }); diff --git a/tests/unit/kernel/operation-lifecycle.test.ts b/tests/unit/kernel/operation-lifecycle.test.ts index 290d343e..f2877b2d 100644 --- a/tests/unit/kernel/operation-lifecycle.test.ts +++ b/tests/unit/kernel/operation-lifecycle.test.ts @@ -597,7 +597,7 @@ describe('KernelOperationBackend — close()/cancel() during an in-flight fetch' } } - it('reports the fetch error to concurrent fetches that fail without a close() or cancel()', async () => { + it('reports the fetch error and rejects queued readers after cleanup closes the statement', async () => { const { statement, backend, inflight, interrupted } = inflightFetch('fetchChunk', sinon.stub().resolves(), { code: 'NetworkError', message: 'connection reset', @@ -605,12 +605,12 @@ describe('KernelOperationBackend — close()/cancel() during an in-flight fetch' const second = backend.hasMore(); interrupted(); - // The first failure's cleanup close sets `isClosed` before the second failure is handled. - for (const thrown of await Promise.all([rejectionOf(inflight), rejectionOf(second)])) { - expect(thrown).to.be.instanceOf(HiveDriverError); - expect(thrown).not.to.be.instanceOf(OperationStateError); - expect((thrown as Error).message).to.equal('connection reset'); - } + const [failure, queuedFailure] = await Promise.all([rejectionOf(inflight), rejectionOf(second)]); + expect(failure).to.be.instanceOf(HiveDriverError); + expect(failure).not.to.be.instanceOf(OperationStateError); + expect((failure as Error).message).to.equal('connection reset'); + expect(queuedFailure).to.be.instanceOf(OperationStateError); + expect((queuedFailure as OperationStateError).errorCode).to.equal(OperationStateErrorCode.Closed); expect(statement.close.calledOnce).to.equal(true); }); }); From 87af3368407106214abc75ec914186e7b336c7b4 Mon Sep 17 00:00:00 2001 From: Vu Anh Phung Date: Wed, 18 Nov 2026 09:37:08 +0000 Subject: [PATCH 3/9] fix: serialize concurrent fetches on one operation across backends Give result consumption one owner in DBSQLOperation through fetching, conversion and slicing. Remove the redundant kernel lock and initialization guard. Cover duplicate Thrift FETCH_FIRST requests and both backends' inline and CloudFetch exact unions. Signed-off-by: Vu Anh Phung Co-authored-by: Isaac --- CHANGELOG.md | 2 +- lib/DBSQLOperation.ts | 44 ++++++--- lib/kernel/KernelOperationBackend.ts | 49 +++------- tests/e2e/concurrent_fetch.test.ts | 15 +-- tests/unit/concurrent-fetch.test.ts | 81 ++++++++++++++++ tests/unit/kernel/concurrent-fetch.test.ts | 92 ++++++++++++------- tests/unit/kernel/operation-lifecycle.test.ts | 14 +-- 7 files changed, 204 insertions(+), 93 deletions(-) create mode 100644 tests/unit/concurrent-fetch.test.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 73ff4782..8a053afa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,7 @@ ## Unreleased -- Kernel backend only: serialize overlapping `fetchAll()`, `fetchChunk()` and `hasMoreRows()` calls per operation so inline Arrow and CloudFetch results do not lose rows; queued readers honor close/cancel state. Thrift is unchanged: inline/single-page overlap is exact, but concurrent CloudFetch consumption remains a known gap. +- Serialize concurrent fetches on one operation across the Thrift and kernel backends so overlapping `fetchAll()`, `fetchChunk()` and `hasMoreRows()` calls preserve every row exactly once with inline Arrow and CloudFetch. Queued readers honor close/cancel state. - The Thrift token federation exchange (`enableTokenFederation`) now goes through the connection's agent, so `proxy`, `checkServerCertificate`, `customCaCert` and `clientCert`/`clientKey` apply to it. Previously it connected directly with Node's default agent, so behind a required proxy the exchange failed and the driver fell back to the original token. (#611) - Kernel backend: a failed status poll while waiting for a statement (a `runAsync: true` statement, or a default execution that outlives the server's inline wait) and a failed result fetch in `hasMoreRows()` now throw the mapped driver error, such as `HiveDriverError: HTTP 500: Internal Server Error` once retries on HTTP 5xx run out, instead of a raw `Error` whose message is the internal `__databricks_error__:{...}` envelope. As in `fetchChunk()`, a failed fetch in `hasMoreRows()` also closes the statement. (#608) - Honor `cloudFetchConcurrentDownloads` when opening kernel sessions, with a default window of 16 and explicit chunk-setting overrides. Values other than positive safe integers raise `HiveDriverError` at session open. diff --git a/lib/DBSQLOperation.ts b/lib/DBSQLOperation.ts index 30f095ef..1f04ff5b 100644 --- a/lib/DBSQLOperation.ts +++ b/lib/DBSQLOperation.ts @@ -62,6 +62,8 @@ export default class DBSQLOperation implements IOperation { private metadata?: ResultMetadata; + private resultConsumption: Promise = Promise.resolve(); + private startTime: number = Date.now(); private pollCount: number = 0; @@ -156,19 +158,23 @@ export default class DBSQLOperation implements IOperation { } await this.waitUntilReadyThroughBackend(options); - await this.failIfClosed(); + return this.withResultLock(async () => { + await this.failIfClosed(); - const defaultMaxRows = this.context.getConfig().fetchChunkDefaultMaxRows; - const limit = options?.maxRows ?? defaultMaxRows; - const result = await this.backend.fetchChunk({ - limit, - disableBuffering: options?.disableBuffering, - isClosed: () => this.closed || this.cancelled, - }); - await this.failIfClosed(); + const defaultMaxRows = this.context.getConfig().fetchChunkDefaultMaxRows; + const limit = options?.maxRows ?? defaultMaxRows; + const result = await this.backend.fetchChunk({ + limit, + disableBuffering: options?.disableBuffering, + isClosed: () => this.closed || this.cancelled, + }); + await this.failIfClosed(); - this.context.getLogger().log(LogLevel.debug, `Fetched chunk of size: ${limit} from operation with id: ${this.id}`); - return result; + this.context + .getLogger() + .log(LogLevel.debug, `Fetched chunk of size: ${limit} from operation with id: ${this.id}`); + return result; + }); } /** @@ -291,7 +297,12 @@ export default class DBSQLOperation implements IOperation { } // If we fetched all the data from server - check if there's anything buffered in result handler - return this.backend.hasMore(); + return this.withResultLock(async () => { + if (this.closed || this.cancelled) { + return false; + } + return this.backend.hasMore(); + }); }); } @@ -361,6 +372,15 @@ export default class DBSQLOperation implements IOperation { }); } + private withResultLock(consume: () => Promise): Promise { + const result = this.resultConsumption.then(consume); + this.resultConsumption = result.then( + () => undefined, + () => undefined, + ); + return result; + } + /** * Wrap a public IOperation method so any thrown error is captured as an * error telemetry event before being rethrown to the caller. Telemetry diff --git a/lib/kernel/KernelOperationBackend.ts b/lib/kernel/KernelOperationBackend.ts index a40e25bb..81647ece 100644 --- a/lib/kernel/KernelOperationBackend.ts +++ b/lib/kernel/KernelOperationBackend.ts @@ -207,9 +207,7 @@ export default class KernelOperationBackend implements IOperationBackend { private readonly rowMode?: RowMode; - private resultSlicer?: Promise>; - - private resultConsumption: Promise = Promise.resolve(); + private resultSlicer?: ResultSlicer; private resultsProvider?: KernelResultsProvider; @@ -330,14 +328,11 @@ export default class KernelOperationBackend implements IOperationBackend { return []; } try { - return await this.withResultLock(async () => { - failIfNotActive(this.lifecycle); - const slicer = await this.getResultSlicer(); - if (isClosed?.()) { - return []; - } - return slicer.fetchNext({ limit, disableBuffering }); - }); + const slicer = await this.getResultSlicer(); + if (isClosed?.()) { + return []; + } + return await slicer.fetchNext({ limit, disableBuffering }); } catch (err) { throw await this.fetchFailure(err); } @@ -347,11 +342,8 @@ export default class KernelOperationBackend implements IOperationBackend { failIfNotActive(this.lifecycle); try { // `hasMore()` prefetches the next batch, so it fails like `fetchChunk()`. - return await this.withResultLock(async () => { - failIfNotActive(this.lifecycle); - const slicer = await this.getResultSlicer(); - return slicer.hasMore(); - }); + const slicer = await this.getResultSlicer(); + return await slicer.hasMore(); } catch (err) { throw await this.fetchFailure(err); } @@ -846,26 +838,10 @@ export default class KernelOperationBackend implements IOperationBackend { return this.fetchHandlePromise; } - private withResultLock(consume: () => Promise): Promise { - const result = this.resultConsumption.then(consume); - this.resultConsumption = result.then( - () => undefined, - () => undefined, - ); - return result; - } - - private getResultSlicer(): Promise> { - if (!this.resultSlicer) { - this.resultSlicer = this.createResultSlicer().catch((error) => { - this.resultSlicer = undefined; - throw error; - }); + private async getResultSlicer(): Promise> { + if (this.resultSlicer) { + return this.resultSlicer; } - return this.resultSlicer; - } - - private async createResultSlicer(): Promise> { const metadata = await this.getResultMetadata(); const handle = await this.getFetchHandle(); // KernelResultsProvider consumes only `fetchNextBatch`; both the async result @@ -880,6 +856,7 @@ export default class KernelOperationBackend implements IOperationBackend { disableRowMaterialization: this.context.getConfig().disableRowMaterialization ?? false, rowMode: this.rowMode, }); - return new ResultSlicer(this.context, converter); + this.resultSlicer = new ResultSlicer(this.context, converter); + return this.resultSlicer; } } diff --git a/tests/e2e/concurrent_fetch.test.ts b/tests/e2e/concurrent_fetch.test.ts index fdb0665c..e4a41bfb 100644 --- a/tests/e2e/concurrent_fetch.test.ts +++ b/tests/e2e/concurrent_fetch.test.ts @@ -1,6 +1,8 @@ import { expect } from 'chai'; import { DBSQLClient } from '../../lib'; +import DBSQLOperation from '../../lib/DBSQLOperation'; import { LogLevel } from '../../lib/contracts/IDBSQLLogger'; +import { ResultFormat } from '../../lib/contracts/ResultMetadata'; import config from './utils/config'; describe('Concurrent result fetching', () => { @@ -10,12 +12,10 @@ describe('Concurrent result fetching', () => { { cloudFetch: false, options: { maxRows: 17 } }, { cloudFetch: true, options: {} }, ]) { - const knownThriftGap = cloudFetch && !useKernel; - const test = knownThriftGap ? it.skip : it; const delivery = cloudFetch ? 'CloudFetch' : `inline, ${JSON.stringify(options)}`; - test(`returns each id once across overlapping fetchAll calls (${useKernel ? 'kernel' : 'thrift'}, ${delivery}${ - knownThriftGap ? '; known Thrift gap' : '' - })`, async () => { + it(`returns each id once across overlapping fetchAll calls (${ + useKernel ? 'kernel' : 'thrift' + }, ${delivery})`, async () => { const messages: string[] = []; const client = new DBSQLClient({ logger: { @@ -44,8 +44,11 @@ describe('Concurrent result fetching', () => { Array.from({ length: rowCount }, (unused, index) => index), ); expect(await operation.hasMoreRows()).to.equal(false); - if (cloudFetch) { + if (cloudFetch && useKernel) { expect(messages.some((message) => message.includes('Using CloudFetch reader:'))).to.equal(true); + } else if (cloudFetch) { + const metadata = await (operation as DBSQLOperation).getResultMetadata(); + expect(metadata.resultFormat).to.equal(ResultFormat.UrlBased); } } finally { await operation.close(); diff --git a/tests/unit/concurrent-fetch.test.ts b/tests/unit/concurrent-fetch.test.ts new file mode 100644 index 00000000..87d8b009 --- /dev/null +++ b/tests/unit/concurrent-fetch.test.ts @@ -0,0 +1,81 @@ +import { expect } from 'chai'; +import sinon from 'sinon'; +import Int64 from 'node-int64'; +import { + TFetchOrientation, + TOperationState, + TOperationType, + TSparkRowSetType, + TStatusCode, + TTypeId, +} from '../../thrift/TCLIService_types'; +import ClientContextStub from './.stubs/ClientContextStub'; +import { createOperationForTest } from './.stubs/createOperationForTest'; + +describe('Concurrent Thrift result consumption', () => { + it('sends FETCH_FIRST once when two first reads start without prefetched results', async () => { + const clock = sinon.useFakeTimers(); + const context = new ClientContextStub(); + const orientations: Array = []; + let release!: () => void; + const gate = new Promise((resolve) => { + release = resolve; + }); + sinon.stub(context.driver, 'fetchResults').callsFake(async ({ orientation }) => { + orientations.push(orientation); + await gate; + return { + status: { statusCode: TStatusCode.SUCCESS_STATUS }, + hasMoreRows: false, + results: { + startRowOffset: new Int64(0), + rows: [], + columns: [ + { + i32Val: { + values: orientation === TFetchOrientation.FETCH_FIRST ? [0, 1, 2] : [], + nulls: Buffer.alloc(1), + }, + }, + ], + }, + }; + }); + const operation = createOperationForTest({ + context, + handle: { + operationId: { guid: Buffer.alloc(16), secret: Buffer.alloc(16) }, + operationType: TOperationType.EXECUTE_STATEMENT, + hasResultSet: true, + }, + directResults: { + operationStatus: { + status: { statusCode: TStatusCode.SUCCESS_STATUS }, + operationState: TOperationState.FINISHED_STATE, + }, + resultSetMetadata: { + status: { statusCode: TStatusCode.SUCCESS_STATUS }, + resultFormat: TSparkRowSetType.COLUMN_BASED_SET, + schema: { + columns: [ + { columnName: 'id', position: 1, typeDesc: { types: [{ primitiveEntry: { type: TTypeId.INT_TYPE } }] } }, + ], + }, + }, + }, + }); + try { + const pending = Promise.all([operation.fetchAll(), operation.fetchAll()]); + await clock.tickAsync(0); + release(); + await clock.runAllAsync(); + const chunks = await pending; + expect({ + firstReads: orientations.filter((orientation) => orientation === TFetchOrientation.FETCH_FIRST).length, + ids: chunks.flat().map((row: any) => row.id), + }).to.deep.equal({ firstReads: 1, ids: [0, 1, 2] }); + } finally { + clock.restore(); + } + }); +}); diff --git a/tests/unit/kernel/concurrent-fetch.test.ts b/tests/unit/kernel/concurrent-fetch.test.ts index ee279311..380a34b3 100644 --- a/tests/unit/kernel/concurrent-fetch.test.ts +++ b/tests/unit/kernel/concurrent-fetch.test.ts @@ -1,6 +1,7 @@ import { expect } from 'chai'; import sinon from 'sinon'; import { tableFromArrays, tableToIPC } from 'apache-arrow'; +import DBSQLOperation from '../../../lib/DBSQLOperation'; import KernelOperationBackend from '../../../lib/kernel/KernelOperationBackend'; import OperationStateError, { OperationStateErrorCode } from '../../../lib/errors/OperationStateError'; import ArrowResultConverter from '../../../lib/result/ArrowResultConverter'; @@ -26,38 +27,64 @@ function createStatement() { }; } -describe('Kernel concurrent result consumption', () => { +function createOperation() { + const context = new ClientContextStub(); + const statement = createStatement(); + const backend = new KernelOperationBackend({ statement, context }); + const operation = new DBSQLOperation({ backend, context }); + return { operation, backend, context, statement }; +} + +describe('Kernel concurrent result consumption through DBSQLOperation', () => { it('constructs one result pipeline for concurrent first readers', async () => { - const backend = new KernelOperationBackend({ statement: createStatement(), context: new ClientContextStub() }); - const [first, second] = await Promise.all([backend['getResultSlicer'](), backend['getResultSlicer']()]); + const { operation, backend } = createOperation(); + const getResultSlicer = sinon.spy(backend as any, 'getResultSlicer'); + await Promise.all([operation.fetchChunk({ maxRows: 1 }), operation.fetchChunk({ maxRows: 1 })]); + const [first, second] = await Promise.all(getResultSlicer.returnValues); expect(first).to.equal(second); }); - for (const fetch of ['fetchChunk', 'hasMore'] as const) { - it(`rejects a queued ${fetch} after close without touching the cursor`, async () => { - const statement = createStatement(); - const backend = new KernelOperationBackend({ statement, context: new ClientContextStub() }); + it('releases a failed read so the next waiter observes cleanup before touching the cursor', async () => { + const { operation, statement } = createOperation(); + statement.fetchNextBatch.rejects(new Error('fetch failed')); + const outcomes = await Promise.allSettled([operation.fetchChunk(), operation.fetchChunk()]); + expect(outcomes.map((outcome) => outcome.status)).to.deep.equal(['rejected', 'rejected']); + if (outcomes[0].status === 'rejected' && outcomes[1].status === 'rejected') { + expect(outcomes[0].reason.message).to.equal('fetch failed'); + expect(outcomes[1].reason).to.be.instanceOf(OperationStateError); + expect(outcomes[1].reason.errorCode).to.equal(OperationStateErrorCode.Closed); + } + expect(statement.fetchNextBatch.callCount).to.equal(1); + expect(statement.close.callCount).to.equal(1); + }); + + for (const fetch of ['fetchChunk', 'hasMoreRows'] as const) { + it(`honors close for queued ${fetch} without touching the cursor`, async () => { + const { operation, backend, statement } = createOperation(); await backend['getResultSlicer'](); let release!: () => void; - backend['resultConsumption'] = new Promise((resolve) => { + (operation as any).resultConsumption = new Promise((resolve) => { release = resolve; }); - const pending = fetch === 'fetchChunk' ? backend.fetchChunk({ limit: 100 }) : backend.hasMore(); - await backend.close(); + const pending = fetch === 'fetchChunk' ? operation.fetchChunk({ maxRows: 100 }) : operation.hasMoreRows(); + await operation.close(); release(); const [outcome] = await Promise.allSettled([pending]); - expect(outcome.status).to.equal('rejected'); - if (outcome.status === 'rejected') { - expect(outcome.reason).to.be.instanceOf(OperationStateError); - expect(outcome.reason.errorCode).to.equal(OperationStateErrorCode.Closed); + if (fetch === 'hasMoreRows') { + expect(outcome).to.deep.equal({ status: 'fulfilled', value: false }); + } else { + expect(outcome.status).to.equal('rejected'); + if (outcome.status === 'rejected') { + expect(outcome.reason).to.be.instanceOf(OperationStateError); + expect(outcome.reason.errorCode).to.equal(OperationStateErrorCode.Closed); + } } expect(statement.fetchNextBatch.called).to.equal(false); }); } it('serializes Arrow converter prefetch on an initialized pipeline', async () => { - const context = new ClientContextStub(); - const backend = new KernelOperationBackend({ statement: createStatement(), context }); + const { operation, backend, context } = createOperation(); const batches = [ [0, 1], [2, 3], @@ -76,11 +103,11 @@ describe('Kernel concurrent result consumption', () => { }); sinon.stub(backend as any, 'getResultSlicer').resolves(new ResultSlicer(context, converter)); const chunks = await Promise.all([ - backend.fetchChunk({ limit: 100, disableBuffering: true }), - backend.fetchChunk({ limit: 100, disableBuffering: true }), + operation.fetchChunk({ maxRows: 100, disableBuffering: true }), + operation.fetchChunk({ maxRows: 100, disableBuffering: true }), ]); - while (await backend.hasMore()) { - chunks.push(await backend.fetchChunk({ limit: 100, disableBuffering: true })); + while (await operation.hasMoreRows()) { + chunks.push(await operation.fetchChunk({ maxRows: 100, disableBuffering: true })); } expect(chunks.flat().map((row: any) => row.id)).to.deep.equal([0, 1, 2, 3, 4, 5]); }); @@ -88,13 +115,16 @@ describe('Kernel concurrent result consumption', () => { for (const initialized of [false, true]) { for (const limit of [1, 4, 100]) { it(`returns an exact union with overlapping chunks (initialized=${initialized}, limit=${limit})`, async () => { - const backend = new KernelOperationBackend({ statement: createStatement(), context: new ClientContextStub() }); + const { operation } = createOperation(); if (initialized) { - await backend.hasMore(); + await operation.hasMoreRows(); } - const chunks = await Promise.all([backend.fetchChunk({ limit }), backend.fetchChunk({ limit })]); - while (await backend.hasMore()) { - chunks.push(await backend.fetchChunk({ limit })); + const chunks = await Promise.all([ + operation.fetchChunk({ maxRows: limit }), + operation.fetchChunk({ maxRows: limit }), + ]); + while (await operation.hasMoreRows()) { + chunks.push(await operation.fetchChunk({ maxRows: limit })); } const ids = chunks.flat().map((row: any) => row.id); expect(ids.sort((left, right) => left - right)).to.deep.equal([0, 1, 2, 3, 4, 5, 6, 7, 8]); @@ -103,15 +133,15 @@ describe('Kernel concurrent result consumption', () => { } it('does not lose prefetched rows when hasMore overlaps unbuffered reads', async () => { - const backend = new KernelOperationBackend({ statement: createStatement(), context: new ClientContextStub() }); + const { operation } = createOperation(); const [, first, second] = await Promise.all([ - backend.hasMore(), - backend.fetchChunk({ limit: 1, disableBuffering: true }), - backend.fetchChunk({ limit: 1, disableBuffering: true }), + operation.hasMoreRows(), + operation.fetchChunk({ maxRows: 1, disableBuffering: true }), + operation.fetchChunk({ maxRows: 1, disableBuffering: true }), ]); const chunks = [first, second]; - while (await backend.hasMore()) { - chunks.push(await backend.fetchChunk({ limit: 1, disableBuffering: true })); + while (await operation.hasMoreRows()) { + chunks.push(await operation.fetchChunk({ maxRows: 1, disableBuffering: true })); } expect(chunks.flat().map((row: any) => row.id)).to.deep.equal([0, 1, 2, 3, 4, 5, 6, 7, 8]); }); diff --git a/tests/unit/kernel/operation-lifecycle.test.ts b/tests/unit/kernel/operation-lifecycle.test.ts index f2877b2d..290d343e 100644 --- a/tests/unit/kernel/operation-lifecycle.test.ts +++ b/tests/unit/kernel/operation-lifecycle.test.ts @@ -597,7 +597,7 @@ describe('KernelOperationBackend — close()/cancel() during an in-flight fetch' } } - it('reports the fetch error and rejects queued readers after cleanup closes the statement', async () => { + it('reports the fetch error to concurrent fetches that fail without a close() or cancel()', async () => { const { statement, backend, inflight, interrupted } = inflightFetch('fetchChunk', sinon.stub().resolves(), { code: 'NetworkError', message: 'connection reset', @@ -605,12 +605,12 @@ describe('KernelOperationBackend — close()/cancel() during an in-flight fetch' const second = backend.hasMore(); interrupted(); - const [failure, queuedFailure] = await Promise.all([rejectionOf(inflight), rejectionOf(second)]); - expect(failure).to.be.instanceOf(HiveDriverError); - expect(failure).not.to.be.instanceOf(OperationStateError); - expect((failure as Error).message).to.equal('connection reset'); - expect(queuedFailure).to.be.instanceOf(OperationStateError); - expect((queuedFailure as OperationStateError).errorCode).to.equal(OperationStateErrorCode.Closed); + // The first failure's cleanup close sets `isClosed` before the second failure is handled. + for (const thrown of await Promise.all([rejectionOf(inflight), rejectionOf(second)])) { + expect(thrown).to.be.instanceOf(HiveDriverError); + expect(thrown).not.to.be.instanceOf(OperationStateError); + expect((thrown as Error).message).to.equal('connection reset'); + } expect(statement.close.calledOnce).to.equal(true); }); }); From b2d8ccf0eba733574f22a52884306d44db9071e0 Mon Sep 17 00:00:00 2001 From: Vu Anh Phung Date: Wed, 18 Nov 2026 09:41:05 +0000 Subject: [PATCH 4/9] test: let readers queue before closing the operation Signed-off-by: Vu Anh Phung Co-authored-by: Isaac --- tests/unit/kernel/concurrent-fetch.test.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/unit/kernel/concurrent-fetch.test.ts b/tests/unit/kernel/concurrent-fetch.test.ts index 380a34b3..d2c4b9b1 100644 --- a/tests/unit/kernel/concurrent-fetch.test.ts +++ b/tests/unit/kernel/concurrent-fetch.test.ts @@ -67,6 +67,7 @@ describe('Kernel concurrent result consumption through DBSQLOperation', () => { release = resolve; }); const pending = fetch === 'fetchChunk' ? operation.fetchChunk({ maxRows: 100 }) : operation.hasMoreRows(); + await new Promise((resolve) => setImmediate(resolve)); await operation.close(); release(); const [outcome] = await Promise.allSettled([pending]); From 98aace247ce6fa75a6ee656ef5b868db3498bfda Mon Sep 17 00:00:00 2001 From: Vu Anh Phung Date: Wed, 18 Nov 2026 09:46:30 +0000 Subject: [PATCH 5/9] test: gate distinct Arrow source batches during concurrent reads Signed-off-by: Vu Anh Phung Co-authored-by: Isaac --- tests/unit/kernel/concurrent-fetch.test.ts | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/tests/unit/kernel/concurrent-fetch.test.ts b/tests/unit/kernel/concurrent-fetch.test.ts index d2c4b9b1..3a857bdd 100644 --- a/tests/unit/kernel/concurrent-fetch.test.ts +++ b/tests/unit/kernel/concurrent-fetch.test.ts @@ -84,7 +84,7 @@ describe('Kernel concurrent result consumption through DBSQLOperation', () => { }); } - it('serializes Arrow converter prefetch on an initialized pipeline', async () => { + it('preserves unread Arrow rows when consumers await different source batches', async () => { const { operation, backend, context } = createOperation(); const batches = [ [0, 1], @@ -95,6 +95,16 @@ describe('Kernel concurrent result consumption through DBSQLOperation', () => { rowCount: ids.length, })); const source = new ResultsProviderStub(batches, { batches: [], rowCount: 0 }); + const releases: Array<() => void> = []; + const gates = batches.map(() => new Promise((resolve) => releases.push(resolve))); + const fetchNext = source.fetchNext.bind(source); + let next = 0; + sinon.stub(source, 'fetchNext').callsFake(async () => { + const gate = gates[next++]; + const batch = await fetchNext(); + await gate; + return batch; + }); const converter = new ArrowResultConverter(context, source, { schema: { columns: [ @@ -103,10 +113,13 @@ describe('Kernel concurrent result consumption through DBSQLOperation', () => { }, }); sinon.stub(backend as any, 'getResultSlicer').resolves(new ResultSlicer(context, converter)); - const chunks = await Promise.all([ + const pending = Promise.all([ operation.fetchChunk({ maxRows: 100, disableBuffering: true }), operation.fetchChunk({ maxRows: 100, disableBuffering: true }), ]); + await new Promise((resolve) => setImmediate(resolve)); + releases.forEach((release) => release()); + const chunks = await pending; while (await operation.hasMoreRows()) { chunks.push(await operation.fetchChunk({ maxRows: 100, disableBuffering: true })); } From f79269a7714f18a5268d09968aee2bd99f96ccda Mon Sep 17 00:00:00 2001 From: Vu Anh Phung Date: Wed, 18 Nov 2026 10:12:47 +0000 Subject: [PATCH 6/9] test: cover queued Thrift reads and clarify concurrency guarantees Signed-off-by: Vu Anh Phung Co-authored-by: Isaac --- CHANGELOG.md | 4 +- tests/unit/concurrent-fetch.test.ts | 125 ++++++++++++++++++++++++++++ 2 files changed, 128 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8a053afa..754b5045 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,9 @@ ## Unreleased -- Serialize concurrent fetches on one operation across the Thrift and kernel backends so overlapping `fetchAll()`, `fetchChunk()` and `hasMoreRows()` calls preserve every row exactly once with inline Arrow and CloudFetch. Queued readers honor close/cancel state. +- Serialize concurrent fetches on one operation across the Thrift and kernel backends so overlapping `fetchAll()`, `fetchChunk()` and `hasMoreRows()` calls return every row exactly once with inline Arrow and CloudFetch when all reads succeed. After a failed Thrift read, other readers may continue past that read's rows, matching existing retry-after-error behavior; the kernel closes the statement on a failed fetch. Queued readers honor close/cancel state. +- Concurrent readers on one operation split the result stream with no call-order guarantee. +- A concurrent `hasMoreRows()` waits for an in-flight fetch before checking for more rows. - The Thrift token federation exchange (`enableTokenFederation`) now goes through the connection's agent, so `proxy`, `checkServerCertificate`, `customCaCert` and `clientCert`/`clientKey` apply to it. Previously it connected directly with Node's default agent, so behind a required proxy the exchange failed and the driver fell back to the original token. (#611) - Kernel backend: a failed status poll while waiting for a statement (a `runAsync: true` statement, or a default execution that outlives the server's inline wait) and a failed result fetch in `hasMoreRows()` now throw the mapped driver error, such as `HiveDriverError: HTTP 500: Internal Server Error` once retries on HTTP 5xx run out, instead of a raw `Error` whose message is the internal `__databricks_error__:{...}` envelope. As in `fetchChunk()`, a failed fetch in `hasMoreRows()` also closes the statement. (#608) - Honor `cloudFetchConcurrentDownloads` when opening kernel sessions, with a default window of 16 and explicit chunk-setting overrides. Values other than positive safe integers raise `HiveDriverError` at session open. diff --git a/tests/unit/concurrent-fetch.test.ts b/tests/unit/concurrent-fetch.test.ts index 87d8b009..5a24c648 100644 --- a/tests/unit/concurrent-fetch.test.ts +++ b/tests/unit/concurrent-fetch.test.ts @@ -1,6 +1,8 @@ import { expect } from 'chai'; import sinon from 'sinon'; import Int64 from 'node-int64'; +import { tableFromArrays, tableToIPC } from 'apache-arrow'; +import OperationStateError, { OperationStateErrorCode } from '../../lib/errors/OperationStateError'; import { TFetchOrientation, TOperationState, @@ -12,7 +14,130 @@ import { import ClientContextStub from './.stubs/ClientContextStub'; import { createOperationForTest } from './.stubs/createOperationForTest'; +function createFinishedOperation(context: ClientContextStub) { + return createOperationForTest({ + context, + handle: { + operationId: { guid: Buffer.alloc(16), secret: Buffer.alloc(16) }, + operationType: TOperationType.EXECUTE_STATEMENT, + hasResultSet: true, + }, + directResults: { + operationStatus: { + status: { statusCode: TStatusCode.SUCCESS_STATUS }, + operationState: TOperationState.FINISHED_STATE, + }, + }, + }); +} + describe('Concurrent Thrift result consumption', () => { + for (const phase of ['metadata', 'fetch'] as const) { + it(`rejects a queued fetchChunk after close during an in-flight ${phase} request without further RPCs`, async () => { + const clock = sinon.useFakeTimers(); + const context = new ClientContextStub(); + let release!: () => void; + const gate = new Promise((resolve) => { + release = resolve; + }); + const metadata = sinon.stub(context.driver, 'getResultSetMetadata').callsFake(async () => { + if (phase === 'metadata') { + await gate; + throw new Error('metadata request interrupted by close'); + } + return context.driver.getResultSetMetadataResp; + }); + const fetch = sinon.stub(context.driver, 'fetchResults').callsFake(async () => { + await gate; + return context.driver.fetchResultsResp; + }); + const operation = createFinishedOperation(context); + try { + const pending = Promise.allSettled([ + operation.fetchChunk({ maxRows: 1 }), + operation.fetchChunk({ maxRows: 1 }), + ]); + await clock.tickAsync(0); + const beforeClose = { metadata: metadata.callCount, fetch: fetch.callCount }; + expect(beforeClose).to.deep.equal({ metadata: 1, fetch: phase === 'fetch' ? 1 : 0 }); + await operation.close(); + release(); + await clock.runAllAsync(); + const outcomes = await pending; + expect({ metadata: metadata.callCount, fetch: fetch.callCount }).to.deep.equal(beforeClose); + expect(outcomes.map((outcome) => outcome.status)).to.deep.equal(['rejected', 'rejected']); + if (outcomes[1].status === 'rejected') { + expect(outcomes[1].reason).to.be.instanceOf(OperationStateError); + expect(outcomes[1].reason.errorCode).to.equal(OperationStateErrorCode.Closed); + } + } finally { + release(); + clock.restore(); + } + }); + } + + it('preserves an exact union through the Thrift Arrow pipeline when concurrent reads succeed', async () => { + const clock = sinon.useFakeTimers(); + const context = new ClientContextStub(); + let arrowSchema!: Buffer; + const batches = [ + [0, 1], + [2, 3], + [4, 5], + ].map((ids) => { + const ipc = Buffer.from(tableToIPC(tableFromArrays({ id: Int32Array.from(ids) }))); + const schemaEnd = 8 + ipc.readInt32LE(4); + arrowSchema = ipc.subarray(0, schemaEnd); + return { batch: ipc.subarray(schemaEnd), rowCount: new Int64(ids.length) }; + }); + context.driver.getResultSetMetadataResp = { + status: { statusCode: TStatusCode.SUCCESS_STATUS }, + resultFormat: TSparkRowSetType.ARROW_BASED_SET, + arrowSchema, + schema: { + columns: [ + { columnName: 'id', position: 1, typeDesc: { types: [{ primitiveEntry: { type: TTypeId.INT_TYPE } }] } }, + ], + }, + }; + let release!: () => void; + const gate = new Promise((resolve) => { + release = resolve; + }); + let next = 0; + sinon.stub(context.driver, 'fetchResults').callsFake(async () => { + const index = next++; + await gate; + return { + status: { statusCode: TStatusCode.SUCCESS_STATUS }, + hasMoreRows: index < batches.length - 1, + results: { + startRowOffset: new Int64(index * 2), + rows: [], + arrowBatches: batches[index] ? [batches[index]] : [], + }, + }; + }); + const operation = createFinishedOperation(context); + try { + const pending = Promise.all([operation.fetchAll(), operation.fetchAll()]); + await clock.tickAsync(0); + release(); + await clock.runAllAsync(); + const chunks = await pending; + expect( + chunks + .flat() + .map((row: any) => row.id) + .sort((left, right) => left - right), + ).to.deep.equal([0, 1, 2, 3, 4, 5]); + } finally { + release(); + clock.restore(); + } + }); + it('sends FETCH_FIRST once when two first reads start without prefetched results', async () => { const clock = sinon.useFakeTimers(); const context = new ClientContextStub(); From 0978bcf49318a14438b5b658ea28270845aea8bf Mon Sep 17 00:00:00 2001 From: Vu Anh Phung Date: Wed, 18 Nov 2026 10:18:54 +0000 Subject: [PATCH 7/9] test: select inline delivery explicitly in overlap cases Signed-off-by: Vu Anh Phung Co-authored-by: Isaac --- CHANGELOG.md | 6 +++--- tests/e2e/concurrent_fetch.test.ts | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 754b5045..1235844f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,9 +2,9 @@ ## Unreleased -- Serialize concurrent fetches on one operation across the Thrift and kernel backends so overlapping `fetchAll()`, `fetchChunk()` and `hasMoreRows()` calls return every row exactly once with inline Arrow and CloudFetch when all reads succeed. After a failed Thrift read, other readers may continue past that read's rows, matching existing retry-after-error behavior; the kernel closes the statement on a failed fetch. Queued readers honor close/cancel state. -- Concurrent readers on one operation split the result stream with no call-order guarantee. -- A concurrent `hasMoreRows()` waits for an in-flight fetch before checking for more rows. +- Serialize concurrent fetches on one operation across the Thrift and kernel backends so overlapping `fetchAll()`, `fetchChunk()` and `hasMoreRows()` calls return every row exactly once with inline Arrow and CloudFetch when all reads succeed. After a failed Thrift read, other readers may continue past that read's rows, matching existing retry-after-error behavior; the kernel closes the statement on a failed fetch. Queued readers honor close/cancel state. (#622) +- Concurrent readers on one operation split the result stream with no call-order guarantee. (#622) +- A concurrent `hasMoreRows()` waits for an in-flight fetch before checking for more rows. (#622) - The Thrift token federation exchange (`enableTokenFederation`) now goes through the connection's agent, so `proxy`, `checkServerCertificate`, `customCaCert` and `clientCert`/`clientKey` apply to it. Previously it connected directly with Node's default agent, so behind a required proxy the exchange failed and the driver fell back to the original token. (#611) - Kernel backend: a failed status poll while waiting for a statement (a `runAsync: true` statement, or a default execution that outlives the server's inline wait) and a failed result fetch in `hasMoreRows()` now throw the mapped driver error, such as `HiveDriverError: HTTP 500: Internal Server Error` once retries on HTTP 5xx run out, instead of a raw `Error` whose message is the internal `__databricks_error__:{...}` envelope. As in `fetchChunk()`, a failed fetch in `hasMoreRows()` also closes the statement. (#608) - Honor `cloudFetchConcurrentDownloads` when opening kernel sessions, with a default window of 16 and explicit chunk-setting overrides. Values other than positive safe integers raise `HiveDriverError` at session open. diff --git a/tests/e2e/concurrent_fetch.test.ts b/tests/e2e/concurrent_fetch.test.ts index e4a41bfb..ffda09fd 100644 --- a/tests/e2e/concurrent_fetch.test.ts +++ b/tests/e2e/concurrent_fetch.test.ts @@ -24,7 +24,7 @@ describe('Concurrent result fetching', () => { }, }, }); - client.getConfig().useCloudFetch = true; + client.getConfig().useCloudFetch = cloudFetch; try { await client.connect({ host: config.host, path: config.path, token: config.token, useKernel } as any); const session = await client.openSession(); From 4bb5f46c5347cc997f14d5cde2ad4d42b57b0961 Mon Sep 17 00:00:00 2001 From: Vu Anh Phung Date: Wed, 18 Nov 2026 10:25:45 +0000 Subject: [PATCH 8/9] test: cover overlap in kernel CI and verify inline delivery Signed-off-by: Vu Anh Phung Co-authored-by: Isaac --- .../concurrent-fetch-e2e.test.ts} | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) rename tests/e2e/{concurrent_fetch.test.ts => kernel/concurrent-fetch-e2e.test.ts} (83%) diff --git a/tests/e2e/concurrent_fetch.test.ts b/tests/e2e/kernel/concurrent-fetch-e2e.test.ts similarity index 83% rename from tests/e2e/concurrent_fetch.test.ts rename to tests/e2e/kernel/concurrent-fetch-e2e.test.ts index ffda09fd..327cd5b5 100644 --- a/tests/e2e/concurrent_fetch.test.ts +++ b/tests/e2e/kernel/concurrent-fetch-e2e.test.ts @@ -1,9 +1,9 @@ import { expect } from 'chai'; -import { DBSQLClient } from '../../lib'; -import DBSQLOperation from '../../lib/DBSQLOperation'; -import { LogLevel } from '../../lib/contracts/IDBSQLLogger'; -import { ResultFormat } from '../../lib/contracts/ResultMetadata'; -import config from './utils/config'; +import { DBSQLClient } from '../../../lib'; +import DBSQLOperation from '../../../lib/DBSQLOperation'; +import { LogLevel } from '../../../lib/contracts/IDBSQLLogger'; +import { ResultFormat } from '../../../lib/contracts/ResultMetadata'; +import config from '../utils/config'; describe('Concurrent result fetching', () => { for (const useKernel of [false, true]) { @@ -24,7 +24,7 @@ describe('Concurrent result fetching', () => { }, }, }); - client.getConfig().useCloudFetch = cloudFetch; + client.getConfig().useCloudFetch = useKernel || cloudFetch; try { await client.connect({ host: config.host, path: config.path, token: config.token, useKernel } as any); const session = await client.openSession(); @@ -44,8 +44,9 @@ describe('Concurrent result fetching', () => { Array.from({ length: rowCount }, (unused, index) => index), ); expect(await operation.hasMoreRows()).to.equal(false); - if (cloudFetch && useKernel) { - expect(messages.some((message) => message.includes('Using CloudFetch reader:'))).to.equal(true); + if (useKernel) { + const reader = cloudFetch ? 'CloudFetch' : 'inline Arrow'; + expect(messages.some((message) => message.includes(`Using ${reader} reader:`))).to.equal(true); } else if (cloudFetch) { const metadata = await (operation as DBSQLOperation).getResultMetadata(); expect(metadata.resultFormat).to.equal(ResultFormat.UrlBased); From 6353d6d6e9b9efe1f1b182cca5f5863f9cb1a661 Mon Sep 17 00:00:00 2001 From: Vu Anh Phung Date: Wed, 18 Nov 2026 10:30:43 +0000 Subject: [PATCH 9/9] test: skip only kernel overlap cases when native binding is absent Signed-off-by: Vu Anh Phung Co-authored-by: Isaac --- tests/e2e/kernel/concurrent-fetch-e2e.test.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/e2e/kernel/concurrent-fetch-e2e.test.ts b/tests/e2e/kernel/concurrent-fetch-e2e.test.ts index 327cd5b5..2123e240 100644 --- a/tests/e2e/kernel/concurrent-fetch-e2e.test.ts +++ b/tests/e2e/kernel/concurrent-fetch-e2e.test.ts @@ -3,6 +3,7 @@ import { DBSQLClient } from '../../../lib'; import DBSQLOperation from '../../../lib/DBSQLOperation'; import { LogLevel } from '../../../lib/contracts/IDBSQLLogger'; import { ResultFormat } from '../../../lib/contracts/ResultMetadata'; +import { tryGetKernelNative } from '../../../lib/kernel/KernelNativeLoader'; import config from '../utils/config'; describe('Concurrent result fetching', () => { @@ -13,7 +14,8 @@ describe('Concurrent result fetching', () => { { cloudFetch: true, options: {} }, ]) { const delivery = cloudFetch ? 'CloudFetch' : `inline, ${JSON.stringify(options)}`; - it(`returns each id once across overlapping fetchAll calls (${ + const test = useKernel && !tryGetKernelNative() ? it.skip : it; + test(`returns each id once across overlapping fetchAll calls (${ useKernel ? 'kernel' : 'thrift' }, ${delivery})`, async () => { const messages: string[] = [];