diff --git a/CHANGELOG.md b/CHANGELOG.md index 938983a3..1235844f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,9 @@ ## Unreleased +- Serialize concurrent fetches on one operation across the Thrift and kernel backends so overlapping `fetchAll()`, `fetchChunk()` and `hasMoreRows()` calls return every row exactly once with inline Arrow and CloudFetch when all reads succeed. After a failed Thrift read, other readers may continue past that read's rows, matching existing retry-after-error behavior; the kernel closes the statement on a failed fetch. Queued readers honor close/cancel state. (#622) +- Concurrent readers on one operation split the result stream with no call-order guarantee. (#622) +- A concurrent `hasMoreRows()` waits for an in-flight fetch before checking for more rows. (#622) - The Thrift token federation exchange (`enableTokenFederation`) now goes through the connection's agent, so `proxy`, `checkServerCertificate`, `customCaCert` and `clientCert`/`clientKey` apply to it. Previously it connected directly with Node's default agent, so behind a required proxy the exchange failed and the driver fell back to the original token. (#611) - Kernel backend: a failed status poll while waiting for a statement (a `runAsync: true` statement, or a default execution that outlives the server's inline wait) and a failed result fetch in `hasMoreRows()` now throw the mapped driver error, such as `HiveDriverError: HTTP 500: Internal Server Error` once retries on HTTP 5xx run out, instead of a raw `Error` whose message is the internal `__databricks_error__:{...}` envelope. As in `fetchChunk()`, a failed fetch in `hasMoreRows()` also closes the statement. (#608) - Honor `cloudFetchConcurrentDownloads` when opening kernel sessions, with a default window of 16 and explicit chunk-setting overrides. Values other than positive safe integers raise `HiveDriverError` at session open. diff --git a/lib/DBSQLOperation.ts b/lib/DBSQLOperation.ts index 30f095ef..1f04ff5b 100644 --- a/lib/DBSQLOperation.ts +++ b/lib/DBSQLOperation.ts @@ -62,6 +62,8 @@ export default class DBSQLOperation implements IOperation { private metadata?: ResultMetadata; + private resultConsumption: Promise = Promise.resolve(); + private startTime: number = Date.now(); private pollCount: number = 0; @@ -156,19 +158,23 @@ export default class DBSQLOperation implements IOperation { } await this.waitUntilReadyThroughBackend(options); - await this.failIfClosed(); + return this.withResultLock(async () => { + await this.failIfClosed(); - const defaultMaxRows = this.context.getConfig().fetchChunkDefaultMaxRows; - const limit = options?.maxRows ?? defaultMaxRows; - const result = await this.backend.fetchChunk({ - limit, - disableBuffering: options?.disableBuffering, - isClosed: () => this.closed || this.cancelled, - }); - await this.failIfClosed(); + const defaultMaxRows = this.context.getConfig().fetchChunkDefaultMaxRows; + const limit = options?.maxRows ?? defaultMaxRows; + const result = await this.backend.fetchChunk({ + limit, + disableBuffering: options?.disableBuffering, + isClosed: () => this.closed || this.cancelled, + }); + await this.failIfClosed(); - this.context.getLogger().log(LogLevel.debug, `Fetched chunk of size: ${limit} from operation with id: ${this.id}`); - return result; + this.context + .getLogger() + .log(LogLevel.debug, `Fetched chunk of size: ${limit} from operation with id: ${this.id}`); + return result; + }); } /** @@ -291,7 +297,12 @@ export default class DBSQLOperation implements IOperation { } // If we fetched all the data from server - check if there's anything buffered in result handler - return this.backend.hasMore(); + return this.withResultLock(async () => { + if (this.closed || this.cancelled) { + return false; + } + return this.backend.hasMore(); + }); }); } @@ -361,6 +372,15 @@ export default class DBSQLOperation implements IOperation { }); } + private withResultLock(consume: () => Promise): Promise { + const result = this.resultConsumption.then(consume); + this.resultConsumption = result.then( + () => undefined, + () => undefined, + ); + return result; + } + /** * Wrap a public IOperation method so any thrown error is captured as an * error telemetry event before being rethrown to the caller. Telemetry diff --git a/tests/e2e/kernel/concurrent-fetch-e2e.test.ts b/tests/e2e/kernel/concurrent-fetch-e2e.test.ts new file mode 100644 index 00000000..2123e240 --- /dev/null +++ b/tests/e2e/kernel/concurrent-fetch-e2e.test.ts @@ -0,0 +1,68 @@ +import { expect } from 'chai'; +import { DBSQLClient } from '../../../lib'; +import DBSQLOperation from '../../../lib/DBSQLOperation'; +import { LogLevel } from '../../../lib/contracts/IDBSQLLogger'; +import { ResultFormat } from '../../../lib/contracts/ResultMetadata'; +import { tryGetKernelNative } from '../../../lib/kernel/KernelNativeLoader'; +import config from '../utils/config'; + +describe('Concurrent result fetching', () => { + for (const useKernel of [false, true]) { + for (const { cloudFetch, options } of [ + { cloudFetch: false, options: {} }, + { cloudFetch: false, options: { maxRows: 17 } }, + { cloudFetch: true, options: {} }, + ]) { + const delivery = cloudFetch ? 'CloudFetch' : `inline, ${JSON.stringify(options)}`; + const test = useKernel && !tryGetKernelNative() ? it.skip : it; + test(`returns each id once across overlapping fetchAll calls (${ + useKernel ? 'kernel' : 'thrift' + }, ${delivery})`, async () => { + const messages: string[] = []; + const client = new DBSQLClient({ + logger: { + log(level, message) { + if (level === LogLevel.info) messages.push(message); + }, + }, + }); + client.getConfig().useCloudFetch = useKernel || cloudFetch; + try { + await client.connect({ host: config.host, path: config.path, token: config.token, useKernel } as any); + const session = await client.openSession(); + try { + const rowCount = options.maxRows ? 1000 : 100000; + const query = cloudFetch + ? `SELECT id, concat( + sha2(CAST(id AS STRING), 512), sha2(CAST(id + 1 AS STRING), 512), + sha2(CAST(id + 2 AS STRING), 512), sha2(CAST(id + 3 AS STRING), 512) + ) AS payload FROM range(0, 100000) AS T(id) ORDER BY id` + : `SELECT id FROM range(0, ${rowCount}) AS T(id) ORDER BY id`; + const operation = await session.executeStatement(query); + try { + const chunks = await Promise.all([operation.fetchAll(options), operation.fetchAll(options)]); + const ids = chunks.flat().map((row: any) => Number(row.id)); + expect(ids.sort((left, right) => left - right)).to.deep.equal( + Array.from({ length: rowCount }, (unused, index) => index), + ); + expect(await operation.hasMoreRows()).to.equal(false); + if (useKernel) { + const reader = cloudFetch ? 'CloudFetch' : 'inline Arrow'; + expect(messages.some((message) => message.includes(`Using ${reader} reader:`))).to.equal(true); + } else if (cloudFetch) { + const metadata = await (operation as DBSQLOperation).getResultMetadata(); + expect(metadata.resultFormat).to.equal(ResultFormat.UrlBased); + } + } finally { + await operation.close(); + } + } finally { + await session.close(); + } + } finally { + await client.close(); + } + }); + } + } +}); diff --git a/tests/unit/concurrent-fetch.test.ts b/tests/unit/concurrent-fetch.test.ts new file mode 100644 index 00000000..5a24c648 --- /dev/null +++ b/tests/unit/concurrent-fetch.test.ts @@ -0,0 +1,206 @@ +import { expect } from 'chai'; +import sinon from 'sinon'; +import Int64 from 'node-int64'; +import { tableFromArrays, tableToIPC } from 'apache-arrow'; +import OperationStateError, { OperationStateErrorCode } from '../../lib/errors/OperationStateError'; +import { + TFetchOrientation, + TOperationState, + TOperationType, + TSparkRowSetType, + TStatusCode, + TTypeId, +} from '../../thrift/TCLIService_types'; +import ClientContextStub from './.stubs/ClientContextStub'; +import { createOperationForTest } from './.stubs/createOperationForTest'; + +function createFinishedOperation(context: ClientContextStub) { + return createOperationForTest({ + context, + handle: { + operationId: { guid: Buffer.alloc(16), secret: Buffer.alloc(16) }, + operationType: TOperationType.EXECUTE_STATEMENT, + hasResultSet: true, + }, + directResults: { + operationStatus: { + status: { statusCode: TStatusCode.SUCCESS_STATUS }, + operationState: TOperationState.FINISHED_STATE, + }, + }, + }); +} + +describe('Concurrent Thrift result consumption', () => { + for (const phase of ['metadata', 'fetch'] as const) { + it(`rejects a queued fetchChunk after close during an in-flight ${phase} request without further RPCs`, async () => { + const clock = sinon.useFakeTimers(); + const context = new ClientContextStub(); + let release!: () => void; + const gate = new Promise((resolve) => { + release = resolve; + }); + const metadata = sinon.stub(context.driver, 'getResultSetMetadata').callsFake(async () => { + if (phase === 'metadata') { + await gate; + throw new Error('metadata request interrupted by close'); + } + return context.driver.getResultSetMetadataResp; + }); + const fetch = sinon.stub(context.driver, 'fetchResults').callsFake(async () => { + await gate; + return context.driver.fetchResultsResp; + }); + const operation = createFinishedOperation(context); + try { + const pending = Promise.allSettled([ + operation.fetchChunk({ maxRows: 1 }), + operation.fetchChunk({ maxRows: 1 }), + ]); + await clock.tickAsync(0); + const beforeClose = { metadata: metadata.callCount, fetch: fetch.callCount }; + expect(beforeClose).to.deep.equal({ metadata: 1, fetch: phase === 'fetch' ? 1 : 0 }); + await operation.close(); + release(); + await clock.runAllAsync(); + const outcomes = await pending; + expect({ metadata: metadata.callCount, fetch: fetch.callCount }).to.deep.equal(beforeClose); + expect(outcomes.map((outcome) => outcome.status)).to.deep.equal(['rejected', 'rejected']); + if (outcomes[1].status === 'rejected') { + expect(outcomes[1].reason).to.be.instanceOf(OperationStateError); + expect(outcomes[1].reason.errorCode).to.equal(OperationStateErrorCode.Closed); + } + } finally { + release(); + clock.restore(); + } + }); + } + + it('preserves an exact union through the Thrift Arrow pipeline when concurrent reads succeed', async () => { + const clock = sinon.useFakeTimers(); + const context = new ClientContextStub(); + let arrowSchema!: Buffer; + const batches = [ + [0, 1], + [2, 3], + [4, 5], + ].map((ids) => { + const ipc = Buffer.from(tableToIPC(tableFromArrays({ id: Int32Array.from(ids) }))); + const schemaEnd = 8 + ipc.readInt32LE(4); + arrowSchema = ipc.subarray(0, schemaEnd); + return { batch: ipc.subarray(schemaEnd), rowCount: new Int64(ids.length) }; + }); + context.driver.getResultSetMetadataResp = { + status: { statusCode: TStatusCode.SUCCESS_STATUS }, + resultFormat: TSparkRowSetType.ARROW_BASED_SET, + arrowSchema, + schema: { + columns: [ + { columnName: 'id', position: 1, typeDesc: { types: [{ primitiveEntry: { type: TTypeId.INT_TYPE } }] } }, + ], + }, + }; + let release!: () => void; + const gate = new Promise((resolve) => { + release = resolve; + }); + let next = 0; + sinon.stub(context.driver, 'fetchResults').callsFake(async () => { + const index = next++; + await gate; + return { + status: { statusCode: TStatusCode.SUCCESS_STATUS }, + hasMoreRows: index < batches.length - 1, + results: { + startRowOffset: new Int64(index * 2), + rows: [], + arrowBatches: batches[index] ? [batches[index]] : [], + }, + }; + }); + const operation = createFinishedOperation(context); + try { + const pending = Promise.all([operation.fetchAll(), operation.fetchAll()]); + await clock.tickAsync(0); + release(); + await clock.runAllAsync(); + const chunks = await pending; + expect( + chunks + .flat() + .map((row: any) => row.id) + .sort((left, right) => left - right), + ).to.deep.equal([0, 1, 2, 3, 4, 5]); + } finally { + release(); + clock.restore(); + } + }); + + it('sends FETCH_FIRST once when two first reads start without prefetched results', async () => { + const clock = sinon.useFakeTimers(); + const context = new ClientContextStub(); + const orientations: Array = []; + let release!: () => void; + const gate = new Promise((resolve) => { + release = resolve; + }); + sinon.stub(context.driver, 'fetchResults').callsFake(async ({ orientation }) => { + orientations.push(orientation); + await gate; + return { + status: { statusCode: TStatusCode.SUCCESS_STATUS }, + hasMoreRows: false, + results: { + startRowOffset: new Int64(0), + rows: [], + columns: [ + { + i32Val: { + values: orientation === TFetchOrientation.FETCH_FIRST ? [0, 1, 2] : [], + nulls: Buffer.alloc(1), + }, + }, + ], + }, + }; + }); + const operation = createOperationForTest({ + context, + handle: { + operationId: { guid: Buffer.alloc(16), secret: Buffer.alloc(16) }, + operationType: TOperationType.EXECUTE_STATEMENT, + hasResultSet: true, + }, + directResults: { + operationStatus: { + status: { statusCode: TStatusCode.SUCCESS_STATUS }, + operationState: TOperationState.FINISHED_STATE, + }, + resultSetMetadata: { + status: { statusCode: TStatusCode.SUCCESS_STATUS }, + resultFormat: TSparkRowSetType.COLUMN_BASED_SET, + schema: { + columns: [ + { columnName: 'id', position: 1, typeDesc: { types: [{ primitiveEntry: { type: TTypeId.INT_TYPE } }] } }, + ], + }, + }, + }, + }); + try { + const pending = Promise.all([operation.fetchAll(), operation.fetchAll()]); + await clock.tickAsync(0); + release(); + await clock.runAllAsync(); + const chunks = await pending; + expect({ + firstReads: orientations.filter((orientation) => orientation === TFetchOrientation.FETCH_FIRST).length, + ids: chunks.flat().map((row: any) => row.id), + }).to.deep.equal({ firstReads: 1, ids: [0, 1, 2] }); + } finally { + clock.restore(); + } + }); +}); diff --git a/tests/unit/kernel/concurrent-fetch.test.ts b/tests/unit/kernel/concurrent-fetch.test.ts new file mode 100644 index 00000000..3a857bdd --- /dev/null +++ b/tests/unit/kernel/concurrent-fetch.test.ts @@ -0,0 +1,162 @@ +import { expect } from 'chai'; +import sinon from 'sinon'; +import { tableFromArrays, tableToIPC } from 'apache-arrow'; +import DBSQLOperation from '../../../lib/DBSQLOperation'; +import KernelOperationBackend from '../../../lib/kernel/KernelOperationBackend'; +import OperationStateError, { OperationStateErrorCode } from '../../../lib/errors/OperationStateError'; +import ArrowResultConverter from '../../../lib/result/ArrowResultConverter'; +import ResultSlicer from '../../../lib/result/ResultSlicer'; +import { TTypeId } from '../../../thrift/TCLIService_types'; +import ClientContextStub from '../.stubs/ClientContextStub'; +import ResultsProviderStub from '../.stubs/ResultsProviderStub'; + +function createStatement() { + const batches = [ + [0, 1, 2], + [3, 4, 5], + [6, 7, 8], + ].map((ids) => ({ + ipcBytes: Buffer.from(tableToIPC(tableFromArrays({ id: Int32Array.from(ids) }))), + })); + return { + statementId: 'concurrent-fetch', + schema: () => batches[0], + fetchNextBatch: sinon.stub().callsFake(async () => batches.shift() ?? null), + close: sinon.stub().resolves(), + cancel: sinon.stub().resolves(), + }; +} + +function createOperation() { + const context = new ClientContextStub(); + const statement = createStatement(); + const backend = new KernelOperationBackend({ statement, context }); + const operation = new DBSQLOperation({ backend, context }); + return { operation, backend, context, statement }; +} + +describe('Kernel concurrent result consumption through DBSQLOperation', () => { + it('constructs one result pipeline for concurrent first readers', async () => { + const { operation, backend } = createOperation(); + const getResultSlicer = sinon.spy(backend as any, 'getResultSlicer'); + await Promise.all([operation.fetchChunk({ maxRows: 1 }), operation.fetchChunk({ maxRows: 1 })]); + const [first, second] = await Promise.all(getResultSlicer.returnValues); + expect(first).to.equal(second); + }); + + it('releases a failed read so the next waiter observes cleanup before touching the cursor', async () => { + const { operation, statement } = createOperation(); + statement.fetchNextBatch.rejects(new Error('fetch failed')); + const outcomes = await Promise.allSettled([operation.fetchChunk(), operation.fetchChunk()]); + expect(outcomes.map((outcome) => outcome.status)).to.deep.equal(['rejected', 'rejected']); + if (outcomes[0].status === 'rejected' && outcomes[1].status === 'rejected') { + expect(outcomes[0].reason.message).to.equal('fetch failed'); + expect(outcomes[1].reason).to.be.instanceOf(OperationStateError); + expect(outcomes[1].reason.errorCode).to.equal(OperationStateErrorCode.Closed); + } + expect(statement.fetchNextBatch.callCount).to.equal(1); + expect(statement.close.callCount).to.equal(1); + }); + + for (const fetch of ['fetchChunk', 'hasMoreRows'] as const) { + it(`honors close for queued ${fetch} without touching the cursor`, async () => { + const { operation, backend, statement } = createOperation(); + await backend['getResultSlicer'](); + let release!: () => void; + (operation as any).resultConsumption = new Promise((resolve) => { + release = resolve; + }); + const pending = fetch === 'fetchChunk' ? operation.fetchChunk({ maxRows: 100 }) : operation.hasMoreRows(); + await new Promise((resolve) => setImmediate(resolve)); + await operation.close(); + release(); + const [outcome] = await Promise.allSettled([pending]); + if (fetch === 'hasMoreRows') { + expect(outcome).to.deep.equal({ status: 'fulfilled', value: false }); + } else { + expect(outcome.status).to.equal('rejected'); + if (outcome.status === 'rejected') { + expect(outcome.reason).to.be.instanceOf(OperationStateError); + expect(outcome.reason.errorCode).to.equal(OperationStateErrorCode.Closed); + } + } + expect(statement.fetchNextBatch.called).to.equal(false); + }); + } + + it('preserves unread Arrow rows when consumers await different source batches', async () => { + const { operation, backend, context } = createOperation(); + const batches = [ + [0, 1], + [2, 3], + [4, 5], + ].map((ids) => ({ + batches: [Buffer.from(tableToIPC(tableFromArrays({ id: Int32Array.from(ids) })))], + rowCount: ids.length, + })); + const source = new ResultsProviderStub(batches, { batches: [], rowCount: 0 }); + const releases: Array<() => void> = []; + const gates = batches.map(() => new Promise((resolve) => releases.push(resolve))); + const fetchNext = source.fetchNext.bind(source); + let next = 0; + sinon.stub(source, 'fetchNext').callsFake(async () => { + const gate = gates[next++]; + const batch = await fetchNext(); + await gate; + return batch; + }); + const converter = new ArrowResultConverter(context, source, { + schema: { + columns: [ + { columnName: 'id', position: 1, typeDesc: { types: [{ primitiveEntry: { type: TTypeId.INT_TYPE } }] } }, + ], + }, + }); + sinon.stub(backend as any, 'getResultSlicer').resolves(new ResultSlicer(context, converter)); + const pending = Promise.all([ + operation.fetchChunk({ maxRows: 100, disableBuffering: true }), + operation.fetchChunk({ maxRows: 100, disableBuffering: true }), + ]); + await new Promise((resolve) => setImmediate(resolve)); + releases.forEach((release) => release()); + const chunks = await pending; + while (await operation.hasMoreRows()) { + chunks.push(await operation.fetchChunk({ maxRows: 100, disableBuffering: true })); + } + expect(chunks.flat().map((row: any) => row.id)).to.deep.equal([0, 1, 2, 3, 4, 5]); + }); + + for (const initialized of [false, true]) { + for (const limit of [1, 4, 100]) { + it(`returns an exact union with overlapping chunks (initialized=${initialized}, limit=${limit})`, async () => { + const { operation } = createOperation(); + if (initialized) { + await operation.hasMoreRows(); + } + const chunks = await Promise.all([ + operation.fetchChunk({ maxRows: limit }), + operation.fetchChunk({ maxRows: limit }), + ]); + while (await operation.hasMoreRows()) { + chunks.push(await operation.fetchChunk({ maxRows: limit })); + } + const ids = chunks.flat().map((row: any) => row.id); + expect(ids.sort((left, right) => left - right)).to.deep.equal([0, 1, 2, 3, 4, 5, 6, 7, 8]); + }); + } + } + + it('does not lose prefetched rows when hasMore overlaps unbuffered reads', async () => { + const { operation } = createOperation(); + const [, first, second] = await Promise.all([ + operation.hasMoreRows(), + operation.fetchChunk({ maxRows: 1, disableBuffering: true }), + operation.fetchChunk({ maxRows: 1, disableBuffering: true }), + ]); + const chunks = [first, second]; + while (await operation.hasMoreRows()) { + chunks.push(await operation.fetchChunk({ maxRows: 1, disableBuffering: true })); + } + expect(chunks.flat().map((row: any) => row.id)).to.deep.equal([0, 1, 2, 3, 4, 5, 6, 7, 8]); + }); +});