Skip to content
Open
3 changes: 3 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,9 @@

## Unreleased

- Serialize concurrent fetches on one operation across the Thrift and kernel backends so overlapping `fetchAll()`, `fetchChunk()` and `hasMoreRows()` calls return every row exactly once with inline Arrow and CloudFetch when all reads succeed. After a failed Thrift read, other readers may continue past that read's rows, matching existing retry-after-error behavior; the kernel closes the statement on a failed fetch. Queued readers honor close/cancel state. (#622)
- Concurrent readers on one operation split the result stream with no call-order guarantee. (#622)
- A concurrent `hasMoreRows()` waits for an in-flight fetch before checking for more rows. (#622)
- The Thrift token federation exchange (`enableTokenFederation`) now goes through the connection's agent, so `proxy`, `checkServerCertificate`, `customCaCert` and `clientCert`/`clientKey` apply to it. Previously it connected directly with Node's default agent, so behind a required proxy the exchange failed and the driver fell back to the original token. (#611)
- Kernel backend: a failed status poll while waiting for a statement (a `runAsync: true` statement, or a default execution that outlives the server's inline wait) and a failed result fetch in `hasMoreRows()` now throw the mapped driver error, such as `HiveDriverError: HTTP 500: Internal Server Error` once retries on HTTP 5xx run out, instead of a raw `Error` whose message is the internal `__databricks_error__:{...}` envelope. As in `fetchChunk()`, a failed fetch in `hasMoreRows()` also closes the statement. (#608)
- Honor `cloudFetchConcurrentDownloads` when opening kernel sessions, with a default window of 16 and explicit chunk-setting overrides. Values other than positive safe integers raise `HiveDriverError` at session open.
Expand Down
44 changes: 32 additions & 12 deletions lib/DBSQLOperation.ts
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,8 @@ export default class DBSQLOperation implements IOperation {

private metadata?: ResultMetadata;

private resultConsumption: Promise<void> = Promise.resolve();

private startTime: number = Date.now();

private pollCount: number = 0;
Expand Down Expand Up @@ -156,19 +158,23 @@ export default class DBSQLOperation implements IOperation {
}

await this.waitUntilReadyThroughBackend(options);
await this.failIfClosed();
return this.withResultLock(async () => {
await this.failIfClosed();

const defaultMaxRows = this.context.getConfig().fetchChunkDefaultMaxRows;
const limit = options?.maxRows ?? defaultMaxRows;
const result = await this.backend.fetchChunk({
limit,
disableBuffering: options?.disableBuffering,
isClosed: () => this.closed || this.cancelled,
});
await this.failIfClosed();
const defaultMaxRows = this.context.getConfig().fetchChunkDefaultMaxRows;
const limit = options?.maxRows ?? defaultMaxRows;
const result = await this.backend.fetchChunk({
limit,
disableBuffering: options?.disableBuffering,
isClosed: () => this.closed || this.cancelled,
});
await this.failIfClosed();

this.context.getLogger().log(LogLevel.debug, `Fetched chunk of size: ${limit} from operation with id: ${this.id}`);
return result;
this.context
.getLogger()
.log(LogLevel.debug, `Fetched chunk of size: ${limit} from operation with id: ${this.id}`);
return result;
});
}

/**
Expand Down Expand Up @@ -291,7 +297,12 @@ export default class DBSQLOperation implements IOperation {
}

// If we fetched all the data from server - check if there's anything buffered in result handler
return this.backend.hasMore();
return this.withResultLock(async () => {
if (this.closed || this.cancelled) {
return false;
}
return this.backend.hasMore();
});
});
}

Expand Down Expand Up @@ -361,6 +372,15 @@ export default class DBSQLOperation implements IOperation {
});
}

private withResultLock<T>(consume: () => Promise<T>): Promise<T> {
const result = this.resultConsumption.then(consume);
Comment thread
vuanhphung marked this conversation as resolved.
Resolved
Comment thread
vuanhphung marked this conversation as resolved.
Resolved
this.resultConsumption = result.then(
() => undefined,
() => undefined,
);
return result;
}

/**
* Wrap a public IOperation method so any thrown error is captured as an
* error telemetry event before being rethrown to the caller. Telemetry
Expand Down
68 changes: 68 additions & 0 deletions tests/e2e/kernel/concurrent-fetch-e2e.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,68 @@
import { expect } from 'chai';
import { DBSQLClient } from '../../../lib';
import DBSQLOperation from '../../../lib/DBSQLOperation';
import { LogLevel } from '../../../lib/contracts/IDBSQLLogger';
import { ResultFormat } from '../../../lib/contracts/ResultMetadata';
import { tryGetKernelNative } from '../../../lib/kernel/KernelNativeLoader';
import config from '../utils/config';

describe('Concurrent result fetching', () => {
Comment thread
peco-review-bot[bot] marked this conversation as resolved.
Resolved
for (const useKernel of [false, true]) {
for (const { cloudFetch, options } of [
{ cloudFetch: false, options: {} },
{ cloudFetch: false, options: { maxRows: 17 } },
{ cloudFetch: true, options: {} },
]) {
const delivery = cloudFetch ? 'CloudFetch' : `inline, ${JSON.stringify(options)}`;
const test = useKernel && !tryGetKernelNative() ? it.skip : it;
test(`returns each id once across overlapping fetchAll calls (${
useKernel ? 'kernel' : 'thrift'
}, ${delivery})`, async () => {
const messages: string[] = [];
const client = new DBSQLClient({
logger: {
log(level, message) {
if (level === LogLevel.info) messages.push(message);
},
},
});
Comment thread
peco-review-bot[bot] marked this conversation as resolved.
Resolved
client.getConfig().useCloudFetch = useKernel || cloudFetch;
try {
await client.connect({ host: config.host, path: config.path, token: config.token, useKernel } as any);
Comment thread
peco-review-bot[bot] marked this conversation as resolved.
Resolved
const session = await client.openSession();
try {
const rowCount = options.maxRows ? 1000 : 100000;
const query = cloudFetch
? `SELECT id, concat(
sha2(CAST(id AS STRING), 512), sha2(CAST(id + 1 AS STRING), 512),
sha2(CAST(id + 2 AS STRING), 512), sha2(CAST(id + 3 AS STRING), 512)
) AS payload FROM range(0, 100000) AS T(id) ORDER BY id`
: `SELECT id FROM range(0, ${rowCount}) AS T(id) ORDER BY id`;
const operation = await session.executeStatement(query);
try {
const chunks = await Promise.all([operation.fetchAll(options), operation.fetchAll(options)]);
const ids = chunks.flat().map((row: any) => Number(row.id));
expect(ids.sort((left, right) => left - right)).to.deep.equal(
Array.from({ length: rowCount }, (unused, index) => index),
);
expect(await operation.hasMoreRows()).to.equal(false);
if (useKernel) {
const reader = cloudFetch ? 'CloudFetch' : 'inline Arrow';
expect(messages.some((message) => message.includes(`Using ${reader} reader:`))).to.equal(true);
} else if (cloudFetch) {
const metadata = await (operation as DBSQLOperation).getResultMetadata();
expect(metadata.resultFormat).to.equal(ResultFormat.UrlBased);
}
} finally {
await operation.close();
}
} finally {
await session.close();
}
} finally {
await client.close();
}
});
}
}
});
206 changes: 206 additions & 0 deletions tests/unit/concurrent-fetch.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,206 @@
import { expect } from 'chai';
import sinon from 'sinon';
import Int64 from 'node-int64';
import { tableFromArrays, tableToIPC } from 'apache-arrow';
import OperationStateError, { OperationStateErrorCode } from '../../lib/errors/OperationStateError';
import {
TFetchOrientation,
TOperationState,
TOperationType,
TSparkRowSetType,
TStatusCode,
TTypeId,
} from '../../thrift/TCLIService_types';
import ClientContextStub from './.stubs/ClientContextStub';
import { createOperationForTest } from './.stubs/createOperationForTest';

function createFinishedOperation(context: ClientContextStub) {
return createOperationForTest({
context,
handle: {
operationId: { guid: Buffer.alloc(16), secret: Buffer.alloc(16) },
operationType: TOperationType.EXECUTE_STATEMENT,
hasResultSet: true,
},
directResults: {
operationStatus: {
status: { statusCode: TStatusCode.SUCCESS_STATUS },
operationState: TOperationState.FINISHED_STATE,
},
},
});
}

describe('Concurrent Thrift result consumption', () => {
for (const phase of ['metadata', 'fetch'] as const) {
it(`rejects a queued fetchChunk after close during an in-flight ${phase} request without further RPCs`, async () => {
const clock = sinon.useFakeTimers();
const context = new ClientContextStub();
let release!: () => void;
const gate = new Promise<void>((resolve) => {
release = resolve;
});
const metadata = sinon.stub(context.driver, 'getResultSetMetadata').callsFake(async () => {
if (phase === 'metadata') {
await gate;
throw new Error('metadata request interrupted by close');
}
return context.driver.getResultSetMetadataResp;
});
const fetch = sinon.stub(context.driver, 'fetchResults').callsFake(async () => {
await gate;
return context.driver.fetchResultsResp;
});
const operation = createFinishedOperation(context);
try {
const pending = Promise.allSettled([
operation.fetchChunk({ maxRows: 1 }),
operation.fetchChunk({ maxRows: 1 }),
]);
await clock.tickAsync(0);
const beforeClose = { metadata: metadata.callCount, fetch: fetch.callCount };
expect(beforeClose).to.deep.equal({ metadata: 1, fetch: phase === 'fetch' ? 1 : 0 });
await operation.close();
release();
await clock.runAllAsync();
const outcomes = await pending;
expect({ metadata: metadata.callCount, fetch: fetch.callCount }).to.deep.equal(beforeClose);
expect(outcomes.map((outcome) => outcome.status)).to.deep.equal(['rejected', 'rejected']);
if (outcomes[1].status === 'rejected') {
expect(outcomes[1].reason).to.be.instanceOf(OperationStateError);
expect(outcomes[1].reason.errorCode).to.equal(OperationStateErrorCode.Closed);
}
} finally {
release();
clock.restore();
}
});
}

it('preserves an exact union through the Thrift Arrow pipeline when concurrent reads succeed', async () => {
const clock = sinon.useFakeTimers();
const context = new ClientContextStub();
let arrowSchema!: Buffer;
const batches = [
[0, 1],
[2, 3],
[4, 5],
].map((ids) => {
const ipc = Buffer.from(tableToIPC(tableFromArrays({ id: Int32Array.from(ids) })));
const schemaEnd = 8 + ipc.readInt32LE(4);
arrowSchema = ipc.subarray(0, schemaEnd);
return { batch: ipc.subarray(schemaEnd), rowCount: new Int64(ids.length) };
});
context.driver.getResultSetMetadataResp = {
status: { statusCode: TStatusCode.SUCCESS_STATUS },
resultFormat: TSparkRowSetType.ARROW_BASED_SET,
arrowSchema,
schema: {
columns: [
{ columnName: 'id', position: 1, typeDesc: { types: [{ primitiveEntry: { type: TTypeId.INT_TYPE } }] } },
],
},
};
let release!: () => void;
const gate = new Promise<void>((resolve) => {
release = resolve;
});
let next = 0;
sinon.stub(context.driver, 'fetchResults').callsFake(async () => {
const index = next++;
await gate;
return {
status: { statusCode: TStatusCode.SUCCESS_STATUS },
hasMoreRows: index < batches.length - 1,
results: {
startRowOffset: new Int64(index * 2),
rows: [],
arrowBatches: batches[index] ? [batches[index]] : [],
},
};
});
const operation = createFinishedOperation(context);
try {
const pending = Promise.all([operation.fetchAll(), operation.fetchAll()]);
await clock.tickAsync(0);
release();
await clock.runAllAsync();
const chunks = await pending;
expect(
chunks
.flat()
.map((row: any) => row.id)
.sort((left, right) => left - right),
).to.deep.equal([0, 1, 2, 3, 4, 5]);
} finally {
release();
clock.restore();
}
});

it('sends FETCH_FIRST once when two first reads start without prefetched results', async () => {
const clock = sinon.useFakeTimers();
const context = new ClientContextStub();
const orientations: Array<TFetchOrientation | undefined> = [];
let release!: () => void;
const gate = new Promise<void>((resolve) => {
release = resolve;
});
sinon.stub(context.driver, 'fetchResults').callsFake(async ({ orientation }) => {
orientations.push(orientation);
await gate;
return {
status: { statusCode: TStatusCode.SUCCESS_STATUS },
hasMoreRows: false,
results: {
startRowOffset: new Int64(0),
rows: [],
columns: [
{
i32Val: {
values: orientation === TFetchOrientation.FETCH_FIRST ? [0, 1, 2] : [],
nulls: Buffer.alloc(1),
},
},
],
},
};
});
const operation = createOperationForTest({
context,
handle: {
operationId: { guid: Buffer.alloc(16), secret: Buffer.alloc(16) },
operationType: TOperationType.EXECUTE_STATEMENT,
hasResultSet: true,
},
directResults: {
operationStatus: {
status: { statusCode: TStatusCode.SUCCESS_STATUS },
operationState: TOperationState.FINISHED_STATE,
},
resultSetMetadata: {
status: { statusCode: TStatusCode.SUCCESS_STATUS },
resultFormat: TSparkRowSetType.COLUMN_BASED_SET,
schema: {
columns: [
{ columnName: 'id', position: 1, typeDesc: { types: [{ primitiveEntry: { type: TTypeId.INT_TYPE } }] } },
],
},
},
},
});
try {
const pending = Promise.all([operation.fetchAll(), operation.fetchAll()]);
await clock.tickAsync(0);
release();
await clock.runAllAsync();
const chunks = await pending;
expect({
firstReads: orientations.filter((orientation) => orientation === TFetchOrientation.FETCH_FIRST).length,
ids: chunks.flat().map((row: any) => row.id),
}).to.deep.equal({ firstReads: 1, ids: [0, 1, 2] });
} finally {
clock.restore();
}
});
});
Loading
Loading