diff --git a/CHANGELOG.md b/CHANGELOG.md index cd6726b5..938983a3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ - The Thrift token federation exchange (`enableTokenFederation`) now goes through the connection's agent, so `proxy`, `checkServerCertificate`, `customCaCert` and `clientCert`/`clientKey` apply to it. Previously it connected directly with Node's default agent, so behind a required proxy the exchange failed and the driver fell back to the original token. (#611) - Kernel backend: a failed status poll while waiting for a statement (a `runAsync: true` statement, or a default execution that outlives the server's inline wait) and a failed result fetch in `hasMoreRows()` now throw the mapped driver error, such as `HiveDriverError: HTTP 500: Internal Server Error` once retries on HTTP 5xx run out, instead of a raw `Error` whose message is the internal `__databricks_error__:{...}` envelope. As in `fetchChunk()`, a failed fetch in `hasMoreRows()` also closes the statement. (#608) +- Honor `cloudFetchConcurrentDownloads` when opening kernel sessions, with a default window of 16 and explicit chunk-setting overrides. Values other than positive safe integers raise `HiveDriverError` at session open. - Reduce result chunking and `fetchAll()` overhead for large query results. - Thrift CloudFetch downloads are now bounded end to end by `retriesTimeout` (default 15 minutes), including every attempt and retry backoff. Previously a stalled download, where the response headers or body never arrived, blocked the fetch forever. A download still running when the deadline expires is aborted and fails with `RetryError` (`TIMEOUT_EXCEEDED`, message `CloudFetch result download failed: Retry timeout exceeded`). (#520) - Thrift connections made through `proxy` now apply `checkServerCertificate`, `customCaCert` and `clientCert`/`clientKey`. Previously the proxied path ignored them and verified against Node's process-wide trust store, so a certificate outside the configured trust set could be accepted. **Compatibility:** proxied connections now verify unless `checkServerCertificate: false` is set; as on direct connections, `NODE_TLS_REJECT_UNAUTHORIZED=0` no longer turns verification off. For an `https` proxy these options also apply to the proxy's own certificate. (#485) diff --git a/CONNECTION_PARAMETERS.md b/CONNECTION_PARAMETERS.md index e0503021..edba6002 100644 --- a/CONNECTION_PARAMETERS.md +++ b/CONNECTION_PARAMETERS.md @@ -130,6 +130,22 @@ still use HTTPS to `host`. Kernel uses `http://host[:port]` for all of its requests, including its own OAuth and telemetry calls. CloudFetch downloads use the presigned URLs returned by the server on both backends. +## CloudFetch download window + +Set `client.getConfig().cloudFetchConcurrentDownloads` before `openSession()` +to tune downloads on either backend (default 16 on the kernel, 10 on Thrift). Kernel sessions snapshot this +value at open; changes apply to subsequent sessions. Thrift reads it on each +fetch. Kernel mode requires a positive safe integer and rejects invalid values +at session open. Values above 256 are clamped by the kernel. + +Before connecting, an unset value reads as 10; it reads as 16 while connected +through the kernel. Explicit assignments, including 10 before connecting, are retained. + +The kernel bounds downloading and queued chunks ahead of consumption, excluding +consumer buffers. This is not a byte limit. An explicit +`cloudfetch_max_chunks_in_memory` in `openSession({ configuration: ... })` wins +case-insensitively. Invalid raw overrides warn and retain the kernel default 16. + ## Results and type rendering | Option | Type | Thrift | Kernel | Default Value | Note | diff --git a/lib/DBSQLClient.ts b/lib/DBSQLClient.ts index 6c5fa2de..efc150d9 100644 --- a/lib/DBSQLClient.ts +++ b/lib/DBSQLClient.ts @@ -195,6 +195,19 @@ export default class DBSQLClient extends EventEmitter implements IDBSQLClient, I constructor(options?: ClientOptions) { super(); this.config = DBSQLClient.getDefaultConfig(); + // Explicit 10 must override the kernel default of 16. + let cloudFetchConcurrentDownloads: number | undefined; + Object.defineProperty(this.config, 'cloudFetchConcurrentDownloads', { + enumerable: true, + configurable: true, + get: () => { + if (cloudFetchConcurrentDownloads !== undefined) return cloudFetchConcurrentDownloads; + return this.backend instanceof KernelBackend ? 16 : 10; + }, + set: (value: number) => { + cloudFetchConcurrentDownloads = value; + }, + }); this.logger = options?.logger ?? DBSQLClient.getDefaultLogger(); this.logger.log(LogLevel.info, 'Created DBSQLClient'); } diff --git a/lib/contracts/IClientContext.ts b/lib/contracts/IClientContext.ts index d12f9e6e..1fbaed39 100644 --- a/lib/contracts/IClientContext.ts +++ b/lib/contracts/IClientContext.ts @@ -20,6 +20,9 @@ export interface ClientConfig { retryDelayMax: number; // in milliseconds useCloudFetch: boolean; + /** Download window (Thrift 10, connected kernel 16). Kernel sessions snapshot it at openSession; + * it bounds downloading and queued chunks ahead of consumption, excluding + * consumer buffers. It is not a byte limit. */ cloudFetchConcurrentDownloads: number; cloudFetchSpeedThresholdMBps: number; diff --git a/lib/kernel/KernelBackend.ts b/lib/kernel/KernelBackend.ts index 51c44413..5c622ea0 100644 --- a/lib/kernel/KernelBackend.ts +++ b/lib/kernel/KernelBackend.ts @@ -152,6 +152,15 @@ export default class KernelBackend implements IBackend { if (request.configuration !== undefined) { sessionOptions.sessionConf = { ...request.configuration }; } + const configuredWindow = this.context.getConfig().cloudFetchConcurrentDownloads; + const downloadWindow = configuredWindow === undefined ? 16 : configuredWindow; + if (!Number.isSafeInteger(downloadWindow) || downloadWindow <= 0) { + throw new HiveDriverError('cloudFetchConcurrentDownloads must be a positive safe integer'); + } + const chunkKey = 'cloudfetch_max_chunks_in_memory'; + if (!Object.keys(sessionOptions.sessionConf ?? {}).some((key) => key.toLowerCase() === chunkKey)) { + sessionOptions.sessionConf = { ...sessionOptions.sessionConf, [chunkKey]: String(downloadWindow) }; + } if (this.context.getConfig().useCloudFetch === false) { sessionOptions.sessionConf = { ...sessionOptions.sessionConf, cloudfetch_enabled: 'false' }; } diff --git a/tests/unit/DBSQLClient.test.ts b/tests/unit/DBSQLClient.test.ts index b8e007ba..497889db 100644 --- a/tests/unit/DBSQLClient.test.ts +++ b/tests/unit/DBSQLClient.test.ts @@ -308,6 +308,30 @@ describe('DBSQLClient.connect', () => { logSpy.restore(); }); + for (const [useKernel, configuredWindow, expectedWindow] of [ + [false, undefined, 10], + [true, undefined, 16], + [true, 10, 10], + ] as const) { + it(`resolves the download window for kernel=${useKernel}, setting=${configuredWindow}`, async () => { + const client = new DBSQLClient(); + expect(client.getConfig().cloudFetchConcurrentDownloads).to.equal(10); + if (configuredWindow !== undefined) { + client.getConfig().cloudFetchConcurrentDownloads = configuredWindow; + } + sinon.stub(KernelNativeLoader, 'getKernelNative').returns({} as any); + sinon.stub(KernelBackend.prototype, 'connect').resolves(); + sinon.stub(KernelBackend.prototype, 'close').resolves(); + try { + await client.connect({ ...connectOptions, telemetryEnabled: false, useKernel } as any); + expect(client.getConfig().cloudFetchConcurrentDownloads).to.equal(expectedWindow); + } finally { + await client.close(); + } + expect(client.getConfig().cloudFetchConcurrentDownloads).to.equal(10); + }); + } + it('useKernel: true routes to KernelBackend and leaves `backend` unset when connect() throws', async () => { const client = new DBSQLClient(); @@ -333,6 +357,7 @@ describe('DBSQLClient.connect', () => { // failed connect, so the next openSession surfaces "not connected" rather // than the KernelBackend's own connect/auth error. expect((client as any).backend).to.equal(undefined); + expect(client.getConfig().cloudFetchConcurrentDownloads).to.equal(10); try { await client.openSession(); diff --git a/tests/unit/kernel/auth-m2m.test.ts b/tests/unit/kernel/auth-m2m.test.ts index 0b30f2ef..6b2490f2 100644 --- a/tests/unit/kernel/auth-m2m.test.ts +++ b/tests/unit/kernel/auth-m2m.test.ts @@ -381,6 +381,7 @@ describe('KernelAuth + KernelBackend — OAuth M2M auth flow', () => { expect(calls).to.have.lengthOf(1); expect(calls[0].method).to.equal('openSession'); expectNativeConnectionOptions(calls[0].args[0], { + sessionConf: { cloudfetch_max_chunks_in_memory: '16' }, hostName: 'example.cloud.databricks.com', httpPath: '/sql/1.0/warehouses/abc', intervalsAsString: true, diff --git a/tests/unit/kernel/auth-u2m.test.ts b/tests/unit/kernel/auth-u2m.test.ts index 690995a2..7a450920 100644 --- a/tests/unit/kernel/auth-u2m.test.ts +++ b/tests/unit/kernel/auth-u2m.test.ts @@ -223,6 +223,7 @@ describe('KernelAuth + KernelBackend — OAuth U2M auth flow', () => { expect(calls).to.have.lengthOf(1); expect(calls[0].method).to.equal('openSession'); expectNativeConnectionOptions(calls[0].args[0], { + sessionConf: { cloudfetch_max_chunks_in_memory: '16' }, hostName: 'example.cloud.databricks.com', httpPath: '/sql/1.0/warehouses/abc', intervalsAsString: true, diff --git a/tests/unit/kernel/execution.test.ts b/tests/unit/kernel/execution.test.ts index a8b86657..9aae43f5 100644 --- a/tests/unit/kernel/execution.test.ts +++ b/tests/unit/kernel/execution.test.ts @@ -557,6 +557,7 @@ describe('KernelBackend', () => { expectNativeConnectionOptions(args, { hostName: 'workspace.example', httpPath: '/sql/1.0/warehouses/xyz', + sessionConf: { cloudfetch_max_chunks_in_memory: '16' }, authMode: 'Pat', token: 'dapi-token', intervalsAsString: true, @@ -753,12 +754,77 @@ describe('KernelBackend', () => { await backend.openSession({ configuration }); const conf = (binding.openSessionStub.firstCall.args[0] as { sessionConf?: Record }).sessionConf; expect(conf?.cloudfetch_enabled).to.equal(useCloudFetch === false ? 'false' : undefined); + expect(conf?.cloudfetch_max_chunks_in_memory).to.equal('16'); expect(conf?.can_cloud_download).to.equal(undefined); expect(conf?.TIMEZONE).to.equal('UTC'); expect(configuration).to.deep.equal({ TIMEZONE: 'UTC' }); }); }); + for (const [window, override] of [ + [1, undefined], + [4, undefined], + [10, undefined], + [1000, undefined], + [4, 'cloudfetch_max_chunks_in_memory'], + [4, 'CLOUDFETCH_MAX_CHUNKS_IN_MEMORY'], + ] as const) { + it(`openSession() maps download window ${window}, override=${override}`, async () => { + const binding = makeBinding(new FakeNativeConnection()); + const context = makeContext(undefined, { cloudFetchConcurrentDownloads: window }); + const backend = new KernelBackend({ context, nativeBinding: binding }); + await backend.connect({ host: 'h', path: '/p', token: 't' } as ConnectionOptions); + const configuration = override ? { [override]: '2' } : {}; + await backend.openSession({ configuration }); + const options = binding.openSessionStub.firstCall.args[0] as { sessionConf: Record }; + expect(options.sessionConf).to.deep.equal( + override + ? { [override]: '2' } + : { + cloudfetch_max_chunks_in_memory: String(window), + }, + ); + expect(configuration).to.deep.equal(override ? { [override]: '2' } : {}); + }); + } + + for (const window of [0, -1, 1.5, NaN, Infinity, Number.MAX_SAFE_INTEGER + 1, null as unknown as number]) { + it(`openSession() rejects invalid download window ${window}`, async () => { + const binding = makeBinding(new FakeNativeConnection()); + const backend = new KernelBackend({ + context: makeContext(undefined, { cloudFetchConcurrentDownloads: window }), + nativeBinding: binding, + }); + await backend.connect({ host: 'h', path: '/p', token: 't' } as ConnectionOptions); + try { + await backend.openSession({}); + expect.fail('invalid window accepted'); + } catch (error) { + expect(error).to.be.instanceOf(HiveDriverError); + expect((error as Error).message).to.include('positive safe integer'); + } + expect(binding.openSessionStub.called).to.equal(false); + }); + } + + it('snapshots the download window separately for each opened session', async () => { + const binding = makeBinding(new FakeNativeConnection()); + const context = makeContext(undefined, { cloudFetchConcurrentDownloads: 1 }); + const backend = new KernelBackend({ context, nativeBinding: binding }); + await backend.connect({ host: 'h', path: '/p', token: 't' } as ConnectionOptions); + await backend.openSession({}); + context.getConfig().cloudFetchConcurrentDownloads = 4; + await backend.openSession({}); + expect(binding.openSessionStub.firstCall.args[0]).to.have.nested.property( + 'sessionConf.cloudfetch_max_chunks_in_memory', + '1', + ); + expect(binding.openSessionStub.secondCall.args[0]).to.have.nested.property( + 'sessionConf.cloudfetch_max_chunks_in_memory', + '4', + ); + }); + ['can_cloud_download', 'CLOUDFETCH_ENABLED', 'cloudfetch_enabled'].forEach((key) => { it(`openSession() forwards the client disable with ${key}`, async () => { const binding = makeBinding(new FakeNativeConnection()); @@ -770,7 +836,12 @@ describe('KernelBackend', () => { const configuration = { [key]: 'true', TIMEZONE: 'UTC' }; await backend.openSession({ configuration }); const conf = (binding.openSessionStub.firstCall.args[0] as { sessionConf?: Record }).sessionConf; - expect(conf).to.deep.equal({ [key]: 'true', TIMEZONE: 'UTC', cloudfetch_enabled: 'false' }); + expect(conf).to.deep.equal({ + [key]: 'true', + TIMEZONE: 'UTC', + cloudfetch_enabled: 'false', + cloudfetch_max_chunks_in_memory: '16', + }); expect(configuration).to.deep.equal({ [key]: 'true', TIMEZONE: 'UTC' }); }); }); @@ -847,7 +918,7 @@ describe('KernelBackend', () => { token: 't', catalog: 'main', schema: 'default', - sessionConf: { 'spark.sql.execution.arrow.enabled': 'true' }, + sessionConf: { 'spark.sql.execution.arrow.enabled': 'true', cloudfetch_max_chunks_in_memory: '16' }, }); // And the SQL still threads through executeStatement (now with no