diff --git a/CHANGELOG.md b/CHANGELOG.md index 938983a3..c92862ec 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ## Unreleased +- Thrift now honors the per-statement `rowLimit` server-side cap, matching the kernel backend. Previously ignored values now cap results: `0` or values that truncate to zero return no rows (unlike JDBC's unlimited `setMaxRows(0)`), while out-of-range integers (valid Thrift range: `0..2147483646`) and infinities fail `executeStatement`. Runtime coercion matches the kernel: numeric strings become numbers, and `null`/`NaN` produce a zero-row cap; `maxRows` fetch sizing is unchanged. - The Thrift token federation exchange (`enableTokenFederation`) now goes through the connection's agent, so `proxy`, `checkServerCertificate`, `customCaCert` and `clientCert`/`clientKey` apply to it. Previously it connected directly with Node's default agent, so behind a required proxy the exchange failed and the driver fell back to the original token. (#611) - Kernel backend: a failed status poll while waiting for a statement (a `runAsync: true` statement, or a default execution that outlives the server's inline wait) and a failed result fetch in `hasMoreRows()` now throw the mapped driver error, such as `HiveDriverError: HTTP 500: Internal Server Error` once retries on HTTP 5xx run out, instead of a raw `Error` whose message is the internal `__databricks_error__:{...}` envelope. As in `fetchChunk()`, a failed fetch in `hasMoreRows()` also closes the statement. (#608) - Honor `cloudFetchConcurrentDownloads` when opening kernel sessions, with a default window of 16 and explicit chunk-setting overrides. Values other than positive safe integers raise `HiveDriverError` at session open. diff --git a/CONNECTION_PARAMETERS.md b/CONNECTION_PARAMETERS.md index edba6002..1c141923 100644 --- a/CONNECTION_PARAMETERS.md +++ b/CONNECTION_PARAMETERS.md @@ -217,9 +217,12 @@ telemetry implementation, so only the options explicitly forwarded by | `namedParameters` | `Record` | ✅ | ✅ | — | Named SQL parameters. Cannot be combined with non-empty `ordinalParameters`. Thrift requires a protocol that supports parameterized queries. | | `ordinalParameters` | `Array` | ✅ | ✅ | — | Positional SQL parameters. Cannot be combined with non-empty `namedParameters`. | | `queryTags` | `Record` | ✅ | ✅ | — | Serialized into the per-statement `query_tags` conf overlay on both. | -| `rowLimit` | `number` | ❌ | ✅ | — | Kernel-only server-side row cap. Thrift logs and ignores it. | +| `rowLimit` | `number` | ✅ | ✅ | — | Per-statement server-side row cap, separate from `maxRows` fetch sizing. Omitted by default (no driver-imposed cap). | | `statementConf` | `Record` | ❌ | ✅ | — | Kernel-only per-statement Spark conf overlay. Structured `queryTags` overwrite its `query_tags` key when non-empty. | +Thrift sends `resultRowLimit` without a protocol-version gate, as JDBC and Python do. +Servers that do not support this field ignore it. + ## Metadata-operation request parameters The public session metadata methods also accept request objects. These are not @@ -249,6 +252,6 @@ volume operations. Kernel-only capabilities include connection-pool sizing, proxy bypass hosts, independent TLS hostname verification, optional U2M disk caching, and -per-statement `rowLimit`/`statementConf`. +per-statement `statementConf`. The Kernel path remains internal and unstable; its behavior may change. diff --git a/lib/contracts/IDBSQLSession.ts b/lib/contracts/IDBSQLSession.ts index 12d7d6c2..45b82ca6 100644 --- a/lib/contracts/IDBSQLSession.ts +++ b/lib/contracts/IDBSQLSession.ts @@ -48,9 +48,13 @@ export type ExecuteStatementOptions = { */ queryTags?: Record; /** - * kernel-only: server-side row cap for this statement (kernel `row_limit`). The - * Thrift backend has no execute-time server cap, so this is a no-op there; - * use `maxRows` for the cross-backend client-side fetch limit. + * Per-statement server-side row cap on both backends. Omitted by default (no + * driver-imposed cap). Distinct from `maxRows`, which controls fetch-page size. + * On Thrift, values truncate to integers in 0..2147483646; zero returns no rows + * (unlike JDBC's unlimited setMaxRows(0)). Out-of-range values and infinities + * fail executeStatement; Thrift previously ignored this option. Runtime + * coercion matches the kernel: numeric strings become numbers; null and NaN + * produce a zero-row cap, as do values that truncate to zero. */ rowLimit?: number; /** diff --git a/lib/kernel/KernelSessionBackend.ts b/lib/kernel/KernelSessionBackend.ts index dd55dd4c..3b2b949a 100644 --- a/lib/kernel/KernelSessionBackend.ts +++ b/lib/kernel/KernelSessionBackend.ts @@ -143,7 +143,7 @@ export default class KernelSessionBackend implements ISessionBackend { * - `queryTimeout` → NO-OP on kernel (SQL Warehouses use `STATEMENT_TIMEOUT`); * never forwarded to the kernel and never applied as a client-side * deadline — see the note in `executeStatement`; - * - `rowLimit` → `rowLimit` (kernel-only server-side row cap); + * - `rowLimit` → `rowLimit` (server-side row cap); * - `queryTags` → serialised into the conf overlay's reserved * `query_tags` key (the same wire shape Thrift's `serializeQueryTags` * produces), merged with any explicit `statementConf`. diff --git a/lib/thrift-backend/ThriftSessionBackend.ts b/lib/thrift-backend/ThriftSessionBackend.ts index 49ec539a..8ed90683 100644 --- a/lib/thrift-backend/ThriftSessionBackend.ts +++ b/lib/thrift-backend/ThriftSessionBackend.ts @@ -184,17 +184,13 @@ export default class ThriftSessionBackend implements ISessionBackend { const driver = await this.context.getDriver(); const clientConfig = this.context.getConfig(); - // `rowLimit` / `statementConf` are kernel-backend (kernel) options with no - // Thrift wire equivalent. Surface a debug breadcrumb rather than dropping - // them silently, so a caller that set them on the Thrift path has signal. - if (options.rowLimit !== undefined || options.statementConf !== undefined) { + if (options.statementConf !== undefined) { this.context .getLogger() .log( LogLevel.warn, - 'ThriftSessionBackend.executeStatement: rowLimit / statementConf are kernel-backend (useKernel) ' + - 'options with no Thrift wire equivalent — they are IGNORED on the Thrift path (e.g. rowLimit ' + - 'will not cap the result set). Use the kernel backend (useKernel) to honour them.', + 'ThriftSessionBackend.executeStatement: statementConf is a kernel-backend (useKernel) option ' + + 'and is IGNORED on the Thrift path. Use the kernel backend (useKernel) to honour it.', ); } @@ -202,6 +198,7 @@ export default class ThriftSessionBackend implements ISessionBackend { sessionHandle: this.sessionHandle, statement, queryTimeout: options.queryTimeout ? numberToInt64(options.queryTimeout) : undefined, + resultRowLimit: options.rowLimit !== undefined ? numberToInt64(Number(options.rowLimit)) : undefined, runAsync: true, ...getDirectResultsOptions(options.maxRows, clientConfig), ...getArrowOptions(clientConfig, this.serverProtocolVersion), diff --git a/tests/unit/DBSQLSession.test.ts b/tests/unit/DBSQLSession.test.ts index 00498c54..42ac1c1b 100644 --- a/tests/unit/DBSQLSession.test.ts +++ b/tests/unit/DBSQLSession.test.ts @@ -8,6 +8,7 @@ import DBSQLOperation from '../../lib/DBSQLOperation'; import { PrimaryKeysRequest } from '../../lib/contracts/IDBSQLSession'; import ISessionBackend from '../../lib/contracts/ISessionBackend'; import ParameterError from '../../lib/errors/ParameterError'; +import { LogLevel } from '../../lib/contracts/IDBSQLLogger'; import { DBSQLParameter } from '../../lib/DBSQLParameter'; import { TSessionHandle, TProtocolVersion, TOperationState, TStatusCode } from '../../thrift/TCLIService_types'; import ClientContextStub from './.stubs/ClientContextStub'; @@ -80,6 +81,43 @@ describe('DBSQLSession', () => { expect(result).instanceOf(DBSQLOperation); }); + for (const rowLimit of [7, 0, undefined, '100', null]) { + it(`should ${rowLimit === undefined ? 'omit an unset' : `forward ${rowLimit} as the`} rowLimit`, async () => { + const context = new ClientContextStub(); + const driver = sinon.spy(context.driver); + const logger = sinon.spy(context.logger, 'log'); + const session = createSessionForTest({ handle: sessionHandleStub, context }); + + await session.executeStatement('SELECT * FROM table', { + rowLimit: rowLimit as number | undefined, + maxRows: 100, + }); + + expect(driver.executeStatement.callCount).to.equal(1); + const request = driver.executeStatement.firstCall.args[0]; + if (rowLimit === undefined) { + expect(request.resultRowLimit).to.not.exist; + } else { + expect(request.resultRowLimit).to.be.instanceOf(Int64); + expect(request.resultRowLimit?.toNumber()).to.equal(Number(rowLimit)); + } + expect(request.getDirectResults?.maxRows.toNumber()).to.equal(100); + expect(logger.calledWith(LogLevel.warn)).to.be.false; + }); + } + + it('should warn only about statementConf when rowLimit is also set', async () => { + const context = new ClientContextStub(); + const logger = sinon.spy(context.logger, 'log'); + const session = createSessionForTest({ handle: sessionHandleStub, context }); + + await session.executeStatement('SELECT 1', { rowLimit: 7, statementConf: {} }); + + const warnings = logger.getCalls().filter((call) => call.args[0] === LogLevel.warn); + expect(warnings).to.have.lengthOf(1); + expect(warnings[0].args[1]).to.include('statementConf').and.not.include('rowLimit'); + }); + describe('Arrow support', () => { it('should not use Arrow if disabled in options', async () => { const session = createSessionForTest({