From 0970af698244e5eebe22b26e3155ecaff18f0d1b Mon Sep 17 00:00:00 2001 From: Vu Anh Phung Date: Wed, 11 Nov 2026 23:15:17 +0000 Subject: [PATCH 1/4] fix: treat a failed close of a finished operation as best effort Once the backend has reported the statement finished, a failed close request (e.g. a 404 on CloseOperation / CloseStatement) is logged as a warning, emitted as error telemetry and reported in the returned status, and the operation is closed locally. Before the statement finishes, close failures still throw, since closing is what stops a running statement. Fixes #480 Signed-off-by: Vu Anh Phung Co-authored-by: Isaac --- CHANGELOG.md | 1 + lib/DBSQLOperation.ts | 28 +++- tests/unit/DBSQLOperation.test.ts | 121 +++++++++++++++++- tests/unit/DBSQLSession.test.ts | 17 +++ tests/unit/kernel/operation-lifecycle.test.ts | 59 +++++++++ 5 files changed, 223 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 77d44b38..9bc027cc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ - CloudFetch retry-exhaustion errors now identify the download and the last failure: the `RetryError` message reads `CloudFetch result download failed: (last error: )`. `` is `HTTP : ` (or `HTTP ` when the status text is empty) for HTTP failures, and `network error `, `network error ()` (e.g. `aborted`, when there is no `code`) or a bare `network error` for network failures (the raw network-error message, which can contain the presigned URL, is not copied). If no cause can be determined, the ` (last error: …)` suffix is omitted. The `RetryError` instance, `errorCode` and `payload` (including the node-fetch `Request` / `Response`) are unchanged. (#471) - Report untyped NULL columns as STRING in kernel result metadata, matching Thrift. (#470) +- `IOperation.close()` no longer throws when the close request fails after the operation has finished (for example, once its results were read): the failure is logged as a warning and reported in the returned status's `info`, and the operation is still closed locally. Closing an operation that hasn't finished still throws on failure. (#480) - Reject non-scalar query parameter values at bind time with a `ParameterError` on both the Thrift and kernel backends, instead of stringifying them (which could leave a Thrift operation stuck before FINISHED). Newly rejected on the Thrift backend: arrays, plain objects, boxed primitives (e.g. `new String('x')`, `new Number(1)`), functions, symbols, and a `DBSQLParameter` that wraps another `DBSQLParameter`. Primitives, `Date`, `Int64`, and `DBSQLParameter` are still accepted. - `DBSQLLogger`: a `filepath` naming an existing directory now writes to `databricks-sql-nodejs.log` inside it, instead of silently producing no logs. (#565) - `DBSQLLogger`: if the log file can't be opened (including when its directory can't be created, which used to throw from the constructor), the logger now logs to the console only, leaves `transports.file` undefined, and emits a process warning with code `DBSQL_LOG_FILE_UNAVAILABLE`. Previously an unwritable file could silently stall all logging, including the console. (#565) diff --git a/lib/DBSQLOperation.ts b/lib/DBSQLOperation.ts index 1ec1922c..42a6ecd4 100644 --- a/lib/DBSQLOperation.ts +++ b/lib/DBSQLOperation.ts @@ -55,6 +55,10 @@ export default class DBSQLOperation implements IOperation { private cancelled: boolean = false; + // Set once the backend reports the statement finished successfully, after + // which a failed close cannot leave it running server-side. + private executionFinished: boolean = false; + public readonly _data?: unknown; private metadata?: ResultMetadata; @@ -212,7 +216,8 @@ export default class DBSQLOperation implements IOperation { } /** - * Closes operation + * Closes operation. Once the operation has finished, a failed close request + * is logged and reported in the returned status's `info` instead of thrown. * @throws {StatusError} */ public async close(): Promise { @@ -223,7 +228,25 @@ export default class DBSQLOperation implements IOperation { if (this.closed || this.cancelled) { return Status.success(); } - const result = await this.backend.close(); + let result: Status; + try { + result = await this.backend.close(); + } catch (error: any) { + // Until the statement finished, closing it is what stops it server-side, + // so the failure is the caller's to handle (and the close can be retried). + if (!this.executionFinished) { + throw error; + } + // Afterwards the close only releases server-side results, which closing + // the session also does: report the failure and finish the teardown. + // `StatusError` doesn't extend `Error`, so read `message` structurally. + const message = typeof error?.message === 'string' ? error.message : String(error); + this.context + .getLogger() + .log(LogLevel.warn, `Failed to close finished operation ${this.id}; it is closed locally. Cause: ${message}`); + this.emitErrorEvent(error); + result = Status.success([`Failed to close operation: ${message}`]); + } this.closed = true; // Emit statement.complete telemetry event @@ -367,6 +390,7 @@ export default class DBSQLOperation implements IOperation { }; try { await this.backend.waitUntilReady(backendOptions); + this.executionFinished = true; } catch (err) { // Reflect terminal states back into facade flags so subsequent calls // short-circuit via failIfClosed(). diff --git a/tests/unit/DBSQLOperation.test.ts b/tests/unit/DBSQLOperation.test.ts index f9195105..9cfa57a1 100644 --- a/tests/unit/DBSQLOperation.test.ts +++ b/tests/unit/DBSQLOperation.test.ts @@ -11,7 +11,8 @@ import { TTypeId, } from '../../thrift/TCLIService_types'; import StatusError from '../../lib/errors/StatusError'; -import OperationStateError from '../../lib/errors/OperationStateError'; +import OperationStateError, { OperationStateErrorCode } from '../../lib/errors/OperationStateError'; +import { LogLevel } from '../../lib/contracts/IDBSQLLogger'; import HiveDriverError from '../../lib/errors/HiveDriverError'; import JsonResultHandler from '../../lib/result/JsonResultHandler'; import ArrowResultConverter from '../../lib/result/ArrowResultConverter'; @@ -416,6 +417,108 @@ describe('DBSQLOperation', () => { await expectFailure(() => operation.finished()); await expectFailure(() => operation.getSchema()); }); + + describe('after the operation finished', () => { + // How the HTTP layer reports the 404 a teardown CloseOperation can get. + const closeFailure = () => new Error('Received a response with a bad HTTP status: 404'); + + function finishedOperationWithFailingClose(failure: () => Promise) { + const context = new ClientContextStub(); + const log = sinon.spy(context.logger, 'log'); + const closeOperation = sinon.stub(context.driver, 'closeOperation').callsFake(failure as any); + // One page of rows, then an empty last page. + sinon + .stub(context.driver, 'fetchResults') + .onFirstCall() + .resolves(context.driver.fetchResultsResp) + .resolves({ + status: { statusCode: TStatusCode.SUCCESS_STATUS }, + hasMoreRows: false, + results: { startRowOffset: new Int64(3), rows: [], columns: [] }, + }); + const operation = createOperationForTest({ handle: operationHandleStub({ hasResultSet: true }), context }); + const onClose = sinon.stub(); + operation.onClose = onClose; + return { context, log, closeOperation, operation, onClose }; + } + + it('should complete teardown when CloseOperation fails after results were consumed', async () => { + const { log, closeOperation, operation, onClose } = finishedOperationWithFailingClose(() => + Promise.reject(closeFailure()), + ); + + expect(await operation.fetchAll()).to.have.lengthOf(3); + + const status = await operation.close(); + + expect(closeOperation.calledOnce).to.be.true; + expect(status.isSuccess).to.be.true; + expect(status.info.join()).to.contain('404'); + expect(operation['closed']).to.be.true; + expect(onClose.calledOnce).to.be.true; + expect(log.calledWith(LogLevel.warn, sinon.match(operation.id).and(sinon.match('404')))).to.be.true; + + try { + await operation.fetchAll(); + expect.fail('It should throw an OperationStateError'); + } catch (e) { + if (e instanceof AssertionError) { + throw e; + } + expect(e).to.be.instanceOf(OperationStateError); + expect((e as OperationStateError).errorCode).to.equal(OperationStateErrorCode.Closed); + } + + await operation.close(); + expect(closeOperation.calledOnce).to.be.true; + }); + + it('should complete teardown when CloseOperation returns an error status after finished()', async () => { + const { operation, onClose } = finishedOperationWithFailingClose(async () => ({ + status: { statusCode: TStatusCode.ERROR_STATUS, errorMessage: 'Invalid OperationHandle' }, + })); + + await operation.finished(); + const status = await operation.close(); + + expect(status.isSuccess).to.be.true; + expect(status.info.join()).to.contain('Invalid OperationHandle'); + expect(operation['closed']).to.be.true; + expect(onClose.calledOnce).to.be.true; + }); + + it('should not throw from an autoClose iterator when CloseOperation fails', async () => { + const { closeOperation, operation, onClose } = finishedOperationWithFailingClose(() => + Promise.reject(closeFailure()), + ); + + const rows: Array = []; + for await (const row of operation.iterateRows({ autoClose: true })) { + rows.push(row); + } + + expect(rows).to.have.lengthOf(3); + expect(closeOperation.calledOnce).to.be.true; + expect(operation['closed']).to.be.true; + expect(onClose.calledOnce).to.be.true; + }); + + it('should still rethrow when the operation was not observed to finish', async () => { + const { closeOperation, operation, onClose } = finishedOperationWithFailingClose(() => + Promise.reject(closeFailure()), + ); + + await expectFailure(() => operation.close()); + expect(operation['closed']).to.be.false; + expect(onClose.called).to.be.false; + + // Still open, so the close can be retried. + closeOperation.resolves({ status: { statusCode: TStatusCode.SUCCESS_STATUS } }); + await operation.close(); + expect(operation['closed']).to.be.true; + expect(onClose.calledOnce).to.be.true; + }); + }); }); describe('finished', () => { @@ -1211,6 +1314,22 @@ describe('DBSQLOperation', () => { expect(emitError.calledOnce).to.be.true; }); + it('emits ERROR and statement-complete telemetry when a finished operation fails to close', async () => { + const { context, emitError } = makeContextWithEmitter(); + const completeStatement = sinon.stub(); + context.telemetryAggregator = { completeStatement } as any; + context.driver.closeOperationResp.status.statusCode = TStatusCode.ERROR_STATUS; + const operation = createOperationForTest({ handle: operationHandleStub({ hasResultSet: true }), context }); + + await operation.finished(); + await operation.close(); + + expect(emitError.calledOnce).to.be.true; + expect(emitError.firstCall.args[0].errorName).to.match(/Status\s?Error/); + expect((context.telemetryEmitter!.emitStatementComplete as sinon.SinonStub).calledOnce).to.be.true; + expect(completeStatement.calledOnceWith(operation.id)).to.be.true; + }); + it('emits ERROR telemetry when getMetadata rejects', async () => { const { context, emitError } = makeContextWithEmitter(); // Force the driver to fail metadata fetch diff --git a/tests/unit/DBSQLSession.test.ts b/tests/unit/DBSQLSession.test.ts index ce782207..22e76e4f 100644 --- a/tests/unit/DBSQLSession.test.ts +++ b/tests/unit/DBSQLSession.test.ts @@ -724,6 +724,23 @@ describe('DBSQLSession', () => { expect(session['operations']['items'].size).to.eq(0); }); + it('should close the session when a finished operation fails to close', async () => { + const context = new ClientContextStub(); + const closeSession = sinon.spy(context.driver, 'closeSession'); + sinon + .stub(context.driver, 'closeOperation') + .rejects(new Error('Received a response with a bad HTTP status: 404')); + const session = createSessionForTest({ handle: sessionHandleStub, context }); + const operation = await session.executeStatement('SELECT * FROM table'); + await operation.finished(); + + await session.close(); + + expect(closeSession.calledOnce).to.be.true; + expect(session['isOpen']).to.be.false; + expect(session['operations']['items'].size).to.eq(0); + }); + it('should reject all methods once closed', async () => { const session = createSessionForTest({ handle: sessionHandleStub, context: new ClientContextStub() }); await session.close(); diff --git a/tests/unit/kernel/operation-lifecycle.test.ts b/tests/unit/kernel/operation-lifecycle.test.ts index 839a75d3..ac309816 100644 --- a/tests/unit/kernel/operation-lifecycle.test.ts +++ b/tests/unit/kernel/operation-lifecycle.test.ts @@ -25,6 +25,9 @@ import { expect } from 'chai'; import sinon from 'sinon'; +import { tableFromArrays, tableToIPC } from 'apache-arrow'; +import DBSQLOperation from '../../../lib/DBSQLOperation'; +import ClientContextStub from '../.stubs/ClientContextStub'; import { OperationStatus, OperationState } from '../../../lib/contracts/OperationStatus'; import IClientContext from '../../../lib/contracts/IClientContext'; import IDBSQLLogger, { LogLevel } from '../../../lib/contracts/IDBSQLLogger'; @@ -436,3 +439,59 @@ describe('KernelOperationBackend (lifecycle integration)', () => { expect(op.hasResultSet()).to.equal(true); }); }); + +describe('DBSQLOperation over KernelOperationBackend — close after the result was drained', () => { + function drainedOperation(close: () => Promise) { + const ipcBytes = Buffer.from(tableToIPC(tableFromArrays({ x: Int32Array.from([1, 2]) }), 'stream')); + const batches = [ipcBytes]; + const statement = { + cancel: sinon.stub().resolves(), + close: sinon.stub().callsFake(close), + schema: () => ({ ipcBytes }), + fetchNextBatch: async () => (batches.length > 0 ? { ipcBytes: batches.shift()! } : null), + }; + const context = new ClientContextStub(); + const log = sinon.spy(context.logger, 'log'); + const backend = new KernelOperationBackend({ statement, context, id: 'kernel-op' }); + const operation = new DBSQLOperation({ backend, context }); + const onClose = sinon.stub(); + operation.onClose = onClose; + return { statement, log, operation, onClose }; + } + + it('completes teardown when the CloseStatement RPC fails', async () => { + const { statement, log, operation, onClose } = drainedOperation(async () => { + throw new Error(`__databricks_error__:${JSON.stringify({ code: 'NotFound', message: 'HTTP 404 Not Found' })}`); + }); + + expect(await operation.fetchAll()).to.deep.equal([{ x: 1 }, { x: 2 }]); + const status = await operation.close(); + + expect(statement.close.calledOnce).to.equal(true); + expect(status.isSuccess).to.equal(true); + expect(onClose.calledOnce).to.equal(true); + expect(log.calledWith(LogLevel.warn, sinon.match('kernel-op').and(sinon.match('404')))).to.equal(true); + + let thrown: unknown; + try { + await operation.fetchAll(); + } catch (err) { + thrown = err; + } + expect(thrown).to.be.instanceOf(OperationStateError); + expect((thrown as OperationStateError).errorCode).to.equal(OperationStateErrorCode.Closed); + }); + + it('closes normally when the RPC succeeds', async () => { + const { statement, log, operation, onClose } = drainedOperation(async () => undefined); + + await operation.fetchAll(); + const status = await operation.close(); + + expect(statement.close.calledOnce).to.equal(true); + expect(status.isSuccess).to.equal(true); + expect(status.info).to.deep.equal([]); + expect(onClose.calledOnce).to.equal(true); + expect(log.calledWith(LogLevel.warn)).to.equal(false); + }); +}); From 74a950fe185456b5c39835b7daffc6655a52a24a Mon Sep 17 00:00:00 2001 From: "peco-engineer-bot[bot]" Date: Fri, 13 Nov 2026 04:27:55 +0000 Subject: [PATCH 2/4] engineer-bot: address @mention on PR #592 Signed-off-by: peco-engineer-bot[bot] --- lib/DBSQLOperation.ts | 32 ++++++-- tests/unit/DBSQLOperation.test.ts | 74 +++++++++++++++++++ tests/unit/kernel/operation-lifecycle.test.ts | 17 +++++ 3 files changed, 118 insertions(+), 5 deletions(-) diff --git a/lib/DBSQLOperation.ts b/lib/DBSQLOperation.ts index 42a6ecd4..cd8d34f0 100644 --- a/lib/DBSQLOperation.ts +++ b/lib/DBSQLOperation.ts @@ -9,7 +9,13 @@ import IOperation, { IOperationRowsIterator, NodeStreamOptions, } from './contracts/IOperation'; -import { TGetOperationStatusResp, TGetResultSetMetadataResp, TTableSchema } from '../thrift/TCLIService_types'; +import { + TGetOperationStatusResp, + TGetResultSetMetadataResp, + TOperationState, + TTableSchema, +} from '../thrift/TCLIService_types'; +import { OperationState } from './contracts/OperationStatus'; import Status from './dto/Status'; import { LogLevel } from './contracts/IDBSQLLogger'; import OperationStateError, { OperationStateErrorCode } from './errors/OperationStateError'; @@ -189,11 +195,18 @@ export default class DBSQLOperation implements IOperation { this.pollCount += 1; if (this.backend instanceof ThriftOperationBackend) { // Zero-loss path: the Thrift backend has the wire response on hand. - return this.backend.thriftStatusResponse(progress); + const response = await this.backend.thriftStatusResponse(progress); + if (response.operationState === TOperationState.FINISHED_STATE) { + this.executionFinished = true; + } + return response; } // Non-Thrift backend: synthesize the Thrift-shaped response from the // neutral OperationStatus DTO. const status = await this.backend.status(progress); + if (status.state === OperationState.Succeeded) { + this.executionFinished = true; + } return synthesizeThriftStatus(status); } @@ -244,7 +257,12 @@ export default class DBSQLOperation implements IOperation { this.context .getLogger() .log(LogLevel.warn, `Failed to close finished operation ${this.id}; it is closed locally. Cause: ${message}`); - this.emitErrorEvent(error); + // Buffer the error on the statement instead of reporting it as terminal: + // a terminal error finalizes the statement's aggregation immediately, + // and `emitStatementComplete()` below would then start and finalize a + // second, partial one. Buffered, it's flushed with the single + // statement metric. + this.emitErrorEvent(error, { deferToCompletion: true }); result = Status.success([`Failed to close operation: ${message}`]); } this.closed = true; @@ -460,8 +478,12 @@ export default class DBSQLOperation implements IOperation { * in the aggregator; retryable errors are buffered until the statement * completes. All exceptions from this method itself are swallowed at * debug level — telemetry must never break the driver. + * + * Pass `deferToCompletion` when the caller completes the statement right + * afterwards: the error is then buffered (not terminal) so the statement's + * aggregation is finalized once, by that completion. */ - private emitErrorEvent(error: Error): void { + private emitErrorEvent(error: Error, options?: { deferToCompletion?: boolean }): void { safeEmit(this.context, (emitter) => { emitter.emitError({ statementId: this.id, @@ -469,7 +491,7 @@ export default class DBSQLOperation implements IOperation { errorName: error.name || 'Error', errorMessage: error.message || 'Unknown error', errorStack: error.stack, - isTerminal: ExceptionClassifier.isTerminal(error), + isTerminal: options?.deferToCompletion ? false : ExceptionClassifier.isTerminal(error), }); }); } diff --git a/tests/unit/DBSQLOperation.test.ts b/tests/unit/DBSQLOperation.test.ts index 9cfa57a1..63225707 100644 --- a/tests/unit/DBSQLOperation.test.ts +++ b/tests/unit/DBSQLOperation.test.ts @@ -14,6 +14,12 @@ import StatusError from '../../lib/errors/StatusError'; import OperationStateError, { OperationStateErrorCode } from '../../lib/errors/OperationStateError'; import { LogLevel } from '../../lib/contracts/IDBSQLLogger'; import HiveDriverError from '../../lib/errors/HiveDriverError'; +import AuthenticationError from '../../lib/errors/AuthenticationError'; +import MetricsAggregator from '../../lib/telemetry/MetricsAggregator'; +import DatabricksTelemetryExporter from '../../lib/telemetry/DatabricksTelemetryExporter'; +import TelemetryEventEmitter from '../../lib/telemetry/TelemetryEventEmitter'; +import { TelemetryEventType } from '../../lib/telemetry/types'; +import { mapOperationTypeToTelemetryType } from '../../lib/telemetry/telemetryTypeMappers'; import JsonResultHandler from '../../lib/result/JsonResultHandler'; import ArrowResultConverter from '../../lib/result/ArrowResultConverter'; import ArrowResultHandler from '../../lib/result/ArrowResultHandler'; @@ -518,6 +524,35 @@ describe('DBSQLOperation', () => { expect(operation['closed']).to.be.true; expect(onClose.calledOnce).to.be.true; }); + + it('should complete teardown when status() reported FINISHED_STATE', async () => { + const { context, closeOperation, operation, onClose } = finishedOperationWithFailingClose(() => + Promise.reject(closeFailure()), + ); + context.driver.getOperationStatusResp.operationState = TOperationState.FINISHED_STATE; + + const response = await operation.status(); + expect(response.operationState).to.equal(TOperationState.FINISHED_STATE); + + const status = await operation.close(); + + expect(closeOperation.calledOnce).to.be.true; + expect(status.isSuccess).to.be.true; + expect(status.info.join()).to.contain('404'); + expect(operation['closed']).to.be.true; + expect(onClose.calledOnce).to.be.true; + }); + + it('should still rethrow when status() reported a non-finished state', async () => { + const { context, operation, onClose } = finishedOperationWithFailingClose(() => Promise.reject(closeFailure())); + context.driver.getOperationStatusResp.operationState = TOperationState.RUNNING_STATE; + + await operation.status(); + + await expectFailure(() => operation.close()); + expect(operation['closed']).to.be.false; + expect(onClose.called).to.be.false; + }); }); }); @@ -1330,6 +1365,45 @@ describe('DBSQLOperation', () => { expect(completeStatement.calledOnceWith(operation.id)).to.be.true; }); + it('finalizes a single statement metric when a finished operation fails to close with a terminal error', async () => { + const context = new ClientContextStub({ telemetryEnabled: true }); + const exporter = sinon.createStubInstance(DatabricksTelemetryExporter); + const aggregator = new MetricsAggregator(context, exporter as any); + const emitter = new TelemetryEventEmitter(context); + for (const eventType of Object.values(TelemetryEventType)) { + emitter.on(eventType, (event) => aggregator.processEvent(event)); + } + context.telemetryEmitter = emitter; + context.telemetryAggregator = aggregator; + sinon.stub(context.driver, 'closeOperation').rejects(new AuthenticationError('token expired')); + const operation = createOperationForTest({ handle: operationHandleStub({ hasResultSet: true }), context }); + + try { + await operation.finished(); + emitter.emitCloudFetchChunk({ statementId: operation.id, chunkIndex: 0, bytes: 100, latencyMs: 5 }); + const status = await operation.close(); + expect(status.isSuccess).to.be.true; + } finally { + await aggregator.close(); + } + + const metrics = exporter.export.getCalls().flatMap((call) => call.args[0]); + const statementMetrics = metrics.filter((m) => m.metricType === 'statement'); + expect(statementMetrics).to.have.lengthOf(1); + expect(statementMetrics[0].statementId).to.equal(operation.id); + expect(statementMetrics[0].operationType).to.equal( + mapOperationTypeToTelemetryType(TOperationType.EXECUTE_STATEMENT), + ); + expect(statementMetrics[0].latencyMs).to.be.a('number'); + expect(statementMetrics[0].chunkCount).to.equal(1); + expect(statementMetrics[0].bytesDownloaded).to.equal(100); + + const errorMetrics = metrics.filter((m) => m.metricType === 'error'); + expect(errorMetrics).to.have.lengthOf(1); + expect(errorMetrics[0].statementId).to.equal(operation.id); + expect(errorMetrics[0].errorName).to.equal('AuthenticationError'); + }); + it('emits ERROR telemetry when getMetadata rejects', async () => { const { context, emitError } = makeContextWithEmitter(); // Force the driver to fail metadata fetch diff --git a/tests/unit/kernel/operation-lifecycle.test.ts b/tests/unit/kernel/operation-lifecycle.test.ts index ac309816..e1de0fb7 100644 --- a/tests/unit/kernel/operation-lifecycle.test.ts +++ b/tests/unit/kernel/operation-lifecycle.test.ts @@ -42,6 +42,7 @@ import { import KernelOperationBackend from '../../../lib/kernel/KernelOperationBackend'; import OperationStateError, { OperationStateErrorCode } from '../../../lib/errors/OperationStateError'; import HiveDriverError from '../../../lib/errors/HiveDriverError'; +import { TOperationState } from '../../../thrift/TCLIService_types'; class TestLogger implements IDBSQLLogger { public readonly entries: Array<{ level: LogLevel; message: string }> = []; @@ -482,6 +483,22 @@ describe('DBSQLOperation over KernelOperationBackend — close after the result expect((thrown as OperationStateError).errorCode).to.equal(OperationStateErrorCode.Closed); }); + it('completes teardown when the CloseStatement RPC fails after status() reported success', async () => { + const { statement, operation, onClose } = drainedOperation(async () => { + throw new Error(`__databricks_error__:${JSON.stringify({ code: 'NotFound', message: 'HTTP 404 Not Found' })}`); + }); + + const response = await operation.status(); + expect(response.operationState).to.equal(TOperationState.FINISHED_STATE); + + const status = await operation.close(); + + expect(statement.close.calledOnce).to.equal(true); + expect(status.isSuccess).to.equal(true); + expect(status.info.join()).to.contain('404'); + expect(onClose.calledOnce).to.equal(true); + }); + it('closes normally when the RPC succeeds', async () => { const { statement, log, operation, onClose } = drainedOperation(async () => undefined); From ab187a34cb9ea424c9bd4aea9e8be376292446e9 Mon Sep 17 00:00:00 2001 From: "peco-engineer-bot[bot]" Date: Fri, 13 Nov 2026 04:54:09 +0000 Subject: [PATCH 3/4] engineer-bot: address @mention on PR #592 Signed-off-by: peco-engineer-bot[bot] --- CHANGELOG.md | 2 +- lib/DBSQLOperation.ts | 26 +--- lib/contracts/IOperationBackend.ts | 8 ++ lib/kernel/KernelOperationBackend.ts | 32 ++++- lib/thrift-backend/EmptyOperationBackend.ts | 4 + lib/thrift-backend/ThriftOperationBackend.ts | 6 + tests/unit/DBSQLOperation.test.ts | 52 +++++++ tests/unit/DBSQLSession.test.ts | 62 ++++++++- tests/unit/kernel/operation-lifecycle.test.ts | 130 ++++++++++++++++++ 9 files changed, 296 insertions(+), 26 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9bc027cc..e2ff5913 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ - CloudFetch retry-exhaustion errors now identify the download and the last failure: the `RetryError` message reads `CloudFetch result download failed: (last error: )`. `` is `HTTP : ` (or `HTTP ` when the status text is empty) for HTTP failures, and `network error `, `network error ()` (e.g. `aborted`, when there is no `code`) or a bare `network error` for network failures (the raw network-error message, which can contain the presigned URL, is not copied). If no cause can be determined, the ` (last error: …)` suffix is omitted. The `RetryError` instance, `errorCode` and `payload` (including the node-fetch `Request` / `Response`) are unchanged. (#471) - Report untyped NULL columns as STRING in kernel result metadata, matching Thrift. (#470) -- `IOperation.close()` no longer throws when the close request fails after the operation has finished (for example, once its results were read): the failure is logged as a warning and reported in the returned status's `info`, and the operation is still closed locally. Closing an operation that hasn't finished still throws on failure. (#480) +- `IOperation.close()` no longer throws when the close request fails after the operation has finished (for example, once its results were read, or when the execute response already reported completion): the failure is logged as a warning and reported in the returned status's `info`, and the operation is still closed locally. Closing an operation that hasn't finished still throws on failure. (#480) - Reject non-scalar query parameter values at bind time with a `ParameterError` on both the Thrift and kernel backends, instead of stringifying them (which could leave a Thrift operation stuck before FINISHED). Newly rejected on the Thrift backend: arrays, plain objects, boxed primitives (e.g. `new String('x')`, `new Number(1)`), functions, symbols, and a `DBSQLParameter` that wraps another `DBSQLParameter`. Primitives, `Date`, `Int64`, and `DBSQLParameter` are still accepted. - `DBSQLLogger`: a `filepath` naming an existing directory now writes to `databricks-sql-nodejs.log` inside it, instead of silently producing no logs. (#565) - `DBSQLLogger`: if the log file can't be opened (including when its directory can't be created, which used to throw from the constructor), the logger now logs to the console only, leaves `transports.file` undefined, and emits a process warning with code `DBSQL_LOG_FILE_UNAVAILABLE`. Previously an unwritable file could silently stall all logging, including the console. (#565) diff --git a/lib/DBSQLOperation.ts b/lib/DBSQLOperation.ts index cd8d34f0..288311e8 100644 --- a/lib/DBSQLOperation.ts +++ b/lib/DBSQLOperation.ts @@ -9,13 +9,7 @@ import IOperation, { IOperationRowsIterator, NodeStreamOptions, } from './contracts/IOperation'; -import { - TGetOperationStatusResp, - TGetResultSetMetadataResp, - TOperationState, - TTableSchema, -} from '../thrift/TCLIService_types'; -import { OperationState } from './contracts/OperationStatus'; +import { TGetOperationStatusResp, TGetResultSetMetadataResp, TTableSchema } from '../thrift/TCLIService_types'; import Status from './dto/Status'; import { LogLevel } from './contracts/IDBSQLLogger'; import OperationStateError, { OperationStateErrorCode } from './errors/OperationStateError'; @@ -61,10 +55,6 @@ export default class DBSQLOperation implements IOperation { private cancelled: boolean = false; - // Set once the backend reports the statement finished successfully, after - // which a failed close cannot leave it running server-side. - private executionFinished: boolean = false; - public readonly _data?: unknown; private metadata?: ResultMetadata; @@ -195,18 +185,11 @@ export default class DBSQLOperation implements IOperation { this.pollCount += 1; if (this.backend instanceof ThriftOperationBackend) { // Zero-loss path: the Thrift backend has the wire response on hand. - const response = await this.backend.thriftStatusResponse(progress); - if (response.operationState === TOperationState.FINISHED_STATE) { - this.executionFinished = true; - } - return response; + return this.backend.thriftStatusResponse(progress); } // Non-Thrift backend: synthesize the Thrift-shaped response from the // neutral OperationStatus DTO. const status = await this.backend.status(progress); - if (status.state === OperationState.Succeeded) { - this.executionFinished = true; - } return synthesizeThriftStatus(status); } @@ -247,7 +230,9 @@ export default class DBSQLOperation implements IOperation { } catch (error: any) { // Until the statement finished, closing it is what stops it server-side, // so the failure is the caller's to handle (and the close can be retried). - if (!this.executionFinished) { + // The backend answers from completion state it already holds (including + // direct results), without another status RPC. + if (!this.backend.isFinished()) { throw error; } // Afterwards the close only releases server-side results, which closing @@ -408,7 +393,6 @@ export default class DBSQLOperation implements IOperation { }; try { await this.backend.waitUntilReady(backendOptions); - this.executionFinished = true; } catch (err) { // Reflect terminal states back into facade flags so subsequent calls // short-circuit via failIfClosed(). diff --git a/lib/contracts/IOperationBackend.ts b/lib/contracts/IOperationBackend.ts index 4be71f07..7a5a379f 100644 --- a/lib/contracts/IOperationBackend.ts +++ b/lib/contracts/IOperationBackend.ts @@ -54,6 +54,14 @@ export default interface IOperationBackend { */ hasResultSet(): boolean; + /** + * Whether the backend already knows the statement finished successfully, + * from state it has observed: the execute response / direct results, a + * status poll, or a completed wait. Synchronous and MUST NOT issue an RPC. + * `false` means execution may still be running server-side. + */ + isFinished(): boolean; + /** * Fetch the next chunk of result rows. * diff --git a/lib/kernel/KernelOperationBackend.ts b/lib/kernel/KernelOperationBackend.ts index 8173a0fe..0107735d 100644 --- a/lib/kernel/KernelOperationBackend.ts +++ b/lib/kernel/KernelOperationBackend.ts @@ -215,6 +215,11 @@ export default class KernelOperationBackend implements IOperationBackend { // already-terminal statement. Drives both fetch and result-metadata. private fetchHandlePromise?: Promise; + // Async path only: set once the kernel reported the statement Succeeded (a + // status poll, or `awaitResult()` resolving). The kernel exposes no + // synchronous terminal-state accessor, so the observed success is retained. + private asyncSucceeded = false; + constructor({ asyncStatement, statement, cancellableExecution, context, id }: KernelOperationBackendOptions) { // Exactly one of the three handle kinds must be supplied. const providedCount = @@ -269,6 +274,19 @@ export default class KernelOperationBackend implements IOperationBackend { return true; } + public isFinished(): boolean { + if (this.asyncStatement) { + return this.asyncSucceeded; + } + if (this.cancellableExecution) { + // `blockingStatement` is set only once `result()` resolved a terminal statement. + return this.blockingStatement !== undefined; + } + // Metadata path and the directResults Completed arm: the statement was + // terminal when the backend was constructed. + return true; + } + // --------------------------------------------------------------------------- // Fetch / metadata (owned by the kernel-results pipeline). // --------------------------------------------------------------------------- @@ -397,6 +415,7 @@ export default class KernelOperationBackend implements IOperationBackend { // result. const state = statusStringToOperationState(await this.asyncStatement.status()); if (state === OperationState.Succeeded) { + this.asyncSucceeded = true; return { state, hasResultSet: true, ...(await this.readRichStatusFields()) }; } return { state, hasResultSet: true }; @@ -612,6 +631,7 @@ export default class KernelOperationBackend implements IOperationBackend { case OperationState.Running: break; case OperationState.Succeeded: + this.asyncSucceeded = true; // Materialise the result stream now so the first fetch/metadata call // doesn't pay an extra await_result round-trip. // eslint-disable-next-line no-await-in-loop @@ -725,9 +745,15 @@ export default class KernelOperationBackend implements IOperationBackend { private getFetchHandle(): Promise { if (!this.fetchHandlePromise) { if (this.asyncStatement) { - this.fetchHandlePromise = this.asyncStatement.awaitResult().catch((err) => { - throw decodeNapiKernelError(err); - }) as Promise; + this.fetchHandlePromise = this.asyncStatement.awaitResult().then( + (handle) => { + this.asyncSucceeded = true; + return handle; + }, + (err) => { + throw decodeNapiKernelError(err); + }, + ) as Promise; } else if (this.cancellableExecution) { // Sync (`runAsync: false`) path: drive the blocking `result()` to the // terminal `Statement`. Store it on `blockingStatement` so `close()` can diff --git a/lib/thrift-backend/EmptyOperationBackend.ts b/lib/thrift-backend/EmptyOperationBackend.ts index daa9c13c..f3e5d5d3 100644 --- a/lib/thrift-backend/EmptyOperationBackend.ts +++ b/lib/thrift-backend/EmptyOperationBackend.ts @@ -14,6 +14,10 @@ export default class EmptyOperationBackend implements IOperationBackend { return true; } + public isFinished(): boolean { + return true; + } + public async fetchChunk(): Promise> { return []; } diff --git a/lib/thrift-backend/ThriftOperationBackend.ts b/lib/thrift-backend/ThriftOperationBackend.ts index dd5ac327..7443e34f 100644 --- a/lib/thrift-backend/ThriftOperationBackend.ts +++ b/lib/thrift-backend/ThriftOperationBackend.ts @@ -147,6 +147,12 @@ export default class ThriftOperationBackend implements IOperationBackend { return Boolean(this.emptyResultOnNotFound || this.operationHandle.hasResultSet); } + public isFinished(): boolean { + // `state` is updated from every status response this backend processes, + // including the direct-results status delivered with the execute response. + return this.state === TOperationState.FINISHED_STATE; + } + public async fetchChunk({ limit, disableBuffering, diff --git a/tests/unit/DBSQLOperation.test.ts b/tests/unit/DBSQLOperation.test.ts index 63225707..42818950 100644 --- a/tests/unit/DBSQLOperation.test.ts +++ b/tests/unit/DBSQLOperation.test.ts @@ -543,6 +543,58 @@ describe('DBSQLOperation', () => { expect(onClose.calledOnce).to.be.true; }); + it('should complete teardown when closed right after direct results reported FINISHED_STATE', async () => { + const context = new ClientContextStub(); + const getOperationStatus = sinon.spy(context.driver, 'getOperationStatus'); + const closeOperation = sinon.stub(context.driver, 'closeOperation').rejects(closeFailure()); + const operation = createOperationForTest({ + handle: operationHandleStub({ hasResultSet: true }), + context, + directResults: { + operationStatus: { + status: { statusCode: TStatusCode.SUCCESS_STATUS }, + operationState: TOperationState.FINISHED_STATE, + hasResultSet: true, + }, + }, + }); + const onClose = sinon.stub(); + operation.onClose = onClose; + + const status = await operation.close(); + + expect(closeOperation.calledOnce).to.be.true; + expect(getOperationStatus.called).to.be.false; + expect(status.isSuccess).to.be.true; + expect(status.info.join()).to.contain('404'); + expect(operation['closed']).to.be.true; + expect(onClose.calledOnce).to.be.true; + }); + + it('should still rethrow when closed right after direct results reported RUNNING_STATE', async () => { + const context = new ClientContextStub(); + const getOperationStatus = sinon.spy(context.driver, 'getOperationStatus'); + sinon.stub(context.driver, 'closeOperation').rejects(closeFailure()); + const operation = createOperationForTest({ + handle: operationHandleStub({ hasResultSet: true }), + context, + directResults: { + operationStatus: { + status: { statusCode: TStatusCode.SUCCESS_STATUS }, + operationState: TOperationState.RUNNING_STATE, + hasResultSet: true, + }, + }, + }); + const onClose = sinon.stub(); + operation.onClose = onClose; + + await expectFailure(() => operation.close()); + expect(getOperationStatus.called).to.be.false; + expect(operation['closed']).to.be.false; + expect(onClose.called).to.be.false; + }); + it('should still rethrow when status() reported a non-finished state', async () => { const { context, operation, onClose } = finishedOperationWithFailingClose(() => Promise.reject(closeFailure())); context.driver.getOperationStatusResp.operationState = TOperationState.RUNNING_STATE; diff --git a/tests/unit/DBSQLSession.test.ts b/tests/unit/DBSQLSession.test.ts index 22e76e4f..5e4cab16 100644 --- a/tests/unit/DBSQLSession.test.ts +++ b/tests/unit/DBSQLSession.test.ts @@ -9,7 +9,7 @@ import { PrimaryKeysRequest } from '../../lib/contracts/IDBSQLSession'; import ISessionBackend from '../../lib/contracts/ISessionBackend'; import ParameterError from '../../lib/errors/ParameterError'; import { DBSQLParameter } from '../../lib/DBSQLParameter'; -import { TSessionHandle, TProtocolVersion } from '../../thrift/TCLIService_types'; +import { TSessionHandle, TProtocolVersion, TOperationState, TStatusCode } from '../../thrift/TCLIService_types'; import ClientContextStub from './.stubs/ClientContextStub'; import { createSessionForTest } from './.stubs/createSessionForTest'; @@ -741,6 +741,66 @@ describe('DBSQLSession', () => { expect(session['operations']['items'].size).to.eq(0); }); + describe('right after execution', () => { + function sessionWithFailingOperationClose(operationState: TOperationState) { + const context = new ClientContextStub(); + const closeSession = sinon.spy(context.driver, 'closeSession'); + const getOperationStatus = sinon.spy(context.driver, 'getOperationStatus'); + sinon + .stub(context.driver, 'closeOperation') + .rejects(new Error('Received a response with a bad HTTP status: 404')); + context.driver.executeStatementResp = { + ...context.driver.executeStatementResp, + directResults: { + operationStatus: { status: { statusCode: TStatusCode.SUCCESS_STATUS }, operationState }, + }, + }; + const session = createSessionForTest({ handle: sessionHandleStub, context }); + return { session, closeSession, getOperationStatus }; + } + + it('should close the operation and session when direct results reported completion', async () => { + const { session, closeSession, getOperationStatus } = sessionWithFailingOperationClose( + TOperationState.FINISHED_STATE, + ); + const operation = await session.executeStatement('SELECT * FROM table'); + + const status = await operation.close(); + expect(status.isSuccess).to.be.true; + expect(session['operations']['items'].size).to.eq(0); + + await session.close(); + expect(getOperationStatus.called).to.be.false; + expect(closeSession.calledOnce).to.be.true; + expect(session['isOpen']).to.be.false; + }); + + it('should close the session when direct results reported completion', async () => { + const { session, closeSession, getOperationStatus } = sessionWithFailingOperationClose( + TOperationState.FINISHED_STATE, + ); + await session.executeStatement('SELECT * FROM table'); + + await session.close(); + + expect(getOperationStatus.called).to.be.false; + expect(closeSession.calledOnce).to.be.true; + expect(session['isOpen']).to.be.false; + expect(session['operations']['items'].size).to.eq(0); + }); + + it('should still throw from session close when the operation may still be running', async () => { + const { session, closeSession } = sessionWithFailingOperationClose(TOperationState.RUNNING_STATE); + await session.executeStatement('SELECT * FROM table'); + + await expectFailure(() => session.close()); + + expect(closeSession.called).to.be.false; + expect(session['isOpen']).to.be.true; + expect(session['operations']['items'].size).to.eq(1); + }); + }); + it('should reject all methods once closed', async () => { const session = createSessionForTest({ handle: sessionHandleStub, context: new ClientContextStub() }); await session.close(); diff --git a/tests/unit/kernel/operation-lifecycle.test.ts b/tests/unit/kernel/operation-lifecycle.test.ts index e1de0fb7..91c71431 100644 --- a/tests/unit/kernel/operation-lifecycle.test.ts +++ b/tests/unit/kernel/operation-lifecycle.test.ts @@ -27,6 +27,9 @@ import { expect } from 'chai'; import sinon from 'sinon'; import { tableFromArrays, tableToIPC } from 'apache-arrow'; import DBSQLOperation from '../../../lib/DBSQLOperation'; +import DBSQLSession from '../../../lib/DBSQLSession'; +import KernelSessionBackend from '../../../lib/kernel/KernelSessionBackend'; +import { KernelConnection } from '../../../lib/kernel/KernelNativeLoader'; import ClientContextStub from '../.stubs/ClientContextStub'; import { OperationStatus, OperationState } from '../../../lib/contracts/OperationStatus'; import IClientContext from '../../../lib/contracts/IClientContext'; @@ -512,3 +515,130 @@ describe('DBSQLOperation over KernelOperationBackend — close after the result expect(log.calledWith(LogLevel.warn)).to.equal(false); }); }); + +describe('DBSQLSession over KernelSessionBackend — close right after execution', () => { + const closeFailure = () => + new Error(`__databricks_error__:${JSON.stringify({ code: 'NotFound', message: 'HTTP 404 Not Found' })}`); + + function emptyResultHandle() { + const ipcBytes = Buffer.from(tableToIPC(tableFromArrays({ x: Int32Array.from([]) }), 'stream')); + return { schema: () => ({ ipcBytes }), fetchNextBatch: async () => null }; + } + + // `executeStatementDirect` returns the terminal `Statement` (Completed arm) + // or, with `asyncState` set, a still-pending `AsyncStatement` (Running arm). + function kernelSession(asyncState?: string) { + const terminalStatement = { + ...emptyResultHandle(), + statementId: 'kernel-direct-op', + cancel: sinon.stub().resolves(), + close: sinon.stub().rejects(closeFailure()), + }; + const asyncStatement = { + statementId: 'kernel-async-op', + status: sinon.stub().resolves(asyncState), + awaitResult: sinon.stub().resolves(emptyResultHandle()), + cancel: sinon.stub().resolves(), + close: sinon.stub().rejects(closeFailure()), + }; + const connection = { + executeStatementDirect: sinon.stub().resolves(asyncState === undefined ? terminalStatement : asyncStatement), + close: sinon.stub().resolves(), + }; + const context = new ClientContextStub(); + const backend = new KernelSessionBackend({ connection: connection as unknown as KernelConnection, context }); + const session = new DBSQLSession({ backend, context }); + return { session, connection, terminalStatement, asyncStatement }; + } + + it('closes the operation and session when execution returned a completed statement', async () => { + const { session, connection, terminalStatement } = kernelSession(); + const operation = await session.executeStatement('SELECT 1'); + + const status = await operation.close(); + expect(terminalStatement.close.calledOnce).to.equal(true); + expect(status.isSuccess).to.equal(true); + expect(status.info.join()).to.contain('404'); + + await session.close(); + expect(connection.close.calledOnce).to.equal(true); + expect(session['isOpen']).to.equal(false); + }); + + it('closes the session when execution returned a completed statement', async () => { + const { session, connection, terminalStatement } = kernelSession(); + await session.executeStatement('SELECT 1'); + + await session.close(); + + expect(terminalStatement.close.calledOnce).to.equal(true); + expect(connection.close.calledOnce).to.equal(true); + expect(session['isOpen']).to.equal(false); + expect(session['operations']['items'].size).to.equal(0); + }); + + it('still throws when execution returned a statement that may still be running', async () => { + const { session, connection, asyncStatement } = kernelSession('Running'); + const operation = await session.executeStatement('SELECT slow'); + + let thrown: unknown; + try { + await operation.close(); + } catch (err) { + thrown = err; + } + expect(thrown).to.be.instanceOf(Error); + expect(asyncStatement.status.called).to.equal(false); + + thrown = undefined; + try { + await session.close(); + } catch (err) { + thrown = err; + } + expect(thrown).to.be.instanceOf(Error); + expect(connection.close.called).to.equal(false); + expect(session['operations']['items'].size).to.equal(1); + }); + + it('retains success observed by status() on a running statement', async () => { + const { session, connection, asyncStatement } = kernelSession('Succeeded'); + const operation = await session.executeStatement('SELECT slow'); + + await operation.status(); + const status = await operation.close(); + + expect(asyncStatement.status.calledOnce).to.equal(true); + expect(asyncStatement.close.calledOnce).to.equal(true); + expect(status.isSuccess).to.equal(true); + await session.close(); + expect(connection.close.calledOnce).to.equal(true); + }); + + it('retains success observed by finished() on a running statement', async () => { + const { session, asyncStatement } = kernelSession('Succeeded'); + const operation = await session.executeStatement('SELECT slow'); + + await operation.finished(); + const status = await operation.close(); + + expect(asyncStatement.close.calledOnce).to.equal(true); + expect(status.isSuccess).to.equal(true); + }); + + it('still throws after status() reported the statement running', async () => { + const { session, asyncStatement } = kernelSession('Running'); + const operation = await session.executeStatement('SELECT slow'); + + await operation.status(); + let thrown: unknown; + try { + await operation.close(); + } catch (err) { + thrown = err; + } + + expect(thrown).to.be.instanceOf(Error); + expect(asyncStatement.close.calledOnce).to.equal(true); + }); +}); From ea8e009640267ad93e6573543188e02175dd0d99 Mon Sep 17 00:00:00 2001 From: "peco-engineer-bot[bot]" Date: Fri, 13 Nov 2026 05:03:07 +0000 Subject: [PATCH 4/4] engineer-bot: address @mention on PR #592 Signed-off-by: peco-engineer-bot[bot] --- lib/kernel/KernelOperationBackend.ts | 6 +++++- tests/unit/kernel/execution.test.ts | 19 +++++++++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/lib/kernel/KernelOperationBackend.ts b/lib/kernel/KernelOperationBackend.ts index 0107735d..2ee1fe44 100644 --- a/lib/kernel/KernelOperationBackend.ts +++ b/lib/kernel/KernelOperationBackend.ts @@ -612,6 +612,11 @@ export default class KernelOperationBackend implements IOperationBackend { // eslint-disable-next-line no-await-in-loop const state = statusStringToOperationState(await this.asyncStatement!.status()); + // Record success before the progress callback runs, so a callback that + // throws on the Succeeded tick still leaves `isFinished()` true. + if (state === OperationState.Succeeded) { + this.asyncSucceeded = true; + } if (options?.callback) { // On the terminal Succeeded tick, carry the rich status fields @@ -631,7 +636,6 @@ export default class KernelOperationBackend implements IOperationBackend { case OperationState.Running: break; case OperationState.Succeeded: - this.asyncSucceeded = true; // Materialise the result stream now so the first fetch/metadata call // doesn't pay an extra await_result round-trip. // eslint-disable-next-line no-await-in-loop diff --git a/tests/unit/kernel/execution.test.ts b/tests/unit/kernel/execution.test.ts index 73e37822..e7517e24 100644 --- a/tests/unit/kernel/execution.test.ts +++ b/tests/unit/kernel/execution.test.ts @@ -1344,6 +1344,25 @@ describe('KernelOperationBackend — async (submitStatement) path', () => { expect(states).to.deep.equal([OperationState.Pending, OperationState.Running, OperationState.Succeeded]); }); + it('isFinished() is true after a progress callback throws on the Succeeded tick', async () => { + const stmt = new FakeAsyncStatement(['Running', 'Succeeded']); + const op = makeAsyncOp(stmt); + let thrown: unknown; + try { + await op.waitUntilReady({ + callback: (s) => { + if (s.state === OperationState.Succeeded) { + throw new Error('callback failure'); + } + }, + }); + } catch (err) { + thrown = err; + } + expect((thrown as Error)?.message).to.equal('callback failure'); + expect(op.isFinished()).to.equal(true); + }); + it('progress callback carries rich-status fields on the terminal Succeeded tick (parity with the sync path)', async () => { // Regression guard: the async poll loop must merge the rich-status fields // into the callback on the Succeeded tick, matching waitUntilReadyCancellable.