Skip to content

Commit ffbb42c

Browse files
fix: populate rowcount after async DML completion (#1016)
* fix: populate rowcount after async DML completion Carry affected-row counts from kernel handles and Thrift operation status into async result sets, then publish them on the cursor. Preserve zero, unknown counts, and fetched counter rows. Signed-off-by: Aakash Saravanan <aakash.saravanan@databricks.com> * fix: harden async result rowcount retrieval Signed-off-by: Aakash Saravanan <aakash.saravanan@databricks.com> * docs: clarify async kernel rowcount requirement Signed-off-by: Aakash Saravanan <aakash.saravanan@databricks.com> * fix: reuse terminal async Thrift status Signed-off-by: Aakash Saravanan <aakash.saravanan@databricks.com> * fix: annotate async status cache Signed-off-by: Aakash Saravanan <aakash.saravanan@databricks.com> * fix: normalize kernel rowcount access Signed-off-by: Aakash Saravanan <aakash.saravanan@databricks.com> * fix: trust terminal async row counts Signed-off-by: Aakash Saravanan <aakash.saravanan@databricks.com> * fix: read async affected-row counts directly Signed-off-by: Aakash Saravanan <aakash.saravanan@databricks.com> * test: focus async rowcount regression coverage Signed-off-by: Aakash Saravanan <aakash.saravanan@databricks.com> * docs: clarify async result status errors Signed-off-by: Aakash Saravanan <aakash.saravanan@databricks.com> --------- Signed-off-by: Aakash Saravanan <aakash.saravanan@databricks.com>
1 parent b4cec22 commit ffbb42c

7 files changed

Lines changed: 138 additions & 4 deletions

File tree

‎CHANGELOG.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
- Fix the kernel backend ignoring `use_cloud_fetch=False`, so complete results can be fetched without cloud-storage downloads.
55
- Fix inline parameters (`use_inline_params=True`) so a sequence directly after `IN` renders as a list again: `x IN %(ids)s` renders as `x IN (1,2,3)` instead of `x IN ARRAY(1,2,3)`, which Spark rejects. Inside the list, tuples render as rows for a multi-column `IN`, and a `set` or `frozenset` is accepted again. Sequences anywhere else still render as `ARRAY(...)` ([#629](https://github.com/databricks/databricks-sql-python/issues/629))
66
- Speed up Thrift response handling: read replies in 1 MiB chunks and decode them with Thrift's C-accelerated protocol, and stop stringifying every request/response for the default no-op debug logger. Wide result schemas, metadata calls with many rows, and inline Arrow results benefit most.
7+
- Report `cursor.rowcount` after asynchronous DML completes on both the Thrift and kernel backends, preserving zero counts and fetched result rows.
78

89
# 4.9.0 (2026-11-04)
910
- Upgrade Databricks SQL Kernel to 1.4.0, which keeps Lakehouse Real-Time session affinity and flushes shared telemetry only after the final session closes.

‎src/databricks/sql/backend/kernel/client.py‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1024,7 +1024,9 @@ def get_execution_result(
10241024
# telemetry — an accepted, narrow gap limited to the case where
10251025
# result-set construction fails after a successful await.
10261026
try:
1027-
return self._make_result_set(stream, cursor, command_id)
1027+
result_set = self._make_result_set(stream, cursor, command_id)
1028+
result_set.num_modified_rows = handle.num_modified_rows
1029+
return result_set
10281030
except Exception as exc:
10291031
raise _wrap_kernel_exception("get_execution_result", exc) from exc
10301032

‎src/databricks/sql/backend/thrift_backend.py‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66
import re
77
import time
88
import threading
9+
import weakref
910
from typing import Dict, List, Optional, Union, Any, TYPE_CHECKING
1011
from uuid import UUID
1112

@@ -273,6 +274,9 @@ def __init__(
273274
raise
274275

275276
self._request_lock = threading.RLock()
277+
self._terminal_operation_statuses: weakref.WeakKeyDictionary[
278+
CommandId, ttypes.TGetOperationStatusResp
279+
] = weakref.WeakKeyDictionary()
276280
self._session_id_hex = None
277281

278282
@property
@@ -935,10 +939,32 @@ def _results_message_to_execute_response(
935939
def get_execution_result(
936940
self, command_id: CommandId, cursor: Cursor
937941
) -> "ResultSet":
942+
"""Fetch a completed async result and its latest operation metadata.
943+
944+
FetchResults does not carry ``numModifiedRows``, so reuse a terminal
945+
status observed for the same ``CommandId`` instance by
946+
``get_query_state``, or refresh the status when the caller resumes
947+
directly by command ID.
948+
Failed or closed operations raise before result rows are fetched.
949+
"""
938950
thrift_handle = command_id.to_thrift_handle()
939951
if not thrift_handle:
940952
raise ValueError("Not a valid Thrift command ID")
941953

954+
with self._request_lock:
955+
operation_status = self._terminal_operation_statuses.pop(command_id, None)
956+
if operation_status is None:
957+
operation_status = self._poll_for_status(thrift_handle)
958+
self._check_command_not_in_error_or_closed_state(
959+
thrift_handle, operation_status
960+
)
961+
962+
num_modified_rows = (
963+
operation_status.numModifiedRows
964+
if operation_status.operationState == ttypes.TOperationState.FINISHED_STATE
965+
else None
966+
)
967+
942968
req = ttypes.TFetchResultsReq(
943969
operationHandle=ttypes.TOperationHandle(
944970
thrift_handle.operationId,
@@ -989,6 +1015,7 @@ def get_execution_result(
9891015
is_staging_operation=is_staging_operation,
9901016
arrow_schema_bytes=schema_bytes,
9911017
result_format=t_result_set_metadata_resp.resultFormat,
1018+
num_modified_rows=num_modified_rows,
9921019
)
9931020

9941021
return ThriftResultSet(
@@ -1037,6 +1064,9 @@ def get_query_state(self, command_id: CommandId) -> CommandState:
10371064
state = CommandState.from_thrift_state(operation_state)
10381065
if state is None:
10391066
raise ValueError(f"Unknown command state: {operation_state}")
1067+
if state == CommandState.SUCCEEDED:
1068+
with self._request_lock:
1069+
self._terminal_operation_statuses[command_id] = poll_resp
10401070
return state
10411071

10421072
@staticmethod

‎src/databricks/sql/client.py‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1714,6 +1714,13 @@ def get_async_execution_result(self):
17141714
self.active_command_id, self
17151715
)
17161716

1717+
num_modified_rows = (
1718+
self.active_result_set.num_modified_rows
1719+
if self.active_result_set
1720+
else None
1721+
)
1722+
self.rowcount = num_modified_rows if num_modified_rows is not None else -1
1723+
17171724
if self.active_result_set and self.active_result_set.is_staging_operation:
17181725
self._handle_staging_operation(
17191726
staging_allowed_local_path=self.connection.staging_allowed_local_path

‎tests/unit/test_client.py‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -303,6 +303,29 @@ def test_rowcount_resets_between_statements(self):
303303
cursor.execute("SELECT 1")
304304
self.assertEqual(cursor.rowcount, -1)
305305

306+
def test_async_rowcount_on_completion_and_reset(self):
307+
for count in (5, 0, None):
308+
with self.subTest(count=count):
309+
result_set = Mock(num_modified_rows=count, is_staging_operation=False)
310+
backend = ThriftDatabricksClientMockFactory.new()
311+
backend.get_execution_result.return_value = result_set
312+
backend.get_query_state.return_value = CommandState.SUCCEEDED
313+
cursor = client.Cursor(connection=Mock(), backend=backend)
314+
cursor.rowcount = 9
315+
316+
cursor.execute_async("UPDATE t SET x = 1")
317+
self.assertEqual(cursor.rowcount, -1)
318+
cursor.active_command_id = Mock(spec=CommandId)
319+
self.assertIs(cursor.get_async_execution_result(), cursor)
320+
self.assertEqual(cursor.rowcount, count if count is not None else -1)
321+
result_set.fetchall.assert_not_called()
322+
323+
result_set.num_modified_rows = None
324+
cursor.execute_async("SELECT 1")
325+
self.assertEqual(cursor.rowcount, -1)
326+
cursor.get_async_execution_result()
327+
self.assertEqual(cursor.rowcount, -1)
328+
306329
def test_closed_cursor_doesnt_allow_operations(self):
307330
cursor = client.Cursor(Mock(), Mock())
308331
cursor.close()

‎tests/unit/test_kernel_client.py‎

Lines changed: 15 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1432,15 +1432,22 @@ def test_get_query_state_uses_reader_after_result_stream_started():
14321432
reader.status.assert_called_once_with()
14331433

14341434

1435-
def test_get_execution_result_uses_retained_owning_handle_first():
1435+
@pytest.mark.parametrize("count", [7, 0, None])
1436+
def test_get_execution_result_uses_retained_owning_handle_first(count):
14361437
"""The first in-process result fetch uses the retained submitting
14371438
handle so the kernel finalizes the original async statement telemetry."""
14381439
c = _make_client()
14391440
c._kernel_session = MagicMock()
14401441
fake_stream = MagicMock()
14411442
fake_stream.arrow_schema.return_value = pa.schema([("n", pa.int64())])
14421443
handle = MagicMock()
1443-
handle.await_result.return_value = fake_stream
1444+
del handle.num_modified_rows
1445+
1446+
def await_result():
1447+
handle.num_modified_rows = count
1448+
return fake_stream
1449+
1450+
handle.await_result.side_effect = await_result
14441451
cursor = MagicMock()
14451452
cursor.arraysize = 100
14461453
cursor.buffer_size_bytes = 1024
@@ -1451,6 +1458,8 @@ def test_get_execution_result_uses_retained_owning_handle_first():
14511458
rs = c.get_execution_result(cid, cursor=cursor)
14521459

14531460
assert rs is not None
1461+
assert rs.num_modified_rows == count
1462+
fake_stream.fetch_next_batch.assert_not_called()
14541463
c._kernel_session.attach_async_statement.assert_not_called()
14551464
handle.await_result.assert_called_once_with()
14561465
assert cid.guid in c._async_result_stream_started
@@ -1463,6 +1472,7 @@ def test_get_execution_result_attaches_by_id_when_no_retained_handle():
14631472
fake_stream = MagicMock()
14641473
fake_stream.arrow_schema.return_value = pa.schema([("n", pa.int64())])
14651474
handle = _attach_returns(c, await_result=fake_stream)
1475+
handle.num_modified_rows = 7
14661476
cursor = MagicMock()
14671477
cursor.arraysize = 100
14681478
cursor.buffer_size_bytes = 1024
@@ -1471,6 +1481,7 @@ def test_get_execution_result_attaches_by_id_when_no_retained_handle():
14711481
rs = c.get_execution_result(cid, cursor=cursor)
14721482

14731483
assert rs is not None
1484+
assert rs.num_modified_rows == 7
14741485
c._kernel_session.attach_async_statement.assert_called_once_with("async-1")
14751486
handle.await_result.assert_called_once_with()
14761487

@@ -1525,6 +1536,7 @@ def test_get_execution_result_construction_failure_retains_marker_and_uses_reade
15251536
retry_stream.arrow_schema.return_value = pa.schema([("n", pa.int64())])
15261537
reader = MagicMock()
15271538
reader.await_result.return_value = retry_stream
1539+
reader.num_modified_rows = 0
15281540
owning_handle.reader.return_value = reader
15291541
cursor = MagicMock()
15301542
cursor.arraysize = 100
@@ -1533,6 +1545,7 @@ def test_get_execution_result_construction_failure_retains_marker_and_uses_reade
15331545
rs = c.get_execution_result(cid, cursor=cursor)
15341546

15351547
assert rs is not None
1548+
assert rs.num_modified_rows == 0
15361549
owning_handle.reader.assert_called_once_with()
15371550
c._kernel_session.attach_async_statement.assert_not_called()
15381551
reader.await_result.assert_called_once_with()

‎tests/unit/test_thrift_backend.py‎

Lines changed: 59 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55
import logging
66
import unittest
77
import pytest
8-
from unittest.mock import patch, MagicMock, Mock
8+
from unittest.mock import patch, MagicMock, Mock, sentinel
99
from ssl import CERT_NONE, CERT_REQUIRED
1010
from urllib3 import HTTPSConnectionPool
1111

@@ -222,6 +222,64 @@ def _make_fake_thrift_backend(self):
222222
thrift_backend._create_arrow_table.return_value = (MagicMock(), Mock())
223223
return thrift_backend
224224

225+
@patch("databricks.sql.backend.thrift_backend.ThriftResultSet")
226+
@patch("databricks.sql.backend.thrift_backend.TCLIService.Client", autospec=True)
227+
def test_async_result_rowcount(self, tcli_service_class, result_set):
228+
for state, count, expected_count, polled in [
229+
(ttypes.TOperationState.FINISHED_STATE, 7, 7, True),
230+
(ttypes.TOperationState.FINISHED_STATE, 0, 0, True),
231+
(ttypes.TOperationState.FINISHED_STATE, None, None, True),
232+
(ttypes.TOperationState.FINISHED_STATE, 7, 7, False),
233+
(ttypes.TOperationState.RUNNING_STATE, 7, None, False),
234+
]:
235+
with self.subTest(state=state, count=count, polled=polled):
236+
backend = self._make_fake_thrift_backend()
237+
backend._poll_for_status = Mock(
238+
return_value=ttypes.TGetOperationStatusResp(
239+
status=self.okay_status,
240+
operationState=state,
241+
numModifiedRows=count,
242+
)
243+
)
244+
backend.make_request = Mock(
245+
return_value=Mock(
246+
resultSetMetadata=self.metadata_resp,
247+
results=sentinel.counter_rows,
248+
hasMoreRows=False,
249+
)
250+
)
251+
cid = CommandId.from_thrift_handle(self.operation_handle)
252+
if polled:
253+
self.assertEqual(
254+
backend.get_query_state(cid), CommandState.SUCCEEDED
255+
)
256+
backend.get_execution_result(cid, Mock())
257+
258+
backend._poll_for_status.assert_called_once_with(cid.to_thrift_handle())
259+
response = result_set.call_args.kwargs["execute_response"]
260+
self.assertEqual(response.num_modified_rows, expected_count)
261+
self.assertIs(
262+
result_set.call_args.kwargs["t_row_set"], sentinel.counter_rows
263+
)
264+
265+
@patch("databricks.sql.backend.thrift_backend.TCLIService.Client", autospec=True)
266+
def test_async_result_checks_status_before_fetch(self, tcli_service_class):
267+
backend = self._make_fake_thrift_backend()
268+
backend._poll_for_status = Mock(
269+
return_value=ttypes.TGetOperationStatusResp(
270+
status=self.okay_status,
271+
operationState=ttypes.TOperationState.ERROR_STATE,
272+
errorMessage="async operation failed",
273+
)
274+
)
275+
backend.make_request = Mock()
276+
cid = CommandId.from_thrift_handle(self.operation_handle)
277+
278+
with self.assertRaises(ServerOperationError):
279+
backend.get_execution_result(cid, Mock())
280+
281+
backend.make_request.assert_not_called()
282+
225283
def test_hive_schema_to_arrow_schema_preserves_column_names(self):
226284
columns = [
227285
ttypes.TColumnDesc(

0 commit comments

Comments
 (0)