Skip to content
Merged
1 change: 1 addition & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
- Fix the kernel backend ignoring `use_cloud_fetch=False`, so complete results can be fetched without cloud-storage downloads.
- Fix inline parameters (`use_inline_params=True`) so a sequence directly after `IN` renders as a list again: `x IN %(ids)s` renders as `x IN (1,2,3)` instead of `x IN ARRAY(1,2,3)`, which Spark rejects. Inside the list, tuples render as rows for a multi-column `IN`, and a `set` or `frozenset` is accepted again. Sequences anywhere else still render as `ARRAY(...)` ([#629](https://github.com/databricks/databricks-sql-python/issues/629))
- Speed up Thrift response handling: read replies in 1 MiB chunks and decode them with Thrift's C-accelerated protocol, and stop stringifying every request/response for the default no-op debug logger. Wide result schemas, metadata calls with many rows, and inline Arrow results benefit most.
- Report `cursor.rowcount` after asynchronous DML completes on both the Thrift and kernel backends, preserving zero counts and fetched result rows.

# 4.9.0 (2026-11-04)
- Upgrade Databricks SQL Kernel to 1.4.0, which keeps Lakehouse Real-Time session affinity and flushes shared telemetry only after the final session closes.
Expand Down
4 changes: 3 additions & 1 deletion src/databricks/sql/backend/kernel/client.py
Original file line number Diff line number Diff line change
Expand Up @@ -1024,7 +1024,9 @@ def get_execution_result(
# telemetry — an accepted, narrow gap limited to the case where
# result-set construction fails after a successful await.
try:
return self._make_result_set(stream, cursor, command_id)
result_set = self._make_result_set(stream, cursor, command_id)
result_set.num_modified_rows = handle.num_modified_rows
Comment thread
aakash-saravanan-db marked this conversation as resolved.
Resolved
Comment thread
aakash-saravanan-db marked this conversation as resolved.
Resolved
Comment thread
aakash-saravanan-db marked this conversation as resolved.
Resolved
Comment thread
aakash-saravanan-db marked this conversation as resolved.
Resolved
Comment thread
aakash-saravanan-db marked this conversation as resolved.
Resolved
return result_set
except Exception as exc:
raise _wrap_kernel_exception("get_execution_result", exc) from exc

Expand Down
30 changes: 30 additions & 0 deletions src/databricks/sql/backend/thrift_backend.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
import re
import time
import threading
import weakref
from typing import Dict, List, Optional, Union, Any, TYPE_CHECKING
from uuid import UUID

Expand Down Expand Up @@ -273,6 +274,9 @@ def __init__(
raise

self._request_lock = threading.RLock()
self._terminal_operation_statuses: weakref.WeakKeyDictionary[
CommandId, ttypes.TGetOperationStatusResp
] = weakref.WeakKeyDictionary()
self._session_id_hex = None

@property
Expand Down Expand Up @@ -935,10 +939,32 @@ def _results_message_to_execute_response(
def get_execution_result(
self, command_id: CommandId, cursor: Cursor
) -> "ResultSet":
"""Fetch a completed async result and its latest operation metadata.

FetchResults does not carry ``numModifiedRows``, so reuse a terminal
status observed for the same ``CommandId`` instance by
``get_query_state``, or refresh the status when the caller resumes
directly by command ID.
Failed or closed operations raise before result rows are fetched.
"""
thrift_handle = command_id.to_thrift_handle()
if not thrift_handle:
raise ValueError("Not a valid Thrift command ID")

with self._request_lock:
operation_status = self._terminal_operation_statuses.pop(command_id, None)
Comment thread
aakash-saravanan-db marked this conversation as resolved.
Resolved
if operation_status is None:
Comment thread
aakash-saravanan-db marked this conversation as resolved.
Resolved
Comment thread
aakash-saravanan-db marked this conversation as resolved.
Resolved
Comment thread
aakash-saravanan-db marked this conversation as resolved.
Resolved
Comment thread
aakash-saravanan-db marked this conversation as resolved.
Resolved
operation_status = self._poll_for_status(thrift_handle)
self._check_command_not_in_error_or_closed_state(
Comment thread
aakash-saravanan-db marked this conversation as resolved.
Resolved
thrift_handle, operation_status
)

num_modified_rows = (
operation_status.numModifiedRows
if operation_status.operationState == ttypes.TOperationState.FINISHED_STATE
else None
)

req = ttypes.TFetchResultsReq(
operationHandle=ttypes.TOperationHandle(
thrift_handle.operationId,
Expand Down Expand Up @@ -989,6 +1015,7 @@ def get_execution_result(
is_staging_operation=is_staging_operation,
arrow_schema_bytes=schema_bytes,
result_format=t_result_set_metadata_resp.resultFormat,
num_modified_rows=num_modified_rows,
)

return ThriftResultSet(
Expand Down Expand Up @@ -1037,6 +1064,9 @@ def get_query_state(self, command_id: CommandId) -> CommandState:
state = CommandState.from_thrift_state(operation_state)
if state is None:
raise ValueError(f"Unknown command state: {operation_state}")
if state == CommandState.SUCCEEDED:
with self._request_lock:
self._terminal_operation_statuses[command_id] = poll_resp
return state

@staticmethod
Expand Down
7 changes: 7 additions & 0 deletions src/databricks/sql/client.py
Original file line number Diff line number Diff line change
Expand Up @@ -1714,6 +1714,13 @@ def get_async_execution_result(self):
self.active_command_id, self
)

num_modified_rows = (
self.active_result_set.num_modified_rows
if self.active_result_set
else None
)
self.rowcount = num_modified_rows if num_modified_rows is not None else -1

if self.active_result_set and self.active_result_set.is_staging_operation:
self._handle_staging_operation(
staging_allowed_local_path=self.connection.staging_allowed_local_path
Expand Down
23 changes: 23 additions & 0 deletions tests/unit/test_client.py
Original file line number Diff line number Diff line change
Expand Up @@ -303,6 +303,29 @@ def test_rowcount_resets_between_statements(self):
cursor.execute("SELECT 1")
self.assertEqual(cursor.rowcount, -1)

def test_async_rowcount_on_completion_and_reset(self):
for count in (5, 0, None):
with self.subTest(count=count):
result_set = Mock(num_modified_rows=count, is_staging_operation=False)
backend = ThriftDatabricksClientMockFactory.new()
backend.get_execution_result.return_value = result_set
backend.get_query_state.return_value = CommandState.SUCCEEDED
cursor = client.Cursor(connection=Mock(), backend=backend)
cursor.rowcount = 9

cursor.execute_async("UPDATE t SET x = 1")
self.assertEqual(cursor.rowcount, -1)
cursor.active_command_id = Mock(spec=CommandId)
self.assertIs(cursor.get_async_execution_result(), cursor)
self.assertEqual(cursor.rowcount, count if count is not None else -1)
result_set.fetchall.assert_not_called()

result_set.num_modified_rows = None
cursor.execute_async("SELECT 1")
self.assertEqual(cursor.rowcount, -1)
cursor.get_async_execution_result()
self.assertEqual(cursor.rowcount, -1)

def test_closed_cursor_doesnt_allow_operations(self):
cursor = client.Cursor(Mock(), Mock())
cursor.close()
Expand Down
17 changes: 15 additions & 2 deletions tests/unit/test_kernel_client.py
Original file line number Diff line number Diff line change
Expand Up @@ -1432,15 +1432,22 @@ def test_get_query_state_uses_reader_after_result_stream_started():
reader.status.assert_called_once_with()


def test_get_execution_result_uses_retained_owning_handle_first():
@pytest.mark.parametrize("count", [7, 0, None])
def test_get_execution_result_uses_retained_owning_handle_first(count):
"""The first in-process result fetch uses the retained submitting
handle so the kernel finalizes the original async statement telemetry."""
c = _make_client()
c._kernel_session = MagicMock()
fake_stream = MagicMock()
fake_stream.arrow_schema.return_value = pa.schema([("n", pa.int64())])
handle = MagicMock()
handle.await_result.return_value = fake_stream
del handle.num_modified_rows

def await_result():
handle.num_modified_rows = count
return fake_stream

handle.await_result.side_effect = await_result
cursor = MagicMock()
cursor.arraysize = 100
cursor.buffer_size_bytes = 1024
Expand All @@ -1451,6 +1458,8 @@ def test_get_execution_result_uses_retained_owning_handle_first():
rs = c.get_execution_result(cid, cursor=cursor)

assert rs is not None
assert rs.num_modified_rows == count
fake_stream.fetch_next_batch.assert_not_called()
c._kernel_session.attach_async_statement.assert_not_called()
handle.await_result.assert_called_once_with()
assert cid.guid in c._async_result_stream_started
Expand All @@ -1463,6 +1472,7 @@ def test_get_execution_result_attaches_by_id_when_no_retained_handle():
fake_stream = MagicMock()
fake_stream.arrow_schema.return_value = pa.schema([("n", pa.int64())])
handle = _attach_returns(c, await_result=fake_stream)
handle.num_modified_rows = 7
cursor = MagicMock()
cursor.arraysize = 100
cursor.buffer_size_bytes = 1024
Expand All @@ -1471,6 +1481,7 @@ def test_get_execution_result_attaches_by_id_when_no_retained_handle():
rs = c.get_execution_result(cid, cursor=cursor)

assert rs is not None
assert rs.num_modified_rows == 7
c._kernel_session.attach_async_statement.assert_called_once_with("async-1")
handle.await_result.assert_called_once_with()

Expand Down Expand Up @@ -1525,6 +1536,7 @@ def test_get_execution_result_construction_failure_retains_marker_and_uses_reade
retry_stream.arrow_schema.return_value = pa.schema([("n", pa.int64())])
reader = MagicMock()
reader.await_result.return_value = retry_stream
reader.num_modified_rows = 0
owning_handle.reader.return_value = reader
cursor = MagicMock()
cursor.arraysize = 100
Expand All @@ -1533,6 +1545,7 @@ def test_get_execution_result_construction_failure_retains_marker_and_uses_reade
rs = c.get_execution_result(cid, cursor=cursor)

assert rs is not None
assert rs.num_modified_rows == 0
owning_handle.reader.assert_called_once_with()
c._kernel_session.attach_async_statement.assert_not_called()
reader.await_result.assert_called_once_with()
Expand Down
60 changes: 59 additions & 1 deletion tests/unit/test_thrift_backend.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
import logging
import unittest
import pytest
from unittest.mock import patch, MagicMock, Mock
from unittest.mock import patch, MagicMock, Mock, sentinel
from ssl import CERT_NONE, CERT_REQUIRED
from urllib3 import HTTPSConnectionPool

Expand Down Expand Up @@ -222,6 +222,64 @@ def _make_fake_thrift_backend(self):
thrift_backend._create_arrow_table.return_value = (MagicMock(), Mock())
return thrift_backend

@patch("databricks.sql.backend.thrift_backend.ThriftResultSet")
@patch("databricks.sql.backend.thrift_backend.TCLIService.Client", autospec=True)
def test_async_result_rowcount(self, tcli_service_class, result_set):
for state, count, expected_count, polled in [
(ttypes.TOperationState.FINISHED_STATE, 7, 7, True),
(ttypes.TOperationState.FINISHED_STATE, 0, 0, True),
(ttypes.TOperationState.FINISHED_STATE, None, None, True),
(ttypes.TOperationState.FINISHED_STATE, 7, 7, False),
(ttypes.TOperationState.RUNNING_STATE, 7, None, False),
]:
with self.subTest(state=state, count=count, polled=polled):
backend = self._make_fake_thrift_backend()
backend._poll_for_status = Mock(
return_value=ttypes.TGetOperationStatusResp(
status=self.okay_status,
operationState=state,
numModifiedRows=count,
)
)
backend.make_request = Mock(
return_value=Mock(
resultSetMetadata=self.metadata_resp,
results=sentinel.counter_rows,
hasMoreRows=False,
)
)
cid = CommandId.from_thrift_handle(self.operation_handle)
if polled:
self.assertEqual(
backend.get_query_state(cid), CommandState.SUCCEEDED
)
backend.get_execution_result(cid, Mock())

backend._poll_for_status.assert_called_once_with(cid.to_thrift_handle())
response = result_set.call_args.kwargs["execute_response"]
self.assertEqual(response.num_modified_rows, expected_count)
self.assertIs(
result_set.call_args.kwargs["t_row_set"], sentinel.counter_rows
)

@patch("databricks.sql.backend.thrift_backend.TCLIService.Client", autospec=True)
def test_async_result_checks_status_before_fetch(self, tcli_service_class):
backend = self._make_fake_thrift_backend()
backend._poll_for_status = Mock(
return_value=ttypes.TGetOperationStatusResp(
status=self.okay_status,
operationState=ttypes.TOperationState.ERROR_STATE,
errorMessage="async operation failed",
)
)
backend.make_request = Mock()
cid = CommandId.from_thrift_handle(self.operation_handle)

with self.assertRaises(ServerOperationError):
backend.get_execution_result(cid, Mock())

backend.make_request.assert_not_called()

def test_hive_schema_to_arrow_schema_preserves_column_names(self):
columns = [
ttypes.TColumnDesc(
Expand Down
Loading