Skip to content

Commit fd10cc1

Browse files
vuanhphungIsaac
andcommitted
perf(thrift): close only the session in Connection.close()
Connection.close() sent one CloseOperation per open cursor, one after another, before CloseSession. On Thrift, CloseSession already releases every operation of the session (finished, undrained CloudFetch, and running statements, which are cancelled), so those round trips only added latency: about one RTT per open cursor. On Thrift, close the session first; the cursors then close client-side only, through the existing path for cursors whose connection is already closed (queues and CloudFetch downloads shut down, result sets marked closed). Kernel and SEA keep closing each command first. Signed-off-by: Vu Anh Phung <vu.phung@databricks.com> Co-authored-by: Isaac <no-reply@databricks.com>
1 parent 8d6db43 commit fd10cc1

3 files changed

Lines changed: 136 additions & 7 deletions

File tree

‎CHANGELOG.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
# Unreleased
44
- Fix inline parameters (`use_inline_params=True`) so a sequence directly after `IN` renders as a list again: `x IN %(ids)s` renders as `x IN (1,2,3)` instead of `x IN ARRAY(1,2,3)`, which Spark rejects. Inside the list, tuples render as rows for a multi-column `IN`, and a `set` or `frozenset` is accepted again. Sequences anywhere else still render as `ARRAY(...)` ([#629](https://github.com/databricks/databricks-sql-python/issues/629))
55
- Speed up Thrift response handling: read replies in 1 MiB chunks and decode them with Thrift's C-accelerated protocol, and stop stringifying every request/response for the default no-op debug logger. Wide result schemas, metadata calls with many rows, and inline Arrow results benefit most.
6+
- Speed up `Connection.close()` on Thrift when cursors are still open: it now sends only CloseSession, which releases the session's statements on the server, instead of first sending one CloseOperation per open cursor.
67

78
# 4.9.0 (2026-11-04)
89
- Upgrade Databricks SQL Kernel to 1.4.0, which keeps Lakehouse Real-Time session affinity and flushes shared telemetry only after the final session closes.

‎src/databricks/sql/client.py‎

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -763,7 +763,13 @@ def close(self) -> None:
763763
self._close()
764764

765765
def _close(self, close_cursors=True) -> None:
766-
if close_cursors:
766+
# Thrift CloseSession also releases the session's operations, so close the
767+
# session first: the cursors then close client-side only, without one
768+
# CloseOperation round trip each.
769+
close_session_first = (
770+
close_cursors and self.session.session_id.backend_type == BackendType.THRIFT
771+
)
772+
if close_cursors and not close_session_first:
767773
for cursor in self._cursors:
768774
cursor.close()
769775

@@ -772,6 +778,10 @@ def _close(self, close_cursors=True) -> None:
772778
except Exception as e:
773779
logger.error(f"Attempt to close session raised a local exception: {e}")
774780

781+
if close_session_first:
782+
for cursor in self._cursors:
783+
cursor.close()
784+
775785
TelemetryClientFactory.close(host_url=self.session.host)
776786

777787
# Close HTTP client that was created by this connection

‎tests/unit/test_client.py‎

Lines changed: 124 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,12 @@
3131
)
3232
from databricks.sql.types import Row
3333
from databricks.sql.result_set import ResultSet, ThriftResultSet
34-
from databricks.sql.backend.types import CommandId, CommandState
34+
from databricks.sql.backend.types import (
35+
BackendType,
36+
CommandId,
37+
CommandState,
38+
SessionId,
39+
)
3540
from databricks.sql.backend.types import ExecuteResponse
3641

3742
from tests.unit.test_fetches import FetchTests
@@ -97,14 +102,17 @@ def test_closing_connection_closes_commands(self, mock_thrift_client_class):
97102
This test verifies that when a connection is closed:
98103
1. the active result set is marked as closed server-side
99104
2. The operation state is set to CLOSED
100-
3. backend.close_command is called only for commands that weren't already closed
105+
3. backend.close_command is called only for commands that weren't already
106+
closed, and never on Thrift, where CloseSession releases them
101107
102108
Args:
103109
mock_thrift_client_class: Mock for ThriftBackend class
104110
"""
105111

106-
for closed in (True, False):
107-
with self.subTest(closed=closed):
112+
for backend_type, closed in itertools.product(
113+
(BackendType.THRIFT, BackendType.SEA), (True, False)
114+
):
115+
with self.subTest(backend_type=backend_type, closed=closed):
108116
# Set initial state based on whether the command is already closed
109117
initial_state = (
110118
CommandState.CLOSED if closed else CommandState.SUCCEEDED
@@ -122,6 +130,9 @@ def test_closing_connection_closes_commands(self, mock_thrift_client_class):
122130
mock_backend = Mock(spec=ThriftDatabricksClient)
123131
mock_backend.staging_allowed_local_path = None
124132
mock_backend.fetch_results.return_value = (Mock(), False, 0)
133+
mock_backend.open_session.return_value = SessionId(
134+
backend_type, b"\x01" * 16, b"\x02" * 16
135+
)
125136

126137
# Configure the decorator's mock to return our specific mock_backend
127138
mock_thrift_client_class.return_value = mock_backend
@@ -152,14 +163,121 @@ def test_closing_connection_closes_commands(self, mock_thrift_client_class):
152163
assert real_result_set.status == CommandState.CLOSED
153164

154165
# 3. Backend close_command should be called appropriately
155-
if not closed:
166+
if not closed and backend_type != BackendType.THRIFT:
156167
# Should have called backend.close_command during the close chain
157168
mock_backend.close_command.assert_called_once_with(
158169
mock_execute_response.command_id
159170
)
160171
else:
161-
# Should NOT have called backend.close_command (already closed)
172+
# Should NOT have called backend.close_command (already closed,
173+
# or released by CloseSession on Thrift)
162174
mock_backend.close_command.assert_not_called()
175+
mock_backend.close_session.assert_called_once()
176+
177+
def _connection_with_open_result_sets(self, mock_client_class, backend_type, n):
178+
"""Connect with a mocked backend and give n cursors a result set still open server-side."""
179+
calls = []
180+
mock_backend = Mock(spec=ThriftDatabricksClient)
181+
mock_backend.staging_allowed_local_path = None
182+
mock_backend.open_session.return_value = SessionId(
183+
backend_type, b"\x01" * 16, b"\x02" * 16
184+
)
185+
mock_backend.close_session.side_effect = lambda *_: calls.append(
186+
"close_session"
187+
)
188+
mock_backend.close_command.side_effect = lambda *_: calls.append(
189+
"close_command"
190+
)
191+
mock_client_class.return_value = mock_backend
192+
193+
connection = databricks.sql.connect(**self.DUMMY_CONNECTION_ARGS)
194+
cursors, result_sets = [], []
195+
for _ in range(n):
196+
execute_response = Mock(spec=ExecuteResponse)
197+
execute_response.status = CommandState.SUCCEEDED
198+
execute_response.has_been_closed_server_side = False
199+
execute_response.is_staging_operation = False
200+
execute_response.command_id = Mock(spec=CommandId)
201+
execute_response.description = []
202+
queue = Mock()
203+
queue.close.side_effect = lambda: calls.append("queue.close")
204+
mock_backend.fetch_results.return_value = (queue, False, 0)
205+
result_set = ThriftResultSet(
206+
connection=connection,
207+
execute_response=execute_response,
208+
thrift_client=mock_backend,
209+
)
210+
cursor = connection.cursor()
211+
cursor.backend.execute_command = Mock(return_value=result_set)
212+
cursor.execute("SELECT 1")
213+
cursors.append(cursor)
214+
result_sets.append(result_set)
215+
return connection, mock_backend, cursors, result_sets, calls
216+
217+
@patch("%s.session.ThriftDatabricksClient" % PACKAGE_NAME)
218+
def test_thrift_connection_close_closes_cursors_client_side_after_close_session(
219+
self, mock_client_class
220+
):
221+
connection, mock_backend, cursors, result_sets, calls = (
222+
self._connection_with_open_result_sets(
223+
mock_client_class, BackendType.THRIFT, 3
224+
)
225+
)
226+
227+
connection.close()
228+
229+
self.assertEqual(calls, ["close_session"] + ["queue.close"] * 3)
230+
mock_backend.close_command.assert_not_called()
231+
self.assertFalse(connection.open)
232+
for cursor, result_set in zip(cursors, result_sets):
233+
self.assertFalse(cursor.open)
234+
self.assertIsNone(cursor.active_result_set)
235+
self.assertIsNone(cursor.active_command_id)
236+
self.assertTrue(result_set.has_been_closed_server_side)
237+
self.assertEqual(result_set.status, CommandState.CLOSED)
238+
with self.assertRaises(InterfaceError):
239+
cursor.execute("SELECT 1")
240+
241+
@patch("%s.session.ThriftDatabricksClient" % PACKAGE_NAME)
242+
def test_non_thrift_connection_close_closes_each_command_first(
243+
self, mock_client_class
244+
):
245+
connection, mock_backend, cursors, result_sets, calls = (
246+
self._connection_with_open_result_sets(
247+
mock_client_class, BackendType.SEA, 2
248+
)
249+
)
250+
251+
connection.close()
252+
253+
self.assertEqual(
254+
calls, ["queue.close", "close_command"] * 2 + ["close_session"]
255+
)
256+
for cursor, result_set in zip(cursors, result_sets):
257+
self.assertFalse(cursor.open)
258+
self.assertEqual(result_set.status, CommandState.CLOSED)
259+
260+
@patch("%s.session.ThriftDatabricksClient" % PACKAGE_NAME)
261+
def test_thrift_connection_close_closes_cursors_when_close_session_fails(
262+
self, mock_client_class
263+
):
264+
connection, mock_backend, cursors, result_sets, calls = (
265+
self._connection_with_open_result_sets(
266+
mock_client_class, BackendType.THRIFT, 2
267+
)
268+
)
269+
mock_backend.close_session.side_effect = OSError("connection reset")
270+
271+
connection.close()
272+
273+
mock_backend.close_session.assert_called_once()
274+
mock_backend.close_command.assert_not_called()
275+
self.assertEqual(calls, ["queue.close"] * 2)
276+
self.assertFalse(connection.open)
277+
for cursor, result_set in zip(cursors, result_sets):
278+
self.assertFalse(cursor.open)
279+
self.assertTrue(result_set.has_been_closed_server_side)
280+
self.assertEqual(result_set.status, CommandState.CLOSED)
163281

164282
@patch("%s.session.ThriftDatabricksClient" % PACKAGE_NAME)
165283
def test_cant_open_cursor_on_closed_connection(self, mock_client_class):

0 commit comments

Comments
 (0)