Skip to content

Commit 51214d2

Browse files
vuanhphungIsaac
andcommitted
chore(kernel): log when the pandas-free conversion falls back on an error
Review feedback: a to_pylist() failure silently switched to the pandas path, so a pyarrow regression that broke it for a common type would only show up as a slowdown. Log the fallback at debug level with the traceback; the catch stays broad so pandas keeps deciding the result or error. Signed-off-by: Vu Anh Phung <vu.phung@databricks.com> Co-authored-by: Isaac <no-reply@databricks.com>
1 parent 5cd7941 commit 51214d2

2 files changed

Lines changed: 15 additions & 0 deletions

File tree

‎src/databricks/sql/backend/kernel/result_set.py‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -139,6 +139,11 @@ def _rows_without_pandas(
139139
except Exception:
140140
# E.g. a date past 9999-12-31: pandas then raises its own error,
141141
# as it does without this fast path.
142+
logger.debug(
143+
"to_pylist() failed for a %s column; using pandas",
144+
column.type,
145+
exc_info=True,
146+
)
142147
return None
143148
if column.type.id in _FLOAT_TYPE_IDS and any(v != v for v in values):
144149
return None

‎tests/unit/test_kernel_result_set.py‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
from __future__ import annotations
66

77
import datetime
8+
import logging
89
from collections import deque
910
from decimal import Decimal
1011
from typing import Deque
@@ -483,6 +484,15 @@ def test_out_of_range_values_convert_as_on_the_pandas_path(column, expected):
483484
convert(table)
484485

485486

487+
def test_conversion_errors_are_logged_before_falling_back(caplog):
488+
table = pa.table({"c": pa.array([2_932_897], pa.date32())})
489+
rs = _make_rs(_FakeKernelHandle(table.schema, []))
490+
with caplog.at_level(logging.DEBUG, logger=kernel_result_set.__name__):
491+
with pytest.raises(ValueError):
492+
rs._convert_arrow_table(table)
493+
assert "to_pylist() failed for a date32[day] column" in caplog.text
494+
495+
486496
def test_empty_results_skip_pandas():
487497
rows, expected, used_pandas = _convert(pa.table({"n": pa.array([], pa.int64())}))
488498
assert rows == expected == []

0 commit comments

Comments
 (0)