|
4 | 4 |
|
5 | 5 | from __future__ import annotations |
6 | 6 |
|
| 7 | +import datetime |
7 | 8 | from collections import deque |
| 9 | +from decimal import Decimal |
8 | 10 | from typing import Deque |
9 | | -from unittest.mock import MagicMock |
| 11 | +from unittest.mock import MagicMock, patch |
10 | 12 |
|
| 13 | +import numpy |
11 | 14 | import pytest |
12 | 15 |
|
13 | 16 | # pyarrow is an optional connector dep; the default-deps CI test |
14 | 17 | # job runs without it. KernelResultSet imports pyarrow eagerly, |
15 | 18 | # so the whole module must skip when pyarrow is unavailable. |
16 | 19 | pa = pytest.importorskip("pyarrow") |
17 | 20 |
|
| 21 | +from databricks.sql.backend.kernel import result_set as kernel_result_set |
18 | 22 | from databricks.sql.backend.kernel.result_set import KernelResultSet |
19 | 23 | from databricks.sql.backend.types import CommandId, CommandState |
| 24 | +from databricks.sql.result_set import ResultSet |
20 | 25 |
|
21 | 26 |
|
22 | 27 | class _FakeKernelHandle: |
@@ -243,3 +248,206 @@ def test_close_skips_kernel_call_when_connection_already_closed(int_schema): |
243 | 248 | # …but local state is still terminal so __del__ is idempotent: |
244 | 249 | assert rs.status == CommandState.CLOSED |
245 | 250 | assert rs.has_been_closed_server_side is True |
| 251 | + |
| 252 | + |
| 253 | +# ----- small results converted without pandas ----- |
| 254 | + |
| 255 | +_UTC = datetime.timezone.utc |
| 256 | + |
| 257 | + |
| 258 | +def _columns_skipping_pandas(): |
| 259 | + return { |
| 260 | + "i8": pa.array([1, None, -128, 127], pa.int8()), |
| 261 | + "i16": pa.array([1, None, -(2**15), 2**15 - 1], pa.int16()), |
| 262 | + "i32": pa.array([1, None, -(2**31), 2**31 - 1], pa.int32()), |
| 263 | + "i64": pa.array([1, None, -(2**63), 2**63 - 1], pa.int64()), |
| 264 | + "u8": pa.array([1, None, 0, 255], pa.uint8()), |
| 265 | + "u64": pa.array([1, None, 0, 2**64 - 1], pa.uint64()), |
| 266 | + "bool": pa.array([True, None, False, True]), |
| 267 | + "f32": pa.array([0.1, None, float("inf"), -0.0], pa.float32()), |
| 268 | + "f64": pa.array([0.1, None, float("-inf"), -0.0], pa.float64()), |
| 269 | + "str": pa.array(["a", None, "", "é\U0001f600"]), |
| 270 | + "large_str": pa.array(["a", None, "", "z"], pa.large_string()), |
| 271 | + "bin": pa.array([b"a", None, b"", b"\x00\xff"]), |
| 272 | + "large_bin": pa.array([b"a", None, b"", b"\x00"], pa.large_binary()), |
| 273 | + "fixed_bin": pa.array([b"ab", None, b"cd", b"\x00\x00"], pa.binary(2)), |
| 274 | + "dec": pa.array( |
| 275 | + [Decimal("123141078.2283"), None, Decimal("-0.0001"), Decimal("0.0000")], |
| 276 | + pa.decimal128(38, 4), |
| 277 | + ), |
| 278 | + "dec256": pa.array( |
| 279 | + [Decimal("1.5"), None, Decimal("-2.5"), Decimal("0.0")], |
| 280 | + pa.decimal256(40, 1), |
| 281 | + ), |
| 282 | + "date": pa.array( |
| 283 | + [ |
| 284 | + datetime.date(2024, 1, 1), |
| 285 | + None, |
| 286 | + datetime.date(1, 1, 1), |
| 287 | + datetime.date(9999, 12, 31), |
| 288 | + ] |
| 289 | + ), |
| 290 | + "date64": pa.array( |
| 291 | + [datetime.date(2024, 2, 29), None, datetime.date(1970, 1, 1), None], |
| 292 | + pa.date64(), |
| 293 | + ), |
| 294 | + "ts": pa.array( |
| 295 | + [ |
| 296 | + datetime.datetime(2024, 1, 1, 1, 2, 3, 4), |
| 297 | + None, |
| 298 | + datetime.datetime(1, 1, 1), |
| 299 | + datetime.datetime(9999, 12, 31, 23, 59, 59, 999999), |
| 300 | + ], |
| 301 | + pa.timestamp("us"), |
| 302 | + ), |
| 303 | + "ts_ms": pa.array([0, None, 1, 2], pa.timestamp("ms")), |
| 304 | + "ts_utc": pa.array( |
| 305 | + [ |
| 306 | + datetime.datetime(2024, 1, 1, tzinfo=_UTC), |
| 307 | + None, |
| 308 | + datetime.datetime(1, 1, 1, tzinfo=_UTC), |
| 309 | + datetime.datetime(2024, 3, 10, 10, tzinfo=_UTC), |
| 310 | + ], |
| 311 | + pa.timestamp("us", tz="Etc/UTC"), |
| 312 | + ), |
| 313 | + # Includes the instant when DST starts. |
| 314 | + "ts_la": pa.array( |
| 315 | + [0, None, 1710064800000000, 1699174800000000], |
| 316 | + pa.timestamp("us", tz="America/Los_Angeles"), |
| 317 | + ), |
| 318 | + "ts_offset": pa.array([0, None, 1, 2], pa.timestamp("us", tz="+05:30")), |
| 319 | + "time": pa.array([0, None, 1, 86399999999], pa.time64("us")), |
| 320 | + "time32": pa.array([0, None, 1, 2], pa.time32("ms")), |
| 321 | + "null": pa.nulls(4), |
| 322 | + } |
| 323 | + |
| 324 | + |
| 325 | +def _columns_needing_pandas(): |
| 326 | + return { |
| 327 | + # pandas maps NaN to None or keeps it, depending on its version. |
| 328 | + "f64_nan": pa.array([float("nan"), None, 1.0, 2.0]), |
| 329 | + "f32_nan": pa.array([1.0, float("nan"), None, 2.0], pa.float32()), |
| 330 | + "ts_ns": pa.array([1, None, 1000, 2], pa.timestamp("ns")), |
| 331 | + "time_ns": pa.array([1000, None, 2000, 0], pa.time64("ns")), |
| 332 | + "duration": pa.array([0, None, 1, 2], pa.duration("us")), |
| 333 | + "interval": pa.array( |
| 334 | + [pa.MonthDayNano([1, 2, 3]), None, pa.MonthDayNano([0, 0, 0]), None], |
| 335 | + pa.month_day_nano_interval(), |
| 336 | + ), |
| 337 | + "list": pa.array([[1, 2], None, [], [None]], pa.list_(pa.int64())), |
| 338 | + "struct": pa.array( |
| 339 | + [{"x": 1}, None, {"x": None}, {"x": 2}], pa.struct([("x", pa.int64())]) |
| 340 | + ), |
| 341 | + "map": pa.array( |
| 342 | + [[("a", 1)], None, [], [("b", None)]], pa.map_(pa.string(), pa.int64()) |
| 343 | + ), |
| 344 | + "dictionary": pa.array(["a", None, "a", "b"]).dictionary_encode(), |
| 345 | + "f16": pa.array( |
| 346 | + numpy.array([1, 0, 2, 3], numpy.float16), |
| 347 | + mask=numpy.array([False, True, False, False]), |
| 348 | + ), |
| 349 | + } |
| 350 | + |
| 351 | + |
| 352 | +def _assert_same_rows(rows, expected): |
| 353 | + assert len(rows) == len(expected) |
| 354 | + for row, expected_row in zip(rows, expected): |
| 355 | + assert row.__fields__ == expected_row.__fields__ |
| 356 | + assert len(row) == len(expected_row) |
| 357 | + for value, expected_value in zip(row, expected_row): |
| 358 | + assert type(value) is type(expected_value) |
| 359 | + assert repr(value) == repr(expected_value) |
| 360 | + if isinstance(value, datetime.datetime): |
| 361 | + assert type(value.tzinfo) is type(expected_value.tzinfo) |
| 362 | + assert value.tzinfo == expected_value.tzinfo |
| 363 | + |
| 364 | + |
| 365 | +def _convert(table): |
| 366 | + """(rows from KernelResultSet, rows from the pandas path, whether the |
| 367 | + kernel result set used the pandas path).""" |
| 368 | + rs = _make_rs(_FakeKernelHandle(table.schema, [])) |
| 369 | + pandas_path = ResultSet._convert_arrow_table |
| 370 | + with patch.object( |
| 371 | + ResultSet, "_convert_arrow_table", autospec=True, side_effect=pandas_path |
| 372 | + ) as spy: |
| 373 | + rows = rs._convert_arrow_table(table) |
| 374 | + return rows, pandas_path(rs, table), spy.called |
| 375 | + |
| 376 | + |
| 377 | +@pytest.mark.parametrize("name", sorted(_columns_skipping_pandas())) |
| 378 | +def test_small_results_skip_pandas_with_the_same_values(name): |
| 379 | + column = _columns_skipping_pandas()[name] |
| 380 | + tables = [ |
| 381 | + pa.table({name: column}), |
| 382 | + pa.table({name: column.slice(0, 1)}), |
| 383 | + pa.table({name: column.slice(1, 1)}), |
| 384 | + # Chunked, next to other columns, with a duplicate column name. |
| 385 | + pa.table( |
| 386 | + [ |
| 387 | + pa.chunked_array([[1, 2], [3, 4]]), |
| 388 | + pa.chunked_array([column.slice(0, 1), column.slice(1)]), |
| 389 | + pa.array(["a", "b", None, "d"]), |
| 390 | + ], |
| 391 | + names=["k", name, "k"], |
| 392 | + ), |
| 393 | + ] |
| 394 | + for table in tables: |
| 395 | + rows, expected, used_pandas = _convert(table) |
| 396 | + assert not used_pandas |
| 397 | + _assert_same_rows(rows, expected) |
| 398 | + |
| 399 | + |
| 400 | +@pytest.mark.parametrize("name", sorted(_columns_needing_pandas())) |
| 401 | +def test_results_whose_values_could_differ_use_pandas(name): |
| 402 | + table = pa.table( |
| 403 | + {"k": pa.array([1, 2, 3, 4]), name: _columns_needing_pandas()[name]} |
| 404 | + ) |
| 405 | + rows, expected, used_pandas = _convert(table) |
| 406 | + assert used_pandas |
| 407 | + assert len(rows) == len(expected) == 4 |
| 408 | + |
| 409 | + |
| 410 | +def test_large_results_use_pandas(): |
| 411 | + limit = kernel_result_set._MAX_ROWS_WITHOUT_PANDAS |
| 412 | + for num_rows, expect_pandas in ((limit, False), (limit + 1, True)): |
| 413 | + rows, expected, used_pandas = _convert(pa.table({"n": list(range(num_rows))})) |
| 414 | + assert used_pandas is expect_pandas |
| 415 | + _assert_same_rows(rows, expected) |
| 416 | + |
| 417 | + |
| 418 | +def test_tz_aware_timestamps_skip_pandas_only_in_tiny_results(): |
| 419 | + limit = kernel_result_set._MAX_ROWS_WITHOUT_PANDAS_TZ |
| 420 | + for tz, num_rows, expect_pandas in ( |
| 421 | + ("Etc/UTC", limit, False), |
| 422 | + ("Etc/UTC", limit + 1, True), |
| 423 | + (None, limit + 1, False), |
| 424 | + ): |
| 425 | + column = pa.array(range(num_rows), pa.timestamp("us", tz=tz)) |
| 426 | + rows, expected, used_pandas = _convert(pa.table({"t": column})) |
| 427 | + assert used_pandas is expect_pandas |
| 428 | + _assert_same_rows(rows, expected) |
| 429 | + |
| 430 | + |
| 431 | +def test_empty_results_skip_pandas(): |
| 432 | + rows, expected, used_pandas = _convert(pa.table({"n": pa.array([], pa.int64())})) |
| 433 | + assert rows == expected == [] |
| 434 | + assert not used_pandas |
| 435 | + |
| 436 | + |
| 437 | +def test_fetch_methods_return_the_pandas_path_values(): |
| 438 | + schema = pa.schema([("d", pa.decimal128(38, 4)), ("t", pa.timestamp("us", "UTC"))]) |
| 439 | + batch = pa.RecordBatch.from_arrays( |
| 440 | + [ |
| 441 | + pa.array( |
| 442 | + [Decimal("1.5000"), None, Decimal("-2.0001")], schema.field(0).type |
| 443 | + ), |
| 444 | + pa.array([0, 1, None], schema.field(1).type), |
| 445 | + ], |
| 446 | + schema=schema, |
| 447 | + ) |
| 448 | + expected = ResultSet._convert_arrow_table( |
| 449 | + _make_rs(_FakeKernelHandle(schema, [])), pa.Table.from_batches([batch]) |
| 450 | + ) |
| 451 | + _assert_same_rows(_make_rs(_FakeKernelHandle(schema, [batch])).fetchall(), expected) |
| 452 | + rs = _make_rs(_FakeKernelHandle(schema, [batch])) |
| 453 | + _assert_same_rows([rs.fetchone()] + rs.fetchmany(5), expected) |
0 commit comments