Skip to content

Commit 916638b

Browse files
vuanhphungIsaac
andcommitted
test: pin that _disable_pandas never imports pandas; bound subprocess time
Build the test table from IPC bytes, since constructing it from Python lists makes pyarrow import pandas itself. Co-authored-by: Isaac <no-reply@databricks.com> Signed-off-by: Vu Anh Phung <vu.phung@databricks.com>
1 parent 1c07208 commit 916638b

1 file changed

Lines changed: 42 additions & 14 deletions

File tree

‎tests/unit/test_lazy_pandas_import.py‎

Lines changed: 42 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -25,9 +25,44 @@ def _run(script: str) -> subprocess.CompletedProcess:
2525
[sys.executable, "-c", textwrap.dedent(script)],
2626
capture_output=True,
2727
text=True,
28+
timeout=120,
2829
)
2930

3031

32+
def _table_ipc_hex() -> str:
33+
# Children read the table from IPC bytes: building it from Python lists
34+
# makes pyarrow import pandas itself, which would hide what the
35+
# conversion imports.
36+
import io
37+
38+
import pyarrow
39+
import pyarrow.ipc
40+
41+
table = pyarrow.table({"a": [1, None], "b": ["x", "y"]})
42+
sink = io.BytesIO()
43+
with pyarrow.ipc.new_stream(sink, table.schema) as writer:
44+
writer.write_table(table)
45+
return sink.getvalue().hex()
46+
47+
48+
def _convert_rows_script(disable_pandas: bool) -> str:
49+
return f"""
50+
import sys
51+
from unittest.mock import MagicMock
52+
import pyarrow.ipc
53+
from databricks.sql.result_set import ResultSet
54+
55+
table = pyarrow.ipc.open_stream(bytes.fromhex("{_table_ipc_hex()}")).read_all()
56+
assert "pandas" not in sys.modules
57+
rs = MagicMock()
58+
rs.connection.disable_pandas = {disable_pandas}
59+
rs.description = [("a",), ("b",)]
60+
rows = ResultSet._convert_arrow_table(rs, table)
61+
assert [tuple(r) for r in rows] == [(1, "x"), (None, "y")], rows
62+
print("pandas" in sys.modules)
63+
"""
64+
65+
3166
# Sentinel exit codes, distinct from the 1 of an uncaught exception.
3267
_NO_PANDAS, _PANDAS_LOADED, _MISSING_OPTIONAL_DEP = 10, 11, 12
3368
# Only a missing optional extra may skip; any other import error fails.
@@ -64,19 +99,12 @@ def test_connect_path_does_not_import_pandas(module_name):
6499

65100

66101
def test_row_conversion_imports_pandas_on_first_use():
67-
proc = _run("""
68-
import sys
69-
from unittest.mock import MagicMock
70-
import pyarrow
71-
from databricks.sql.result_set import ResultSet
102+
proc = _run(_convert_rows_script(disable_pandas=False))
103+
assert proc.returncode == 0, proc.stderr
104+
assert proc.stdout.strip() == "True", "the pandas path should import pandas"
72105

73-
assert "pandas" not in sys.modules
74-
rs = MagicMock()
75-
rs.connection.disable_pandas = False
76-
rs.description = [("a",), ("b",)]
77-
table = pyarrow.table({"a": [1, None], "b": ["x", "y"]})
78-
rows = ResultSet._convert_arrow_table(rs, table)
79-
assert "pandas" in sys.modules
80-
assert [tuple(r) for r in rows] == [(1, "x"), (None, "y")], rows
81-
""")
106+
107+
def test_disable_pandas_never_imports_pandas():
108+
proc = _run(_convert_rows_script(disable_pandas=True))
82109
assert proc.returncode == 0, proc.stderr
110+
assert proc.stdout.strip() == "False", "_disable_pandas=True imported pandas"

0 commit comments

Comments
 (0)