Skip to content

Commit 2dfe0ac

Browse files
vuanhphungIsaac
andcommitted
test(thrift): skip one-call decode assertions without the lz4 codec
Two tests assert that lz4.frame.decompress isn't called, which only holds when pyarrow has the lz4 codec; skip them otherwise, like the sibling tests. Reword the gate comment: without a map it limits the one-call decode by compressed size, so a highly compressible result can still decode into the heap. Signed-off-by: Vu Anh Phung <vu.phung@databricks.com> Co-authored-by: Isaac <no-reply@databricks.com>
1 parent 6d3787f commit 2dfe0ac

2 files changed

Lines changed: 9 additions & 3 deletions

File tree

‎src/databricks/sql/utils.py‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1056,8 +1056,9 @@ def convert_arrow_based_set_to_arrow_table(arrow_batches, lz4_compressed, schema
10561056
stream.append(schema_bytes)
10571057
n_rows = 0
10581058
# The Arrow path releases the GIL about twice, so it only helps from 3 batches.
1059-
# Large results take it only into a map (none on non-Linux, or if mmap failed):
1060-
# decoding them into the heap through Arrow used more memory than main with glibc.
1059+
# Without a map (non-Linux, or mmap failed) it is limited to results under 1 MiB
1060+
# compressed: decoding large ones into the heap through Arrow used more memory
1061+
# than main with glibc. A highly compressible result can still decode into the heap.
10611062
if (
10621063
lz4_compressed
10631064
and len(arrow_batches) > 2

‎tests/unit/test_util.py‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -319,6 +319,9 @@ def _patch_map(map_factory):
319319
linux_only = pytest.mark.skipif(
320320
not sys.platform.startswith("linux"), reason="inline Arrow maps are Linux-only"
321321
)
322+
needs_arrow_lz4 = pytest.mark.skipif(
323+
not utils._PYARROW_HAS_LZ4, reason="the one-call decode needs pyarrow's lz4 codec"
324+
)
322325

323326

324327
class _RecordingMap(mmap.mmap):
@@ -501,6 +504,7 @@ def recording_getbuffer(stream):
501504
assert states == [True]
502505

503506
@linux_only
507+
@needs_arrow_lz4
504508
def test_many_lz4_batches_decode_as_one_stream_into_the_map(self):
505509
table, schema_bytes, batches = _inline_arrow_set(72, 3000, True)
506510
assert sum(len(b.batch) for b in batches) >= utils._INLINE_ARROW_MMAP_MIN_BYTES
@@ -536,7 +540,8 @@ def test_two_large_lz4_batches_use_the_per_batch_path_into_the_map(self):
536540
assert decompress_spy.call_count == 2
537541

538542
@pytest.mark.parametrize(
539-
"rows_per_batch, expect_one_call", [(3000, False), (100, True)]
543+
"rows_per_batch, expect_one_call",
544+
[(3000, False), pytest.param(100, True, marks=needs_arrow_lz4)],
540545
)
541546
def test_without_the_map_only_small_results_use_the_one_call_decode(
542547
self, rows_per_batch, expect_one_call

0 commit comments

Comments
 (0)