Skip to content

Commit ccdbbf4

Browse files
vuanhphungIsaac
andcommitted
fix(thrift): don't grow an exactly filled inline Arrow map
When the decoded stream filled the reservation exactly, append_stream grew the map before seeing EOF: a wasted mremap that, under memory pressure, could push a finished result onto the heap. Probe with a one-byte read and grow only if more data follows. Reuse _LZ4_RATIO_ESTIMATE for the first read size. Signed-off-by: Vu Anh Phung <vu.phung@databricks.com> Co-authored-by: Isaac <no-reply@databricks.com>
1 parent 2dfe0ac commit ccdbbf4

2 files changed

Lines changed: 23 additions & 2 deletions

File tree

‎src/databricks/sql/utils.py‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1030,7 +1030,7 @@ def _decompress_lz4_frames_into(out: "_ArrowStreamBuffer", frames: List[bytes])
10301030
# release can wait a full switch interval.
10311031
# A temporary copy of the compressed row set, freed once it is decoded.
10321032
compressed = b"".join(frames)
1033-
nbytes = min(4 * len(compressed) + 65536, _LZ4_MAX_READ_BYTES)
1033+
nbytes = min(_LZ4_RATIO_ESTIMATE * len(compressed) + 65536, _LZ4_MAX_READ_BYTES)
10341034
try:
10351035
with pyarrow.CompressedInputStream(
10361036
pyarrow.BufferReader(compressed), "lz4"
@@ -1122,7 +1122,11 @@ def append_stream(self, stream, chunk_size: int) -> None:
11221122
self.append(chunk)
11231123
continue
11241124
if self._size == len(self._map):
1125-
self._ensure_capacity(self._size + 1)
1125+
# Grow only if the stream has more: an exact fill must not resize
1126+
probe = stream.read(1)
1127+
if not probe:
1128+
return
1129+
self.append(probe)
11261130
continue
11271131
with memoryview(self._map) as view:
11281132
n = stream.readinto(view[self._size :])

‎tests/unit/test_util.py‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -412,6 +412,23 @@ def recording_getbuffer(stream):
412412
assert map_size > buffer_size == _stream_size(schema_bytes, batches)
413413
assert result.equals(table)
414414

415+
@linux_only
416+
@needs_arrow_lz4
417+
def test_exactly_filled_map_is_not_resized(self):
418+
table, schema_bytes, batches = _large_inline_arrow_set()
419+
stream_size = _stream_size(schema_bytes, batches)
420+
_RecordingMap.resizes = []
421+
422+
with _patch_map(_RecordingMap):
423+
buffer = utils._ArrowStreamBuffer(stream_size, stream_size)
424+
buffer.append(schema_bytes)
425+
utils._decompress_lz4_frames_into(buffer, [b.batch for b in batches])
426+
result = pyarrow.ipc.open_stream(buffer.getbuffer()).read_all()
427+
428+
assert buffer.mapped
429+
assert _RecordingMap.resizes == []
430+
assert result.equals(table)
431+
415432
@linux_only
416433
@pytest.mark.parametrize(
417434
"error",

0 commit comments

Comments
 (0)