Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .github/workflows/benchmark.yml
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ jobs:
- name: Install uv
uses: astral-sh/setup-uv@07a047e608db84b3d8c06cd9a7c7dcc9c355c18e # v10.3.2
with:
python-version: "3.13"
python-version: "3.13.18"
enable-cache: true

- name: Install dependencies
Expand Down
105 changes: 53 additions & 52 deletions python_multipart/multipart.py
Original file line number Diff line number Diff line change
Expand Up @@ -1135,6 +1135,51 @@ def write(self, data: bytes) -> int:

return l

def _data_callback(self, name: CallbackName, data: bytes, end_i: int, length: int, remaining: bool = False) -> None:
"""Emit the data between the mark for `name` and `end_i` to the `name` callback.

With `remaining`, the mark is kept and rebased onto the next chunk instead of being deleted.
`_internal_write` uses this at its end to flush what is left of the current chunk.
"""
boundary = self.boundary
marked_index = self.marks.get(name)
if marked_index is None:
return

# Otherwise, we call it from the mark to the current byte we're
# processing.
if end_i <= marked_index:
# There is no additional data to send.
pass
elif marked_index >= 0:
# We are emitting data from the local buffer.
self.callback(name, data, marked_index, end_i)
else:
# Some of the data comes from a partial boundary match.
# and requires look-behind.
# We need to use self.flags (and not flags) because we care about
# the state when we entered the loop.
lookbehind_len = -marked_index
if lookbehind_len <= len(boundary):
self.callback(name, boundary, 0, lookbehind_len)
elif self.flags & FLAG_PART_BOUNDARY:
lookback = boundary + b"\r\n"
self.callback(name, lookback, 0, lookbehind_len)
elif self.flags & FLAG_LAST_BOUNDARY:
lookback = boundary + b"--\r\n"
self.callback(name, lookback, 0, lookbehind_len)
else: # pragma: no cover (error case)
self.logger.warning("Look-back buffer error")

if end_i > 0:
self.callback(name, data, 0, end_i)
# If we're getting remaining data, we have got all the data we
# can be certain is not a boundary, leaving only a partial boundary match.
if remaining:
self.marks[name] = end_i - length
else:
self.marks.pop(name, None)

def _internal_write(self, data: bytes, length: int) -> int:
# Get values from locals.
boundary = self.boundary
Expand Down Expand Up @@ -1165,50 +1210,6 @@ def set_mark(name: str) -> None:
def delete_mark(name: str, reset: bool = False) -> None:
self.marks.pop(name, None)

# Helper function that makes calling a callback with data easier. The
# 'remaining' parameter will callback from the marked value until the
# end of the buffer, and reset the mark, instead of deleting it. This
# is used at the end of the function to call our callbacks with any
# remaining data in this chunk.
def data_callback(name: CallbackName, end_i: int, remaining: bool = False) -> None:
marked_index = self.marks.get(name)
if marked_index is None:
return

# Otherwise, we call it from the mark to the current byte we're
# processing.
if end_i <= marked_index:
# There is no additional data to send.
pass
elif marked_index >= 0:
# We are emitting data from the local buffer.
self.callback(name, data, marked_index, end_i)
else:
# Some of the data comes from a partial boundary match.
# and requires look-behind.
# We need to use self.flags (and not flags) because we care about
# the state when we entered the loop.
lookbehind_len = -marked_index
if lookbehind_len <= boundary_length:
self.callback(name, boundary, 0, lookbehind_len)
elif self.flags & FLAG_PART_BOUNDARY:
lookback = boundary + b"\r\n"
self.callback(name, lookback, 0, lookbehind_len)
elif self.flags & FLAG_LAST_BOUNDARY:
lookback = boundary + b"--\r\n"
self.callback(name, lookback, 0, lookbehind_len)
else: # pragma: no cover (error case)
self.logger.warning("Look-back buffer error")

if end_i > 0:
self.callback(name, data, 0, end_i)
# If we're getting remaining data, we have got all the data we
# can be certain is not a boundary, leaving only a partial boundary match.
if remaining:
self.marks[name] = end_i - length
else:
self.marks.pop(name, None)

# For each byte...
while i < length:
c = data[i]
Expand Down Expand Up @@ -1343,7 +1344,7 @@ def data_callback(name: CallbackName, end_i: int, remaining: bool = False) -> No

# Call our callback with the header field.
i = colon
data_callback("header_field", i)
self._data_callback("header_field", data, i, length)

# Move to parsing the header value.
state = HEADER_VALUE_START
Expand All @@ -1370,7 +1371,7 @@ def data_callback(name: CallbackName, end_i: int, remaining: bool = False) -> No
advance_header_size(end - i)
if cr != -1:
i = cr
data_callback("header_value", i)
self._data_callback("header_value", data, i, length)
self.callback("header_end")
current_header_size = 0
state = HEADER_VALUE_ALMOST_DONE
Expand Down Expand Up @@ -1470,7 +1471,7 @@ def data_callback(name: CallbackName, end_i: int, remaining: bool = False) -> No

# RFC 2046 forbids lines starting with the boundary in a part, so this can't be data.
elif c in b" \t":
data_callback("part_data", i - boundary_length)
self._data_callback("part_data", data, i - boundary_length, length)
delete_mark("part_data")
index = 0
state = BOUNDARY_PADDING
Expand All @@ -1492,7 +1493,7 @@ def data_callback(name: CallbackName, end_i: int, remaining: bool = False) -> No
flags &= ~FLAG_PART_BOUNDARY

# We have identified a boundary, callback for any data before it.
data_callback("part_data", i - index)
self._data_callback("part_data", data, i - index, length)
# Callback indicating that we've reached the end of
# a part, and are starting a new one.
self.callback("part_end")
Expand All @@ -1517,7 +1518,7 @@ def data_callback(name: CallbackName, end_i: int, remaining: bool = False) -> No
# We need a second hyphen here.
if c == HYPHEN:
# We have identified a boundary, callback for any data before it.
data_callback("part_data", i - index)
self._data_callback("part_data", data, i - index, length)
# Callback to end the current part, and then the
# message.
self.callback("part_end")
Expand Down Expand Up @@ -1587,9 +1588,9 @@ def data_callback(name: CallbackName, end_i: int, remaining: bool = False) -> No
# that we haven't yet reached the end of this 'thing'. So, by setting
# the mark to 0, we cause any data callbacks that take place in future
# calls to this function to start from the beginning of that buffer.
data_callback("header_field", length, True)
data_callback("header_value", length, True)
data_callback("part_data", length - index, True)
self._data_callback("header_field", data, length, length, True)
self._data_callback("header_value", data, length, length, True)
self._data_callback("part_data", data, length - index, length, True)

# Save values to locals.
self.state = state
Expand Down
36 changes: 36 additions & 0 deletions tests/test_multipart.py
Original file line number Diff line number Diff line change
Expand Up @@ -913,6 +913,42 @@ def test_multipart_opening_boundary_max_size() -> None:
assert events == ["begin", "end"]


@pytest.mark.parametrize("chunk_size", [*range(1, 32), 1024])
def test_multipart_callbacks_across_chunks(chunk_size: int) -> None:
body = b"first\r\n--boundaryX\r\n--boundary\rX\r\n--boundary-X"
data = (
b"--boundary\r\nX-Test: value\r\n\r\n" + body + b"\r\n"
b"--boundary\r\nX-Test: other\r\n\r\nsecond\r\n--boundary--\r\n"
)
events: list[bytes] = []

def on_data(data: bytes, start: int, end: int) -> None:
events.append(data[start:end])

parser = MultipartParser(
b"boundary",
{
"on_part_begin": lambda: events.append(b"<part>"),
"on_header_begin": lambda: events.append(b"<header>"),
"on_header_field": on_data,
"on_header_value": on_data,
"on_header_end": lambda: events.append(b"</header>"),
"on_headers_finished": lambda: events.append(b"<data>"),
"on_part_data": on_data,
"on_part_end": lambda: events.append(b"</part>"),
"on_end": lambda: events.append(b"<end>"),
},
)
for start in range(0, len(data), chunk_size):
chunk = data[start : start + chunk_size]
assert parser.write(chunk) == len(chunk)
parser.finalize()
assert b"".join(events) == (
b"<part><header>X-Testvalue</header><data>" + body + b"</part>"
b"<part><header>X-Testother</header><data>second</part><end>"
)


@pytest.mark.parametrize(
"config,first,second,message",
[
Expand Down
Loading