diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index f6d42c8..210cf07 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -22,7 +22,7 @@ jobs: - name: Install uv uses: astral-sh/setup-uv@07a047e608db84b3d8c06cd9a7c7dcc9c355c18e # v10.3.2 with: - python-version: "3.13" + python-version: "3.13.18" enable-cache: true - name: Install dependencies diff --git a/python_multipart/multipart.py b/python_multipart/multipart.py index 5954586..75850de 100644 --- a/python_multipart/multipart.py +++ b/python_multipart/multipart.py @@ -1135,6 +1135,51 @@ def write(self, data: bytes) -> int: return l + def _data_callback(self, name: CallbackName, data: bytes, end_i: int, length: int, remaining: bool = False) -> None: + """Emit the data between the mark for `name` and `end_i` to the `name` callback. + + With `remaining`, the mark is kept and rebased onto the next chunk instead of being deleted. + `_internal_write` uses this at its end to flush what is left of the current chunk. + """ + boundary = self.boundary + marked_index = self.marks.get(name) + if marked_index is None: + return + + # Otherwise, we call it from the mark to the current byte we're + # processing. + if end_i <= marked_index: + # There is no additional data to send. + pass + elif marked_index >= 0: + # We are emitting data from the local buffer. + self.callback(name, data, marked_index, end_i) + else: + # Some of the data comes from a partial boundary match. + # and requires look-behind. + # We need to use self.flags (and not flags) because we care about + # the state when we entered the loop. + lookbehind_len = -marked_index + if lookbehind_len <= len(boundary): + self.callback(name, boundary, 0, lookbehind_len) + elif self.flags & FLAG_PART_BOUNDARY: + lookback = boundary + b"\r\n" + self.callback(name, lookback, 0, lookbehind_len) + elif self.flags & FLAG_LAST_BOUNDARY: + lookback = boundary + b"--\r\n" + self.callback(name, lookback, 0, lookbehind_len) + else: # pragma: no cover (error case) + self.logger.warning("Look-back buffer error") + + if end_i > 0: + self.callback(name, data, 0, end_i) + # If we're getting remaining data, we have got all the data we + # can be certain is not a boundary, leaving only a partial boundary match. + if remaining: + self.marks[name] = end_i - length + else: + self.marks.pop(name, None) + def _internal_write(self, data: bytes, length: int) -> int: # Get values from locals. boundary = self.boundary @@ -1165,50 +1210,6 @@ def set_mark(name: str) -> None: def delete_mark(name: str, reset: bool = False) -> None: self.marks.pop(name, None) - # Helper function that makes calling a callback with data easier. The - # 'remaining' parameter will callback from the marked value until the - # end of the buffer, and reset the mark, instead of deleting it. This - # is used at the end of the function to call our callbacks with any - # remaining data in this chunk. - def data_callback(name: CallbackName, end_i: int, remaining: bool = False) -> None: - marked_index = self.marks.get(name) - if marked_index is None: - return - - # Otherwise, we call it from the mark to the current byte we're - # processing. - if end_i <= marked_index: - # There is no additional data to send. - pass - elif marked_index >= 0: - # We are emitting data from the local buffer. - self.callback(name, data, marked_index, end_i) - else: - # Some of the data comes from a partial boundary match. - # and requires look-behind. - # We need to use self.flags (and not flags) because we care about - # the state when we entered the loop. - lookbehind_len = -marked_index - if lookbehind_len <= boundary_length: - self.callback(name, boundary, 0, lookbehind_len) - elif self.flags & FLAG_PART_BOUNDARY: - lookback = boundary + b"\r\n" - self.callback(name, lookback, 0, lookbehind_len) - elif self.flags & FLAG_LAST_BOUNDARY: - lookback = boundary + b"--\r\n" - self.callback(name, lookback, 0, lookbehind_len) - else: # pragma: no cover (error case) - self.logger.warning("Look-back buffer error") - - if end_i > 0: - self.callback(name, data, 0, end_i) - # If we're getting remaining data, we have got all the data we - # can be certain is not a boundary, leaving only a partial boundary match. - if remaining: - self.marks[name] = end_i - length - else: - self.marks.pop(name, None) - # For each byte... while i < length: c = data[i] @@ -1343,7 +1344,7 @@ def data_callback(name: CallbackName, end_i: int, remaining: bool = False) -> No # Call our callback with the header field. i = colon - data_callback("header_field", i) + self._data_callback("header_field", data, i, length) # Move to parsing the header value. state = HEADER_VALUE_START @@ -1370,7 +1371,7 @@ def data_callback(name: CallbackName, end_i: int, remaining: bool = False) -> No advance_header_size(end - i) if cr != -1: i = cr - data_callback("header_value", i) + self._data_callback("header_value", data, i, length) self.callback("header_end") current_header_size = 0 state = HEADER_VALUE_ALMOST_DONE @@ -1470,7 +1471,7 @@ def data_callback(name: CallbackName, end_i: int, remaining: bool = False) -> No # RFC 2046 forbids lines starting with the boundary in a part, so this can't be data. elif c in b" \t": - data_callback("part_data", i - boundary_length) + self._data_callback("part_data", data, i - boundary_length, length) delete_mark("part_data") index = 0 state = BOUNDARY_PADDING @@ -1492,7 +1493,7 @@ def data_callback(name: CallbackName, end_i: int, remaining: bool = False) -> No flags &= ~FLAG_PART_BOUNDARY # We have identified a boundary, callback for any data before it. - data_callback("part_data", i - index) + self._data_callback("part_data", data, i - index, length) # Callback indicating that we've reached the end of # a part, and are starting a new one. self.callback("part_end") @@ -1517,7 +1518,7 @@ def data_callback(name: CallbackName, end_i: int, remaining: bool = False) -> No # We need a second hyphen here. if c == HYPHEN: # We have identified a boundary, callback for any data before it. - data_callback("part_data", i - index) + self._data_callback("part_data", data, i - index, length) # Callback to end the current part, and then the # message. self.callback("part_end") @@ -1587,9 +1588,9 @@ def data_callback(name: CallbackName, end_i: int, remaining: bool = False) -> No # that we haven't yet reached the end of this 'thing'. So, by setting # the mark to 0, we cause any data callbacks that take place in future # calls to this function to start from the beginning of that buffer. - data_callback("header_field", length, True) - data_callback("header_value", length, True) - data_callback("part_data", length - index, True) + self._data_callback("header_field", data, length, length, True) + self._data_callback("header_value", data, length, length, True) + self._data_callback("part_data", data, length - index, length, True) # Save values to locals. self.state = state diff --git a/tests/test_multipart.py b/tests/test_multipart.py index 31b84e3..b6fe74e 100644 --- a/tests/test_multipart.py +++ b/tests/test_multipart.py @@ -913,6 +913,42 @@ def test_multipart_opening_boundary_max_size() -> None: assert events == ["begin", "end"] +@pytest.mark.parametrize("chunk_size", [*range(1, 32), 1024]) +def test_multipart_callbacks_across_chunks(chunk_size: int) -> None: + body = b"first\r\n--boundaryX\r\n--boundary\rX\r\n--boundary-X" + data = ( + b"--boundary\r\nX-Test: value\r\n\r\n" + body + b"\r\n" + b"--boundary\r\nX-Test: other\r\n\r\nsecond\r\n--boundary--\r\n" + ) + events: list[bytes] = [] + + def on_data(data: bytes, start: int, end: int) -> None: + events.append(data[start:end]) + + parser = MultipartParser( + b"boundary", + { + "on_part_begin": lambda: events.append(b""), + "on_header_begin": lambda: events.append(b"
"), + "on_header_field": on_data, + "on_header_value": on_data, + "on_header_end": lambda: events.append(b"
"), + "on_headers_finished": lambda: events.append(b""), + "on_part_data": on_data, + "on_part_end": lambda: events.append(b"
"), + "on_end": lambda: events.append(b""), + }, + ) + for start in range(0, len(data), chunk_size): + chunk = data[start : start + chunk_size] + assert parser.write(chunk) == len(chunk) + parser.finalize() + assert b"".join(events) == ( + b"
X-Testvalue
" + body + b"
" + b"
X-Testother
second
" + ) + + @pytest.mark.parametrize( "config,first,second,message", [