From 0f6eb1e4d40d396afb2f087901b2c1e34081edc5 Mon Sep 17 00:00:00 2001 From: Marcelo Trylesinski Date: Mon, 16 Nov 2026 09:20:33 +0100 Subject: [PATCH 1/4] Extract multipart data callbacks into a method --- python_multipart/multipart.py | 105 +++++++++++++++++----------------- tests/test_multipart.py | 36 ++++++++++++ 2 files changed, 89 insertions(+), 52 deletions(-) diff --git a/python_multipart/multipart.py b/python_multipart/multipart.py index 5954586..75850de 100644 --- a/python_multipart/multipart.py +++ b/python_multipart/multipart.py @@ -1135,6 +1135,51 @@ def write(self, data: bytes) -> int: return l + def _data_callback(self, name: CallbackName, data: bytes, end_i: int, length: int, remaining: bool = False) -> None: + """Emit the data between the mark for `name` and `end_i` to the `name` callback. + + With `remaining`, the mark is kept and rebased onto the next chunk instead of being deleted. + `_internal_write` uses this at its end to flush what is left of the current chunk. + """ + boundary = self.boundary + marked_index = self.marks.get(name) + if marked_index is None: + return + + # Otherwise, we call it from the mark to the current byte we're + # processing. + if end_i <= marked_index: + # There is no additional data to send. + pass + elif marked_index >= 0: + # We are emitting data from the local buffer. + self.callback(name, data, marked_index, end_i) + else: + # Some of the data comes from a partial boundary match. + # and requires look-behind. + # We need to use self.flags (and not flags) because we care about + # the state when we entered the loop. + lookbehind_len = -marked_index + if lookbehind_len <= len(boundary): + self.callback(name, boundary, 0, lookbehind_len) + elif self.flags & FLAG_PART_BOUNDARY: + lookback = boundary + b"\r\n" + self.callback(name, lookback, 0, lookbehind_len) + elif self.flags & FLAG_LAST_BOUNDARY: + lookback = boundary + b"--\r\n" + self.callback(name, lookback, 0, lookbehind_len) + else: # pragma: no cover (error case) + self.logger.warning("Look-back buffer error") + + if end_i > 0: + self.callback(name, data, 0, end_i) + # If we're getting remaining data, we have got all the data we + # can be certain is not a boundary, leaving only a partial boundary match. + if remaining: + self.marks[name] = end_i - length + else: + self.marks.pop(name, None) + def _internal_write(self, data: bytes, length: int) -> int: # Get values from locals. boundary = self.boundary @@ -1165,50 +1210,6 @@ def set_mark(name: str) -> None: def delete_mark(name: str, reset: bool = False) -> None: self.marks.pop(name, None) - # Helper function that makes calling a callback with data easier. The - # 'remaining' parameter will callback from the marked value until the - # end of the buffer, and reset the mark, instead of deleting it. This - # is used at the end of the function to call our callbacks with any - # remaining data in this chunk. - def data_callback(name: CallbackName, end_i: int, remaining: bool = False) -> None: - marked_index = self.marks.get(name) - if marked_index is None: - return - - # Otherwise, we call it from the mark to the current byte we're - # processing. - if end_i <= marked_index: - # There is no additional data to send. - pass - elif marked_index >= 0: - # We are emitting data from the local buffer. - self.callback(name, data, marked_index, end_i) - else: - # Some of the data comes from a partial boundary match. - # and requires look-behind. - # We need to use self.flags (and not flags) because we care about - # the state when we entered the loop. - lookbehind_len = -marked_index - if lookbehind_len <= boundary_length: - self.callback(name, boundary, 0, lookbehind_len) - elif self.flags & FLAG_PART_BOUNDARY: - lookback = boundary + b"\r\n" - self.callback(name, lookback, 0, lookbehind_len) - elif self.flags & FLAG_LAST_BOUNDARY: - lookback = boundary + b"--\r\n" - self.callback(name, lookback, 0, lookbehind_len) - else: # pragma: no cover (error case) - self.logger.warning("Look-back buffer error") - - if end_i > 0: - self.callback(name, data, 0, end_i) - # If we're getting remaining data, we have got all the data we - # can be certain is not a boundary, leaving only a partial boundary match. - if remaining: - self.marks[name] = end_i - length - else: - self.marks.pop(name, None) - # For each byte... while i < length: c = data[i] @@ -1343,7 +1344,7 @@ def data_callback(name: CallbackName, end_i: int, remaining: bool = False) -> No # Call our callback with the header field. i = colon - data_callback("header_field", i) + self._data_callback("header_field", data, i, length) # Move to parsing the header value. state = HEADER_VALUE_START @@ -1370,7 +1371,7 @@ def data_callback(name: CallbackName, end_i: int, remaining: bool = False) -> No advance_header_size(end - i) if cr != -1: i = cr - data_callback("header_value", i) + self._data_callback("header_value", data, i, length) self.callback("header_end") current_header_size = 0 state = HEADER_VALUE_ALMOST_DONE @@ -1470,7 +1471,7 @@ def data_callback(name: CallbackName, end_i: int, remaining: bool = False) -> No # RFC 2046 forbids lines starting with the boundary in a part, so this can't be data. elif c in b" \t": - data_callback("part_data", i - boundary_length) + self._data_callback("part_data", data, i - boundary_length, length) delete_mark("part_data") index = 0 state = BOUNDARY_PADDING @@ -1492,7 +1493,7 @@ def data_callback(name: CallbackName, end_i: int, remaining: bool = False) -> No flags &= ~FLAG_PART_BOUNDARY # We have identified a boundary, callback for any data before it. - data_callback("part_data", i - index) + self._data_callback("part_data", data, i - index, length) # Callback indicating that we've reached the end of # a part, and are starting a new one. self.callback("part_end") @@ -1517,7 +1518,7 @@ def data_callback(name: CallbackName, end_i: int, remaining: bool = False) -> No # We need a second hyphen here. if c == HYPHEN: # We have identified a boundary, callback for any data before it. - data_callback("part_data", i - index) + self._data_callback("part_data", data, i - index, length) # Callback to end the current part, and then the # message. self.callback("part_end") @@ -1587,9 +1588,9 @@ def data_callback(name: CallbackName, end_i: int, remaining: bool = False) -> No # that we haven't yet reached the end of this 'thing'. So, by setting # the mark to 0, we cause any data callbacks that take place in future # calls to this function to start from the beginning of that buffer. - data_callback("header_field", length, True) - data_callback("header_value", length, True) - data_callback("part_data", length - index, True) + self._data_callback("header_field", data, length, length, True) + self._data_callback("header_value", data, length, length, True) + self._data_callback("part_data", data, length - index, length, True) # Save values to locals. self.state = state diff --git a/tests/test_multipart.py b/tests/test_multipart.py index 31b84e3..b6fe74e 100644 --- a/tests/test_multipart.py +++ b/tests/test_multipart.py @@ -913,6 +913,42 @@ def test_multipart_opening_boundary_max_size() -> None: assert events == ["begin", "end"] +@pytest.mark.parametrize("chunk_size", [*range(1, 32), 1024]) +def test_multipart_callbacks_across_chunks(chunk_size: int) -> None: + body = b"first\r\n--boundaryX\r\n--boundary\rX\r\n--boundary-X" + data = ( + b"--boundary\r\nX-Test: value\r\n\r\n" + body + b"\r\n" + b"--boundary\r\nX-Test: other\r\n\r\nsecond\r\n--boundary--\r\n" + ) + events: list[bytes] = [] + + def on_data(data: bytes, start: int, end: int) -> None: + events.append(data[start:end]) + + parser = MultipartParser( + b"boundary", + { + "on_part_begin": lambda: events.append(b""), + "on_header_begin": lambda: events.append(b"
"), + "on_header_field": on_data, + "on_header_value": on_data, + "on_header_end": lambda: events.append(b"
"), + "on_headers_finished": lambda: events.append(b""), + "on_part_data": on_data, + "on_part_end": lambda: events.append(b"
"), + "on_end": lambda: events.append(b""), + }, + ) + for start in range(0, len(data), chunk_size): + chunk = data[start : start + chunk_size] + assert parser.write(chunk) == len(chunk) + parser.finalize() + assert b"".join(events) == ( + b"
X-Testvalue
" + body + b"
" + b"
X-Testother
second
" + ) + + @pytest.mark.parametrize( "config,first,second,message", [ From 47bd519b77213149a906bf4d18646407610f1c21 Mon Sep 17 00:00:00 2001 From: Marcelo Trylesinski Date: Mon, 16 Nov 2026 09:32:00 +0100 Subject: [PATCH 2/4] Pin benchmark runs to Python 3.13.18 --- .github/workflows/benchmark.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index f6d42c8..210cf07 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -22,7 +22,7 @@ jobs: - name: Install uv uses: astral-sh/setup-uv@07a047e608db84b3d8c06cd9a7c7dcc9c355c18e # v10.3.2 with: - python-version: "3.13" + python-version: "3.13.18" enable-cache: true - name: Install dependencies From 267a94b0c63906d182f35d8759e648549e6a8dd9 Mon Sep 17 00:00:00 2001 From: Marcelo Trylesinski Date: Mon, 16 Nov 2026 09:54:49 +0100 Subject: [PATCH 3/4] Inline multipart mark operations --- .github/workflows/benchmark.yml | 2 +- .github/workflows/main.yml | 2 +- python_multipart/multipart.py | 18 +++++------------- 3 files changed, 7 insertions(+), 15 deletions(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 210cf07..53886db 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -4,7 +4,7 @@ on: push: branches: ["main"] pull_request: - branches: ["main"] + branches: ["**"] permissions: id-token: write diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 82bbce7..8a03cbf 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -4,7 +4,7 @@ on: push: branches: ["main"] pull_request: - branches: ["main"] + branches: ["**"] permissions: {} diff --git a/python_multipart/multipart.py b/python_multipart/multipart.py index 75850de..f63bcc6 100644 --- a/python_multipart/multipart.py +++ b/python_multipart/multipart.py @@ -1202,14 +1202,6 @@ def advance_header_size(amount: int = 1) -> None: if current_header_size > self.max_header_size: raise MultipartParseError("Maximum header size exceeded", offset=i) - # Set a mark. - def set_mark(name: str) -> None: - self.marks[name] = i - - # Remove a mark. - def delete_mark(name: str, reset: bool = False) -> None: - self.marks.pop(name, None) - # For each byte... while i < length: c = data[i] @@ -1291,7 +1283,7 @@ def delete_mark(name: str, reset: bool = False) -> None: current_header_size = 0 # Set a mark of our header field. - set_mark("header_field") + self.marks["header_field"] = i # Notify that we're starting a header if the next character is # not a CR; a CR at the beginning of the header will cause us @@ -1309,7 +1301,7 @@ def delete_mark(name: str, reset: bool = False) -> None: # that we've reached the second of 2 newlines, and so there are # no more headers to parse. if c == CR and index == 0: - delete_mark("header_field") + self.marks.pop("header_field", None) state = HEADERS_ALMOST_DONE i += 1 continue @@ -1357,7 +1349,7 @@ def delete_mark(name: str, reset: bool = False) -> None: continue # Mark the start of the header value. - set_mark("header_value") + self.marks["header_value"] = i # Move to the header-value state, reprocessing this character. state = HEADER_VALUE @@ -1404,7 +1396,7 @@ def delete_mark(name: str, reset: bool = False) -> None: elif state == PART_DATA_START: # Mark the start of our part data. - set_mark("part_data") + self.marks["part_data"] = i # Start processing part data, including this character. state = PART_DATA @@ -1472,7 +1464,7 @@ def delete_mark(name: str, reset: bool = False) -> None: # RFC 2046 forbids lines starting with the boundary in a part, so this can't be data. elif c in b" \t": self._data_callback("part_data", data, i - boundary_length, length) - delete_mark("part_data") + self.marks.pop("part_data", None) index = 0 state = BOUNDARY_PADDING From 3c922b01f86497e269c1c93cb2e76e4ed82d7052 Mon Sep 17 00:00:00 2001 From: Marcelo Trylesinski Date: Mon, 16 Nov 2026 10:03:27 +0100 Subject: [PATCH 4/4] Make multipart mark operations standalone --- .github/workflows/benchmark.yml | 4 +- .github/workflows/main.yml | 2 +- python_multipart/multipart.py | 105 ++++++++++++++++---------------- tests/test_multipart.py | 36 ----------- 4 files changed, 55 insertions(+), 92 deletions(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 53886db..f6d42c8 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -4,7 +4,7 @@ on: push: branches: ["main"] pull_request: - branches: ["**"] + branches: ["main"] permissions: id-token: write @@ -22,7 +22,7 @@ jobs: - name: Install uv uses: astral-sh/setup-uv@07a047e608db84b3d8c06cd9a7c7dcc9c355c18e # v10.3.2 with: - python-version: "3.13.18" + python-version: "3.13" enable-cache: true - name: Install dependencies diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 8a03cbf..82bbce7 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -4,7 +4,7 @@ on: push: branches: ["main"] pull_request: - branches: ["**"] + branches: ["main"] permissions: {} diff --git a/python_multipart/multipart.py b/python_multipart/multipart.py index f63bcc6..604bd70 100644 --- a/python_multipart/multipart.py +++ b/python_multipart/multipart.py @@ -1135,51 +1135,6 @@ def write(self, data: bytes) -> int: return l - def _data_callback(self, name: CallbackName, data: bytes, end_i: int, length: int, remaining: bool = False) -> None: - """Emit the data between the mark for `name` and `end_i` to the `name` callback. - - With `remaining`, the mark is kept and rebased onto the next chunk instead of being deleted. - `_internal_write` uses this at its end to flush what is left of the current chunk. - """ - boundary = self.boundary - marked_index = self.marks.get(name) - if marked_index is None: - return - - # Otherwise, we call it from the mark to the current byte we're - # processing. - if end_i <= marked_index: - # There is no additional data to send. - pass - elif marked_index >= 0: - # We are emitting data from the local buffer. - self.callback(name, data, marked_index, end_i) - else: - # Some of the data comes from a partial boundary match. - # and requires look-behind. - # We need to use self.flags (and not flags) because we care about - # the state when we entered the loop. - lookbehind_len = -marked_index - if lookbehind_len <= len(boundary): - self.callback(name, boundary, 0, lookbehind_len) - elif self.flags & FLAG_PART_BOUNDARY: - lookback = boundary + b"\r\n" - self.callback(name, lookback, 0, lookbehind_len) - elif self.flags & FLAG_LAST_BOUNDARY: - lookback = boundary + b"--\r\n" - self.callback(name, lookback, 0, lookbehind_len) - else: # pragma: no cover (error case) - self.logger.warning("Look-back buffer error") - - if end_i > 0: - self.callback(name, data, 0, end_i) - # If we're getting remaining data, we have got all the data we - # can be certain is not a boundary, leaving only a partial boundary match. - if remaining: - self.marks[name] = end_i - length - else: - self.marks.pop(name, None) - def _internal_write(self, data: bytes, length: int) -> int: # Get values from locals. boundary = self.boundary @@ -1202,6 +1157,50 @@ def advance_header_size(amount: int = 1) -> None: if current_header_size > self.max_header_size: raise MultipartParseError("Maximum header size exceeded", offset=i) + # Helper function that makes calling a callback with data easier. The + # 'remaining' parameter will callback from the marked value until the + # end of the buffer, and reset the mark, instead of deleting it. This + # is used at the end of the function to call our callbacks with any + # remaining data in this chunk. + def data_callback(name: CallbackName, end_i: int, remaining: bool = False) -> None: + marked_index = self.marks.get(name) + if marked_index is None: + return + + # Otherwise, we call it from the mark to the current byte we're + # processing. + if end_i <= marked_index: + # There is no additional data to send. + pass + elif marked_index >= 0: + # We are emitting data from the local buffer. + self.callback(name, data, marked_index, end_i) + else: + # Some of the data comes from a partial boundary match. + # and requires look-behind. + # We need to use self.flags (and not flags) because we care about + # the state when we entered the loop. + lookbehind_len = -marked_index + if lookbehind_len <= boundary_length: + self.callback(name, boundary, 0, lookbehind_len) + elif self.flags & FLAG_PART_BOUNDARY: + lookback = boundary + b"\r\n" + self.callback(name, lookback, 0, lookbehind_len) + elif self.flags & FLAG_LAST_BOUNDARY: + lookback = boundary + b"--\r\n" + self.callback(name, lookback, 0, lookbehind_len) + else: # pragma: no cover (error case) + self.logger.warning("Look-back buffer error") + + if end_i > 0: + self.callback(name, data, 0, end_i) + # If we're getting remaining data, we have got all the data we + # can be certain is not a boundary, leaving only a partial boundary match. + if remaining: + self.marks[name] = end_i - length + else: + self.marks.pop(name, None) + # For each byte... while i < length: c = data[i] @@ -1336,7 +1335,7 @@ def advance_header_size(amount: int = 1) -> None: # Call our callback with the header field. i = colon - self._data_callback("header_field", data, i, length) + data_callback("header_field", i) # Move to parsing the header value. state = HEADER_VALUE_START @@ -1363,7 +1362,7 @@ def advance_header_size(amount: int = 1) -> None: advance_header_size(end - i) if cr != -1: i = cr - self._data_callback("header_value", data, i, length) + data_callback("header_value", i) self.callback("header_end") current_header_size = 0 state = HEADER_VALUE_ALMOST_DONE @@ -1463,7 +1462,7 @@ def advance_header_size(amount: int = 1) -> None: # RFC 2046 forbids lines starting with the boundary in a part, so this can't be data. elif c in b" \t": - self._data_callback("part_data", data, i - boundary_length, length) + data_callback("part_data", i - boundary_length) self.marks.pop("part_data", None) index = 0 state = BOUNDARY_PADDING @@ -1485,7 +1484,7 @@ def advance_header_size(amount: int = 1) -> None: flags &= ~FLAG_PART_BOUNDARY # We have identified a boundary, callback for any data before it. - self._data_callback("part_data", data, i - index, length) + data_callback("part_data", i - index) # Callback indicating that we've reached the end of # a part, and are starting a new one. self.callback("part_end") @@ -1510,7 +1509,7 @@ def advance_header_size(amount: int = 1) -> None: # We need a second hyphen here. if c == HYPHEN: # We have identified a boundary, callback for any data before it. - self._data_callback("part_data", data, i - index, length) + data_callback("part_data", i - index) # Callback to end the current part, and then the # message. self.callback("part_end") @@ -1580,9 +1579,9 @@ def advance_header_size(amount: int = 1) -> None: # that we haven't yet reached the end of this 'thing'. So, by setting # the mark to 0, we cause any data callbacks that take place in future # calls to this function to start from the beginning of that buffer. - self._data_callback("header_field", data, length, length, True) - self._data_callback("header_value", data, length, length, True) - self._data_callback("part_data", data, length - index, length, True) + data_callback("header_field", length, True) + data_callback("header_value", length, True) + data_callback("part_data", length - index, True) # Save values to locals. self.state = state diff --git a/tests/test_multipart.py b/tests/test_multipart.py index b6fe74e..31b84e3 100644 --- a/tests/test_multipart.py +++ b/tests/test_multipart.py @@ -913,42 +913,6 @@ def test_multipart_opening_boundary_max_size() -> None: assert events == ["begin", "end"] -@pytest.mark.parametrize("chunk_size", [*range(1, 32), 1024]) -def test_multipart_callbacks_across_chunks(chunk_size: int) -> None: - body = b"first\r\n--boundaryX\r\n--boundary\rX\r\n--boundary-X" - data = ( - b"--boundary\r\nX-Test: value\r\n\r\n" + body + b"\r\n" - b"--boundary\r\nX-Test: other\r\n\r\nsecond\r\n--boundary--\r\n" - ) - events: list[bytes] = [] - - def on_data(data: bytes, start: int, end: int) -> None: - events.append(data[start:end]) - - parser = MultipartParser( - b"boundary", - { - "on_part_begin": lambda: events.append(b""), - "on_header_begin": lambda: events.append(b"
"), - "on_header_field": on_data, - "on_header_value": on_data, - "on_header_end": lambda: events.append(b"
"), - "on_headers_finished": lambda: events.append(b""), - "on_part_data": on_data, - "on_part_end": lambda: events.append(b"
"), - "on_end": lambda: events.append(b""), - }, - ) - for start in range(0, len(data), chunk_size): - chunk = data[start : start + chunk_size] - assert parser.write(chunk) == len(chunk) - parser.finalize() - assert b"".join(events) == ( - b"
X-Testvalue
" + body + b"
" - b"
X-Testother
second
" - ) - - @pytest.mark.parametrize( "config,first,second,message", [