From d86d338e633033209d5721498935b8afcaf35cd2 Mon Sep 17 00:00:00 2001 From: Rupayon Haldar <80724680+rupayon123@users.noreply.github.com> Date: Sat, 31 Oct 2026 09:08:53 -0400 Subject: [PATCH 1/2] Round raw and text field padding up to whole bytes --- src/bitstruct/__init__.py | 4 ++-- tests/test_field_width.py | 46 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 48 insertions(+), 2 deletions(-) create mode 100644 tests/test_field_width.py diff --git a/src/bitstruct/__init__.py b/src/bitstruct/__init__.py index 223e022..0bd9662 100644 --- a/src/bitstruct/__init__.py +++ b/src/bitstruct/__init__.py @@ -114,7 +114,7 @@ def unpack(self, bits): class _Raw(_Info): def pack(self, arg): - number_of_padding_bytes = ((self.size - 8 * len(arg)) // 8) + number_of_padding_bytes = ((self.size - 8 * len(arg) + 7) // 8) arg += (number_of_padding_bytes * b'\x00') return bin(int(b'01' + binascii.hexlify(arg), 16))[3:self.size + 3] @@ -154,7 +154,7 @@ def __init__(self, size, name, encoding, errors): def pack(self, arg): encoded = arg.encode('utf-8') - number_of_padding_bytes = ((self.size - 8 * len(encoded)) // 8) + number_of_padding_bytes = ((self.size - 8 * len(encoded) + 7) // 8) encoded += (number_of_padding_bytes * b'\x00') return _pack_bytearray(self.size, encoded) diff --git a/tests/test_field_width.py b/tests/test_field_width.py new file mode 100644 index 0000000..beabd1b --- /dev/null +++ b/tests/test_field_width.py @@ -0,0 +1,46 @@ +import unittest + +import bitstruct + + +class ShortFieldWidthTest(unittest.TestCase): + + def cases(self): + for kind, values in [('r', [b'', b'A', b'AB']), + ('t', ['', 'A', 'AB'])]: + for width in (1, 7, 8, 9, 13, 16, 17, 24): + for value in values: + for endian in ('>', '<'): + raw = value.encode('utf-8') if kind == 't' else value + bits = ''.join(f'{byte:08b}' for byte in raw) + bits = bits[:width].ljust(width, '0') + if endian == '<': + bits = bits[::-1] + yield kind, width, value, endian, '101' + bits + '11101' + + @staticmethod + def to_bytes(bits): + padded = bits.ljust((len(bits) + 7) // 8 * 8, '0') + return int(padded, 2).to_bytes(len(padded) // 8, 'big') + + def test_pack_short_fields(self): + for kind, width, value, endian, bits in self.cases(): + fmt = f'u3{endian}{kind}{width}>u5' + with self.subTest(fmt=fmt, value=value): + expected = self.to_bytes(bits) + self.assertEqual(bitstruct.pack(fmt, 5, value, 29), expected) + self.assertEqual(bitstruct.compile(fmt).pack(5, value, 29), expected) + self.assertEqual(bitstruct.pack_dict( + fmt, ['first', 'payload', 'last'], + {'first': 5, 'payload': value, 'last': 29}), expected) + self.assertEqual(bitstruct.unpack(f'p{width + 3}u5', expected), (29,)) + + def test_pack_short_fields_into_buffer(self): + for kind, width, value, endian, bits in self.cases(): + fmt = f'u3{endian}{kind}{width}>u5' + with self.subTest(fmt=fmt, value=value): + output = bytearray(b'\xa5' * 8) + original = ''.join(f'{byte:08b}' for byte in output) + expected = self.to_bytes(original[:5] + bits + original[5 + len(bits):]) + bitstruct.pack_into(fmt, output, 5, 5, value, 29) + self.assertEqual(output, expected) From 6bb62329f17067abc84b6d7fd969e6586dbf911e Mon Sep 17 00:00:00 2001 From: Rupayon Haldar <80724680+rupayon123@users.noreply.github.com> Date: Sun, 1 Nov 2026 10:47:13 -0500 Subject: [PATCH 2/2] Pad short raw and text fields in the native C backend --- src/bitstruct/c.c | 28 ++++++++--------- tests/test_c.py | 17 ++++------- tests/test_c_padding.py | 67 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 87 insertions(+), 25 deletions(-) create mode 100644 tests/test_c_padding.py diff --git a/src/bitstruct/c.c b/src/bitstruct/c.c index c300e01..b9ccbd6 100644 --- a/src/bitstruct/c.c +++ b/src/bitstruct/c.c @@ -524,6 +524,18 @@ static PyObject *unpack_bool(struct bitstream_reader_t *self_p, field_info_p->number_of_bits))); } +static void pack_padded_bytes(struct bitstream_writer_t *self_p, + const char *buf_p, + Py_ssize_t size, + int number_of_bytes) +{ + int bytes_to_copy; + + bytes_to_copy = (size < number_of_bytes ? (int)size : number_of_bytes); + bitstream_writer_write_bytes(self_p, (const uint8_t *)buf_p, bytes_to_copy); + bitstream_writer_write_repeated_u8(self_p, 0, number_of_bytes - bytes_to_copy); +} + static void pack_text(struct bitstream_writer_t *self_p, PyObject *value_p, struct field_info_t *field_info_p) @@ -534,13 +546,7 @@ static void pack_text(struct bitstream_writer_t *self_p, buf_p = PyUnicode_AsUTF8AndSize(value_p, &size); if (buf_p != NULL) { - if (size < (field_info_p->number_of_bits / 8)) { - PyErr_SetString(PyExc_NotImplementedError, "Short text."); - } else { - bitstream_writer_write_bytes(self_p, - (uint8_t *)buf_p, - field_info_p->number_of_bits / 8); - } + pack_padded_bytes(self_p, buf_p, size, field_info_p->number_of_bits / 8); } } @@ -576,13 +582,7 @@ static void pack_raw(struct bitstream_writer_t *self_p, res = PyBytes_AsStringAndSize(value_p, &buf_p, &size); if (res != -1) { - if (size < (field_info_p->number_of_bits / 8)) { - PyErr_SetString(PyExc_NotImplementedError, "Short raw data."); - } else { - bitstream_writer_write_bytes(self_p, - (uint8_t *)buf_p, - field_info_p->number_of_bits / 8); - } + pack_padded_bytes(self_p, buf_p, size, field_info_p->number_of_bits / 8); } } diff --git a/tests/test_c.py b/tests/test_c.py index 2aa58ff..943489b 100644 --- a/tests/test_c.py +++ b/tests/test_c.py @@ -416,13 +416,11 @@ def test_pack_unpack_raw(self): if not is_cpython_3(): return - with self.assertRaises(NotImplementedError): - packed = pack('r24', b'') - self.assertEqual(packed, b'\x00\x00\x00') + packed = pack('r24', b'') + self.assertEqual(packed, b'\x00\x00\x00') - with self.assertRaises(NotImplementedError): - packed = pack('r24', b'12') - self.assertEqual(packed, b'12\x00') + packed = pack('r24', b'12') + self.assertEqual(packed, b'12\x00') packed = pack('r24', b'123') self.assertEqual(packed, b'123') @@ -614,11 +612,8 @@ def test_various(self): self.assertEqual(pack('u1p1', 1), b'\x80') self.assertEqual(pack_dict('u1p1', ['foo'], {'foo': 1}), b'\x80') - # Short text. - with self.assertRaises(NotImplementedError) as cm: - pack('t16', '1') - - self.assertEqual(str(cm.exception), 'Short text.') + # Short text is padded with zero bytes. + self.assertEqual(pack('t16', '1'), b'1\x00') # Bad float length. with self.assertRaises(NotImplementedError) as cm: diff --git a/tests/test_c_padding.py b/tests/test_c_padding.py new file mode 100644 index 0000000..09381ed --- /dev/null +++ b/tests/test_c_padding.py @@ -0,0 +1,67 @@ +import unittest + +import bitstruct +import bitstruct.c as c + + +class CFieldPaddingTest(unittest.TestCase): + def cases(self): + for kind, values in [('r', [b'', b'A', b'AB', b'ABCDE']), + ('t', ['', 'A', 'AB', 'ABCDE', 'é'])]: + for width in (8, 16, 24, 40): + for value in values: + raw = value.encode('utf-8') if kind == 't' else value + field = ''.join(f'{byte:08b}' for byte in raw) + field = field[:width].ljust(width, '0') + yield kind, width, value, '101' + field + '11101' + + @staticmethod + def to_bytes(bits): + padded = bits.ljust((len(bits) + 7) // 8 * 8, '0') + return int(padded, 2).to_bytes(len(padded) // 8, 'big') + + def test_pack_short_fields(self): + for kind, width, value, bits in self.cases(): + fmt = f'u3{kind}{width}u5' + values = (5, value, 29) + names = ['before', 'payload', 'after'] + mapping = dict(zip(names, values)) + with self.subTest(fmt=fmt, value=value): + expected = self.to_bytes(bits) + self.assertEqual(c.pack(fmt, *values), expected) + self.assertEqual(c.compile(fmt).pack(*values), expected) + self.assertEqual(c.pack_dict(fmt, names, mapping), expected) + self.assertEqual(c.compile(fmt, names).pack(mapping), expected) + self.assertEqual(bitstruct.pack(fmt, *values), expected) + self.assertEqual(c.unpack(f'p{width + 3}u5', expected), (29,)) + + def test_pack_short_fields_into(self): + for kind, width, value, bits in self.cases(): + fmt = f'u3{kind}{width}u5' + names = ['before', 'payload', 'after'] + mapping = {'before': 5, 'payload': value, 'after': 29} + for variant in range(4): + with self.subTest(fmt=fmt, value=value, variant=variant): + output = bytearray(b'\xa5' * 12) + original = ''.join(f'{byte:08b}' for byte in output) + expected = self.to_bytes(original[:5] + bits + original[5 + len(bits):]) + if variant == 0: + c.pack_into(fmt, output, 5, 5, value, 29) + elif variant == 1: + c.compile(fmt).pack_into(output, 5, 5, value, 29) + elif variant == 2: + c.pack_into_dict(fmt, names, output, 5, mapping) + else: + c.compile(fmt, names).pack_into(output, 5, mapping) + self.assertEqual(output, expected) + + def test_existing_restrictions(self): + for fmt, value in [('r9', b''), ('t9', '')]: + with self.subTest(fmt=fmt): + with self.assertRaises(NotImplementedError): + c.pack(fmt, value) + for fmt, value in [('r16', 'A'), ('r16', bytearray(b'A')), + ('t16', b'A')]: + with self.subTest(fmt=fmt, value=value): + with self.assertRaises(TypeError): + c.pack(fmt, value)