Preserve the declared width of short raw and text fields

src/bitstruct/__init__.py

@@ -114,7 +114,7 @@ def unpack(self, bits):
 class _Raw(_Info):
 
     def pack(self, arg):
-        number_of_padding_bytes = ((self.size - 8 * len(arg)) // 8)
+        number_of_padding_bytes = ((self.size - 8 * len(arg) + 7) // 8)
         arg += (number_of_padding_bytes * b'\x00')
 
         return bin(int(b'01' + binascii.hexlify(arg), 16))[3:self.size + 3]
@@ -154,7 +154,7 @@ def __init__(self, size, name, encoding, errors):
 
     def pack(self, arg):
         encoded = arg.encode('utf-8')
-        number_of_padding_bytes = ((self.size - 8 * len(encoded)) // 8)
+        number_of_padding_bytes = ((self.size - 8 * len(encoded) + 7) // 8)
         encoded += (number_of_padding_bytes * b'\x00')
 
         return _pack_bytearray(self.size, encoded)

src/bitstruct/c.c

@@ -524,6 +524,18 @@ static PyObject *unpack_bool(struct bitstream_reader_t *self_p,
                                 field_info_p->number_of_bits)));
 }
 
+static void pack_padded_bytes(struct bitstream_writer_t *self_p,
+                              const char *buf_p,
+                              Py_ssize_t size,
+                              int number_of_bytes)
+{
+    int bytes_to_copy;
+
+    bytes_to_copy = (size < number_of_bytes ? (int)size : number_of_bytes);
+    bitstream_writer_write_bytes(self_p, (const uint8_t *)buf_p, bytes_to_copy);
+    bitstream_writer_write_repeated_u8(self_p, 0, number_of_bytes - bytes_to_copy);
+}
+
 static void pack_text(struct bitstream_writer_t *self_p,
                       PyObject *value_p,
                       struct field_info_t *field_info_p)
@@ -534,13 +546,7 @@ static void pack_text(struct bitstream_writer_t *self_p,
     buf_p = PyUnicode_AsUTF8AndSize(value_p, &size);
 
     if (buf_p != NULL) {
-        if (size < (field_info_p->number_of_bits / 8)) {
-            PyErr_SetString(PyExc_NotImplementedError, "Short text.");
-        } else {
-            bitstream_writer_write_bytes(self_p,
-                                         (uint8_t *)buf_p,
-                                         field_info_p->number_of_bits / 8);
-        }
+        pack_padded_bytes(self_p, buf_p, size, field_info_p->number_of_bits / 8);
     }
 }
 
@@ -576,13 +582,7 @@ static void pack_raw(struct bitstream_writer_t *self_p,
     res = PyBytes_AsStringAndSize(value_p, &buf_p, &size);
 
     if (res != -1) {
-        if (size < (field_info_p->number_of_bits / 8)) {
-            PyErr_SetString(PyExc_NotImplementedError, "Short raw data.");
-        } else {
-            bitstream_writer_write_bytes(self_p,
-                                         (uint8_t *)buf_p,
-                                         field_info_p->number_of_bits / 8);
-        }
+        pack_padded_bytes(self_p, buf_p, size, field_info_p->number_of_bits / 8);
     }
 }
 

tests/test_c.py

@@ -416,13 +416,11 @@ def test_pack_unpack_raw(self):
         if not is_cpython_3():
             return
 
-        with self.assertRaises(NotImplementedError):
-            packed = pack('r24', b'')
-            self.assertEqual(packed, b'\x00\x00\x00')
+        packed = pack('r24', b'')
+        self.assertEqual(packed, b'\x00\x00\x00')
 
-        with self.assertRaises(NotImplementedError):
-            packed = pack('r24', b'12')
-            self.assertEqual(packed, b'12\x00')
+        packed = pack('r24', b'12')
+        self.assertEqual(packed, b'12\x00')
 
         packed = pack('r24', b'123')
         self.assertEqual(packed, b'123')
@@ -614,11 +612,8 @@ def test_various(self):
         self.assertEqual(pack('u1p1', 1), b'\x80')
         self.assertEqual(pack_dict('u1p1', ['foo'], {'foo': 1}), b'\x80')
 
-        # Short text.
-        with self.assertRaises(NotImplementedError) as cm:
-            pack('t16', '1')
-
-        self.assertEqual(str(cm.exception), 'Short text.')
+        # Short text is padded with zero bytes.
+        self.assertEqual(pack('t16', '1'), b'1\x00')
 
         # Bad float length.
         with self.assertRaises(NotImplementedError) as cm:

tests/test_c_padding.py

@@ -0,0 +1,67 @@
+import unittest
+
+import bitstruct
+import bitstruct.c as c
+
+
+class CFieldPaddingTest(unittest.TestCase):
+    def cases(self):
+        for kind, values in [('r', [b'', b'A', b'AB', b'ABCDE']),
+                             ('t', ['', 'A', 'AB', 'ABCDE', 'é'])]:
+            for width in (8, 16, 24, 40):
+                for value in values:
+                    raw = value.encode('utf-8') if kind == 't' else value
+                    field = ''.join(f'{byte:08b}' for byte in raw)
+                    field = field[:width].ljust(width, '0')
+                    yield kind, width, value, '101' + field + '11101'
+
+    @staticmethod
+    def to_bytes(bits):
+        padded = bits.ljust((len(bits) + 7) // 8 * 8, '0')
+        return int(padded, 2).to_bytes(len(padded) // 8, 'big')
+
+    def test_pack_short_fields(self):
+        for kind, width, value, bits in self.cases():
+            fmt = f'u3{kind}{width}u5'
+            values = (5, value, 29)
+            names = ['before', 'payload', 'after']
+            mapping = dict(zip(names, values))
+            with self.subTest(fmt=fmt, value=value):
+                expected = self.to_bytes(bits)
+                self.assertEqual(c.pack(fmt, *values), expected)
+                self.assertEqual(c.compile(fmt).pack(*values), expected)
+                self.assertEqual(c.pack_dict(fmt, names, mapping), expected)
+                self.assertEqual(c.compile(fmt, names).pack(mapping), expected)
+                self.assertEqual(bitstruct.pack(fmt, *values), expected)
+                self.assertEqual(c.unpack(f'p{width + 3}u5', expected), (29,))
+
+    def test_pack_short_fields_into(self):
+        for kind, width, value, bits in self.cases():
+            fmt = f'u3{kind}{width}u5'
+            names = ['before', 'payload', 'after']
+            mapping = {'before': 5, 'payload': value, 'after': 29}
+            for variant in range(4):
+                with self.subTest(fmt=fmt, value=value, variant=variant):
+                    output = bytearray(b'\xa5' * 12)
+                    original = ''.join(f'{byte:08b}' for byte in output)
+                    expected = self.to_bytes(original[:5] + bits + original[5 + len(bits):])
+                    if variant == 0:
+                        c.pack_into(fmt, output, 5, 5, value, 29)
+                    elif variant == 1:
+                        c.compile(fmt).pack_into(output, 5, 5, value, 29)
+                    elif variant == 2:
+                        c.pack_into_dict(fmt, names, output, 5, mapping)
+                    else:
+                        c.compile(fmt, names).pack_into(output, 5, mapping)
+                    self.assertEqual(output, expected)
+
+    def test_existing_restrictions(self):
+        for fmt, value in [('r9', b''), ('t9', '')]:
+            with self.subTest(fmt=fmt):
+                with self.assertRaises(NotImplementedError):
+                    c.pack(fmt, value)
+        for fmt, value in [('r16', 'A'), ('r16', bytearray(b'A')),
+                           ('t16', b'A')]:
+            with self.subTest(fmt=fmt, value=value):
+                with self.assertRaises(TypeError):
+                    c.pack(fmt, value)

tests/test_field_width.py

@@ -0,0 +1,46 @@
+import unittest
+
+import bitstruct
+
+
+class ShortFieldWidthTest(unittest.TestCase):
+
+    def cases(self):
+        for kind, values in [('r', [b'', b'A', b'AB']),
+                             ('t', ['', 'A', 'AB'])]:
+            for width in (1, 7, 8, 9, 13, 16, 17, 24):
+                for value in values:
+                    for endian in ('>', '<'):
+                        raw = value.encode('utf-8') if kind == 't' else value
+                        bits = ''.join(f'{byte:08b}' for byte in raw)
+                        bits = bits[:width].ljust(width, '0')
+                        if endian == '<':
+                            bits = bits[::-1]
+                        yield kind, width, value, endian, '101' + bits + '11101'
+
+    @staticmethod
+    def to_bytes(bits):
+        padded = bits.ljust((len(bits) + 7) // 8 * 8, '0')
+        return int(padded, 2).to_bytes(len(padded) // 8, 'big')
+
+    def test_pack_short_fields(self):
+        for kind, width, value, endian, bits in self.cases():
+            fmt = f'u3{endian}{kind}{width}>u5'
+            with self.subTest(fmt=fmt, value=value):
+                expected = self.to_bytes(bits)
+                self.assertEqual(bitstruct.pack(fmt, 5, value, 29), expected)
+                self.assertEqual(bitstruct.compile(fmt).pack(5, value, 29), expected)
+                self.assertEqual(bitstruct.pack_dict(
+                    fmt, ['first', 'payload', 'last'],
+                    {'first': 5, 'payload': value, 'last': 29}), expected)
+                self.assertEqual(bitstruct.unpack(f'p{width + 3}u5', expected), (29,))
+
+    def test_pack_short_fields_into_buffer(self):
+        for kind, width, value, endian, bits in self.cases():
+            fmt = f'u3{endian}{kind}{width}>u5'
+            with self.subTest(fmt=fmt, value=value):
+                output = bytearray(b'\xa5' * 8)
+                original = ''.join(f'{byte:08b}' for byte in output)
+                expected = self.to_bytes(original[:5] + bits + original[5 + len(bits):])
+                bitstruct.pack_into(fmt, output, 5, 5, value, 29)
+                self.assertEqual(output, expected)