From c8a3d8e2d666ad5d4f25abcb3d9943868b02748a Mon Sep 17 00:00:00 2001 From: Enoch Groot Date: Sat, 19 Sep 2026 14:19:43 +0000 Subject: [PATCH] feat: pack oversized BigInt as MessagePack ext 0x42 Values outside int64/uint64 pack as msgpackr useBigIntExtension (two's-complement, type 0x42) with a 256-byte payload cap. In-range values still use integer wire. Unpack of ext 0x42 returns bigint. --- CHANGELOG.md | 25 ++++++- COVERAGE.md | 4 +- README.md | 6 +- index.d.ts | 15 ++-- package.json | 2 +- src/msgpack.cc | 167 ++++++++++++++++++++++++++++++++++++++++-- test/bigint.test.js | 174 ++++++++++++++++++++++++++++++++++++++++---- 7 files changed, 362 insertions(+), 31 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 37c527b..83ebb72 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,28 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [3.4.0] - 2026-09-19 + +Pack `bigint` values outside the 64-bit integer range as MessagePack ext +type `0x42` (msgpackr `useBigIntExtension` two's-complement payload), and +unpack that type back to `bigint`. Payload is capped at 256 bytes. Values +that still fit int64/uint64 keep using integer wire. + +### Added + +- Out-of-range `bigint` packs as ext `0x42` and round-trips through + `unpack()`, including nested arrays/maps/objects, uint256, and the + 64-bit boundaries `2^64` and `-2^63-1`. +- Unpack of ext `0x42` with a payload larger than 256 bytes throws + `cannot unpack BigInt: ext payload exceeds 256 bytes`. Other ext types + still throw `cannot unpack ext type`. + +### Changed + +- `cannot pack BigInt outside 64-bit range` is no longer thrown for + values that fit in a 256-byte ext payload. Larger values throw + `cannot pack BigInt: ext payload exceeds 256 bytes`. + ## [3.3.0] - 2026-09-19 `Stream.send` queues packed messages when the underlying writable returns @@ -152,7 +174,8 @@ GitHub Actions tests Node 18/20/22 on Ubuntu, macOS, and Windows 2022. - Pack throw paths free or return pooled sbuffers on every exit. - msgpack-c c-7.0.2 includes unpacker buffer-expansion overflow checks. -[Unreleased]: https://github.com/msgpack/msgpack-node/compare/v3.3.0...HEAD +[Unreleased]: https://github.com/msgpack/msgpack-node/compare/v3.4.0...HEAD +[3.4.0]: https://github.com/msgpack/msgpack-node/compare/v3.3.0...v3.4.0 [3.3.0]: https://github.com/msgpack/msgpack-node/compare/v3.2.0...v3.3.0 [3.2.0]: https://github.com/msgpack/msgpack-node/compare/v3.1.0...v3.2.0 [3.1.0]: https://github.com/msgpack/msgpack-node/compare/v3.0.0...v3.1.0 diff --git a/COVERAGE.md b/COVERAGE.md index 4199a5c..a70e148 100644 --- a/COVERAGE.md +++ b/COVERAGE.md @@ -128,7 +128,9 @@ gcovr --root . --filter src/ --exclude deps/ --no-markers --txt-metric branch -- - `test/bigint.test.js` — `#37` Number-vs-BigInt rule: uint64 of 1 stays Number, `MAX_SAFE_INTEGER` stays Number, reporter uint64 round-trips as - BigInt, out-of-range BigInt throws, rounded Number bits are not recovered. + BigInt, in-range BigInt still uses integer wire (not ext), out-of-range + BigInt packs as ext `0x42` (uint256, 64-bit boundaries, nested, 256-byte + cap), rounded Number bits are not recovered. - `test/coverage-native.test.js` (57 tests) — hand-built wire buffers for every MessagePack format family, including the ones `pack()` never emits (float32, str8/16/32, bin16/32, array32, map16/32, all eight ext forms, diff --git a/README.md b/README.md index 257a310..ffdc5b5 100644 --- a/README.md +++ b/README.md @@ -58,7 +58,8 @@ Packing: * `undefined` / `null` → nil * `boolean` → bool * finite integers (`number` or `bigint` in the 64-bit range) → uint/int -* `bigint` outside uint64/int64 → throws +* `bigint` outside uint64/int64 → ext type 0x42 (msgpackr BigInt, two's-complement, ≤ 256 bytes) +* `bigint` whose two's-complement form exceeds 256 bytes → throws * other numbers → float64 * `string` → str (UTF-8) * `Date` → str (ISO 8601, `toISOString()`), at any nesting level @@ -83,7 +84,8 @@ Unpacking: * str → `string` * bin → `Buffer` * array / map → Array / Object -* ext → throws +* ext type 0x42 (msgpackr BigInt) → bigint (payload ≤ 256 bytes) +* other ext types → throws So `unpack(pack(1n))` is Number `1`, and `unpack(pack(18446464814936021036n))` is that same `bigint`. diff --git a/index.d.ts b/index.d.ts index 9bc879f..8bee345 100644 --- a/index.d.ts +++ b/index.d.ts @@ -1,4 +1,4 @@ -// Type definitions for msgpack 3.3.0 +// Type definitions for msgpack 3.4.0 // Project: https://github.com/msgpack/msgpack-node /// @@ -56,10 +56,12 @@ export interface PackOptions { * and may also set `type` / `family`. * * `bigint` values in the int64/uint64 range pack as MessagePack integers - * (smallest family that fits). Values outside that range throw. A `number` - * that has already lost bits below 2^53 stays on the Number path; lost bits - * are not recovered. BigInt plus an integer `type`/`family` uses the same - * 64-bit path. + * (smallest family that fits). Larger values pack as ext type 0x42 (msgpackr + * BigInt extension) with a two's-complement payload of at most 256 bytes. + * Still larger values throw. A `number` that has already lost bits below + * 2^53 stays on the Number path; lost bits are not recovered. BigInt plus + * an integer `type`/`family` uses the same 64-bit path (hints do not + * truncate into ext). */ export function pack(value: any, options: PackOptions): Buffer; export function pack(...values: any[]): Buffer; @@ -69,7 +71,8 @@ export function pack(...values: any[]): Buffer; * * Integers whose magnitude is greater than `Number.MAX_SAFE_INTEGER` return * as `bigint`. Values that fit stay `number`, even if the wire type is - * uint64 or int64 (a uint64 of 1 is Number 1). + * uint64 or int64 (a uint64 of 1 is Number 1). Ext type 0x42 unpacks as + * `bigint` (payload capped at 256 bytes). Other ext types throw. * * Returns `null` when the buffer holds an incomplete value, in which case * `unpack.bytes_remaining` equals `buf.length`. Throws on malformed input or diff --git a/package.json b/package.json index 7fa82b1..7bc5cea 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "msgpack", "description": "A space-efficient object serialization library for Node.js", - "version": "3.3.0", + "version": "3.4.0", "homepage": "https://github.com/msgpack/msgpack-node", "author": "Peter Griess ", "contributors": [ diff --git a/src/msgpack.cc b/src/msgpack.cc index 79f92cb..92a158a 100644 --- a/src/msgpack.cc +++ b/src/msgpack.cc @@ -37,6 +37,10 @@ const double kInt64Min = -9223372036854775808.0; const uint64_t kMaxSafeInteger = 9007199254740991ULL; const int64_t kMinSafeInteger = -9007199254740991LL; const size_t kSbufferPoolMax = 512; +/* msgpackr useBigIntExtension: two's-complement BigInt as ext type 0x42 ('B'). */ +const int8_t kBigIntExtType = 0x42; +const uint32_t kMaxBigIntExtBytes = 256; +const int kMaxBigIntExtWords = 32; enum ScanStatus { kScanOk = 0, @@ -332,6 +336,149 @@ static v8::Local Error(const char* msg) { return Nan::Error(msg); } +/* + * Pack a BigInt that does not fit int64/uint64 as MessagePack ext 0x42. + * Payload is two's-complement big-endian bytes, minimal length, sign-extended + * so the high bit matches the sign (msgpackr useBigIntExtension algorithm). + * Fail closed at 256 payload bytes (2048-bit). + */ +static void PackBigIntExt(msgpack_packer* pk, v8::Local bi) { + int word_count = bi->WordCount(); + if (word_count > kMaxBigIntExtWords) { + throw MsgpackException( + Error("cannot pack BigInt: ext payload exceeds 256 bytes")); + } + uint64_t words[32]; + memset(words, 0, sizeof(words)); + int sign_bit = 0; + if (word_count > 0) { + int wc = word_count; + bi->ToWordsArray(&sign_bit, &wc, words); + word_count = wc; + } + unsigned char tmp[256]; + memset(tmp, 0, sizeof(tmp)); + size_t n = static_cast(word_count) * 8u; + /* GCOVR_EXCL_START: WordCount is 0 only for 0n, which takes the int64 path. */ + if (n == 0) { + tmp[0] = 0; + n = 1; + } else { + /* GCOVR_EXCL_STOP */ + for (int i = 0; i < word_count; i++) { + uint64_t w = words[i]; + for (int b = 0; b < 8; b++) { + tmp[static_cast(i) * 8u + static_cast(b)] = + static_cast(w & 0xffu); + w >>= 8; + } + } + } + if (sign_bit) { + unsigned int carry = 1; + for (size_t i = 0; i < n; i++) { + unsigned int v = + static_cast(static_cast(~tmp[i])) + carry; + tmp[i] = static_cast(v); + carry = v >> 8; + } + if ((tmp[n - 1] & 0x80u) == 0) { + if (n >= kMaxBigIntExtBytes) { + throw MsgpackException( + Error("cannot pack BigInt: ext payload exceeds 256 bytes")); + } + tmp[n] = 0xff; + n++; + } + } else if ((tmp[n - 1] & 0x80u) != 0) { + if (n >= kMaxBigIntExtBytes) { + throw MsgpackException( + Error("cannot pack BigInt: ext payload exceeds 256 bytes")); + } + tmp[n] = 0x00; + n++; + } + while (n > 1) { + if (tmp[n - 1] == 0x00 && (tmp[n - 2] & 0x80u) == 0) { + n--; + continue; + } + if (tmp[n - 1] == 0xff && (tmp[n - 2] & 0x80u) != 0) { + n--; + continue; + } + break; + } + /* GCOVR_EXCL_START: sign-extend already threw at 257 bytes; strip only shrinks. */ + if (n > kMaxBigIntExtBytes) { + throw MsgpackException( + Error("cannot pack BigInt: ext payload exceeds 256 bytes")); + } + /* GCOVR_EXCL_STOP */ + unsigned char be[256]; + for (size_t i = 0; i < n; i++) { + be[i] = tmp[n - 1 - i]; + } + int rc = msgpack_pack_ext(pk, n, kBigIntExtType); + if (rc == 0) { /* GCOVR_EXCL_BR_LINE: sbuffer write failure */ + rc = msgpack_pack_ext_body(pk, be, n); + } + /* GCOVR_EXCL_START: sbuffer write failure */ + if (rc != 0) { + throw MsgpackException(Error("Error serializing object")); + } + /* GCOVR_EXCL_STOP */ +} + +static v8::Local ExtBigIntToJs(const char* ptr, uint32_t size) { + if (size == 0) { + throw MsgpackException(Error("cannot unpack BigInt")); + } + if (size > kMaxBigIntExtBytes) { + throw MsgpackException( + Error("cannot unpack BigInt: ext payload exceeds 256 bytes")); + } + const unsigned char* p = reinterpret_cast(ptr); + unsigned char mag[256]; + memcpy(mag, p, size); + const bool neg = (mag[0] & 0x80u) != 0; + if (neg) { + unsigned int carry = 1; + for (int i = static_cast(size) - 1; i >= 0; i--) { + unsigned int v = + static_cast(static_cast(~mag[i])) + carry; + mag[i] = static_cast(v); + carry = v >> 8; + } + } + uint32_t start = 0; + while (start + 1u < size && mag[start] == 0) { + start++; + } + const uint32_t nbytes = size - start; + int word_count = static_cast((nbytes + 7u) / 8u); + if (word_count == 0) { /* GCOVR_EXCL_BR_LINE: nbytes is at least 1 after size==0 throw */ + word_count = 1; + } + uint64_t words[32]; + memset(words, 0, sizeof(words)); + int byte_i = 0; + for (int i = static_cast(size) - 1; i >= static_cast(start); i--) { + const int wi = byte_i / 8; + const int sh = (byte_i % 8) * 8; + words[wi] |= static_cast(mag[i]) << sh; + byte_i++; + } + v8::MaybeLocal maybe = v8::BigInt::NewFromWords( + Nan::GetCurrentContext(), neg ? 1 : 0, word_count, words); + /* GCOVR_EXCL_START: NewFromWords fails only on OOM / isolate death. */ + if (maybe.IsEmpty()) { + throw MsgpackException(Error("cannot unpack BigInt")); + } + /* GCOVR_EXCL_STOP */ + return maybe.ToLocalChecked(); +} + /* Persistent identity flag for cycle detection (not enumerable). * thread_local because a v8::Persistent belongs to the isolate that created * it: with a process-global handle, a worker's Init() would dispose the main @@ -552,10 +699,12 @@ static void JsToMsgpack(msgpack_packer* pk, v8::Local o, int depth) { } else { lossless = false; const uint64_t u = bi->Uint64Value(&lossless); - if (!lossless) { - throw MsgpackException(Error("cannot pack BigInt outside 64-bit range")); + if (lossless) { + rc = msgpack_pack_uint64(pk, u); + } else { + PackBigIntExt(pk, bi); + return; } - rc = msgpack_pack_uint64(pk, u); } } else if (o->IsString()) { Nan::Utf8String bytes(o); @@ -638,10 +787,14 @@ static v8::Local MsgpackToJs(const msgpack_object* mo) { return Nan::NewBuffer(0).ToLocalChecked(); } return Nan::CopyBuffer(mo->via.bin.ptr, mo->via.bin.size).ToLocalChecked(); - case MSGPACK_OBJECT_EXT: - /* Fail closed on extension types: callers expecting core JSON-like - * values should not silently receive opaque ext payloads. */ - throw MsgpackException(Error("cannot unpack ext type")); + case MSGPACK_OBJECT_EXT: { + /* ext 0x42 is msgpackr BigInt. Every other ext type stays fail-closed. */ + const msgpack_object_ext& ext = mo->via.ext; + if (ext.type != kBigIntExtType) { + throw MsgpackException(Error("cannot unpack ext type")); + } + return ExtBigIntToJs(ext.ptr, ext.size); + } case MSGPACK_OBJECT_ARRAY: { v8::Local arr = Nan::New(mo->via.array.size); for (uint32_t i = 0; i < mo->via.array.size; i++) { diff --git a/test/bigint.test.js b/test/bigint.test.js index 3f343ff..3676e68 100644 --- a/test/bigint.test.js +++ b/test/bigint.test.js @@ -8,6 +8,51 @@ function b(...bytes) { return Buffer.from(bytes); } +/** msgpackr useBigIntExtension payload: two's-complement BE, min length. */ +function bigintExtPayload(value) { + const bytes = []; + let v = value; + let alignedSign; + do { + const byte = v & 0xffn; + alignedSign = (byte & 0x80n) === (v < 0n ? 0x80n : 0n); + bytes.push(Number(byte)); + v >>= 8n; + } while (!((v === 0n || v === -1n) && alignedSign)); + return Buffer.from(bytes.reverse()); +} + +function ext42Wire(payload) { + const n = payload.length; + const type = 0x42; + if (n === 1) return Buffer.concat([Buffer.from([0xd4, type]), payload]); + if (n === 2) return Buffer.concat([Buffer.from([0xd5, type]), payload]); + if (n === 4) return Buffer.concat([Buffer.from([0xd6, type]), payload]); + if (n === 8) return Buffer.concat([Buffer.from([0xd7, type]), payload]); + if (n === 16) return Buffer.concat([Buffer.from([0xd8, type]), payload]); + if (n <= 0xff) { + return Buffer.concat([Buffer.from([0xc7, n, type]), payload]); + } + if (n <= 0xffff) { + return Buffer.concat([ + Buffer.from([0xc8, (n >> 8) & 0xff, n & 0xff, type]), + payload, + ]); + } + throw new Error('payload too large for test helper'); +} + +function isExt42(buf) { + const b0 = buf[0]; + if (b0 === 0xd4 || b0 === 0xd5 || b0 === 0xd6 || b0 === 0xd7 || b0 === 0xd8) { + return buf[1] === 0x42; + } + if (b0 === 0xc7) return buf[2] === 0x42; + if (b0 === 0xc8) return buf[3] === 0x42; + if (b0 === 0xc9) return buf[4] === 0x42; + return false; +} + describe('BigInt 64-bit integers (#37)', () => { const reporter = 18446464814936021036n; const reporterWire = b(0xcf, 0xff, 0xff, 0x02, 0x04, 0x00, 0x00, 0xd8, 0x2c); @@ -68,19 +113,12 @@ describe('BigInt 64-bit integers (#37)', () => { assert.equal(msgpack.unpack(msgpack.pack(-(2n ** 63n))), -(2n ** 63n)); }); - it('throws a catchable error for BigInt outside uint64/int64', () => { - assert.throws(() => msgpack.pack(2n ** 64n), /cannot pack BigInt outside 64-bit range/); - assert.throws( - () => msgpack.pack(-(2n ** 63n) - 1n), - /cannot pack BigInt outside 64-bit range/ - ); - try { - msgpack.pack(2n ** 64n); - assert.fail('expected throw'); - } catch (err) { - assert.ok(err instanceof Error); - assert.match(err.message, /cannot pack BigInt outside 64-bit range/); - } + it('does not use ext 0x42 for in-range bigint', () => { + assert.equal(isExt42(msgpack.pack(0n)), false); + assert.equal(isExt42(msgpack.pack(1n)), false); + assert.equal(isExt42(msgpack.pack(2n ** 64n - 1n)), false); + assert.equal(isExt42(msgpack.pack(-(2n ** 63n))), false); + assert.equal(isExt42(msgpack.pack(reporter)), false); }); it('does not recover lost bits from a Number that is already rounded', () => { @@ -100,3 +138,113 @@ describe('BigInt 64-bit integers (#37)', () => { assert.equal(got.a[1], reporter); }); }); + +describe('BigInt ext 0x42', () => { + const uint256Max = (1n << 256n) - 1n; + const justOverUint64 = 2n ** 64n; + const justUnderInt64 = -(2n ** 63n) - 1n; + + it('round-trips 2^64 as ext 0x42 matching msgpackr payload', () => { + const packed = msgpack.pack(justOverUint64); + assert.equal(isExt42(packed), true); + assert.deepEqual(packed, ext42Wire(bigintExtPayload(justOverUint64))); + const got = msgpack.unpack(packed); + assert.equal(typeof got, 'bigint'); + assert.equal(got, justOverUint64); + }); + + it('round-trips -2^63-1 as ext 0x42 matching msgpackr payload', () => { + const packed = msgpack.pack(justUnderInt64); + assert.equal(isExt42(packed), true); + assert.deepEqual(packed, ext42Wire(bigintExtPayload(justUnderInt64))); + assert.equal(msgpack.unpack(packed), justUnderInt64); + }); + + it('round-trips uint256 max (2^256-1)', () => { + const packed = msgpack.pack(uint256Max); + assert.equal(isExt42(packed), true); + assert.deepEqual(packed, ext42Wire(bigintExtPayload(uint256Max))); + assert.equal(msgpack.unpack(packed), uint256Max); + }); + + it('round-trips nested ext BigInt in arrays, maps, and objects', () => { + const payload = { n: uint256Max, a: [justOverUint64, { k: justUnderInt64 }] }; + const got = msgpack.unpack(msgpack.pack(payload)); + assert.equal(got.n, uint256Max); + assert.equal(got.a[0], justOverUint64); + assert.equal(got.a[1].k, justUnderInt64); + }); + + it('lazy-unpacks nested ext BigInt', () => { + const packed = msgpack.pack({ n: uint256Max, a: [justOverUint64] }); + const got = msgpack.unpack(packed, { lazy: true }); + assert.equal(got.n, uint256Max); + assert.equal(got.a[0], justOverUint64); + }); + + it('packs the 256-byte payload limit and rejects one bit over', () => { + const atLimit = 2n ** 2047n - 1n; + const packed = msgpack.pack(atLimit); + assert.equal(isExt42(packed), true); + assert.equal(msgpack.unpack(packed), atLimit); + + assert.throws( + () => msgpack.pack(2n ** 2047n), + /cannot pack BigInt: ext payload exceeds 256 bytes/ + ); + assert.throws( + () => msgpack.pack(2n ** 2048n), + /cannot pack BigInt: ext payload exceeds 256 bytes/ + ); + assert.doesNotThrow(() => msgpack.pack(-(2n ** 2047n))); + assert.equal(msgpack.unpack(msgpack.pack(-(2n ** 2047n))), -(2n ** 2047n)); + assert.throws( + () => msgpack.pack(-(2n ** 2047n) - 1n), + /cannot pack BigInt: ext payload exceeds 256 bytes/ + ); + }); + + it('unpacks a crafted ext 0x42 larger than 256 bytes as a catchable error', () => { + const n = 257; + const payload = Buffer.alloc(n, 0x01); + payload[0] = 0x00; + const wire = Buffer.concat([ + Buffer.from([0xc8, (n >> 8) & 0xff, n & 0xff, 0x42]), + payload, + ]); + assert.throws( + () => msgpack.unpack(wire), + /cannot unpack BigInt: ext payload exceeds 256 bytes/ + ); + }); + + it('throws on empty ext 0x42 payload', () => { + assert.throws(() => msgpack.unpack(Buffer.from([0xc7, 0x00, 0x42])), /cannot unpack BigInt/); + }); + + it('still refuses other ext types', () => { + assert.throws(() => msgpack.unpack(Buffer.from([0xd4, 0x01, 0x00])), /cannot unpack ext type/); + assert.throws(() => msgpack.unpack(Buffer.from([0xc7, 0x01, 0x00, 0xff])), /cannot unpack ext type/); + }); + + it('unpacks a non-minimal ext 0x42 payload', () => { + /* 0x00 0x01 is 1n with an extra sign byte. */ + assert.equal(msgpack.unpack(Buffer.from([0xd5, 0x42, 0x00, 0x01])), 1n); + assert.equal(typeof msgpack.unpack(Buffer.from([0xd5, 0x42, 0x00, 0x01])), 'bigint'); + }); + + it('does not let pack() type/family hints truncate into 64-bit integers', () => { + assert.throws( + () => msgpack.pack(uint256Max, { type: 'uint64' }), + /cannot pack value as uint64/ + ); + assert.throws( + () => msgpack.pack(justOverUint64, { family: 'int' }), + /cannot pack value as int/ + ); + assert.throws( + () => msgpack.pack(justUnderInt64, { type: 'int64' }), + /cannot pack value as int64/ + ); + }); +});