From 11247b32e8e21bd505eb26be206fd5f34041579c Mon Sep 17 00:00:00 2001 From: Enoch Groot Date: Sat, 19 Sep 2026 04:12:29 +0000 Subject: [PATCH] feat: pack and unpack 64-bit integers as BigInt Integers outside Number.MAX_SAFE_INTEGER unpack as bigint instead of a rounded number. Values that fit stay number regardless of wire width. pack() accepts bigint in the int64/uint64 range and throws for anything larger. Closes #37. --- CHANGELOG.md | 33 +++++++++++- COVERAGE.md | 7 ++- README.md | 28 ++++++---- index.d.ts | 11 +++- package-lock.json | 4 +- package.json | 4 +- src/msgpack.cc | 34 ++++++++++-- test/bigint.test.js | 102 +++++++++++++++++++++++++++++++++++ test/coverage-native.test.js | 38 +++++++++---- test/regression.test.js | 9 +++- 10 files changed, 238 insertions(+), 32 deletions(-) create mode 100644 test/bigint.test.js diff --git a/CHANGELOG.md b/CHANGELOG.md index db49a26..e3dabfc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,36 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [3.0.0] - 2026-09-19 + +Integers whose magnitude is greater than `Number.MAX_SAFE_INTEGER` unpack as +`bigint` instead of a rounded `number`. Values that fit stay `number` +regardless of wire width. `pack()` accepts `bigint` in the signed/unsigned +64-bit range. + +### Added + +- `pack()` encodes `bigint` via `v8::BigInt` `Int64Value` / `Uint64Value` as + the smallest MessagePack integer family that fits. +- Unpack of uint64/int64 values outside `Number.MAX_SAFE_INTEGER` returns + `bigint` so 64-bit integers stay exact (`#37`). + +### Changed + +- A uint64 of `1` still unpacks as Number `1`. `Number.MAX_SAFE_INTEGER` + stays Number even when the wire type is uint64. +- A JS `number` that is already rounded (for example `18446464814936021000`) + still packs on the Number path; lost bits are not recovered. + +### Breaking + +- Unpacking a 64-bit integer larger than `Number.MAX_SAFE_INTEGER` now + returns `bigint` instead of the nearest double. Code that assumed + `typeof unpack(...) === 'number'` for every integer must accept `bigint`. +- `pack(10n)` no longer throws `cannot pack object`. BigInt outside + uint64/int64 (`2n ** 64n`, `-(2n ** 63n) - 1n`) throws + `cannot pack BigInt outside 64-bit range`. + ## [2.0.0] - 2026-09-18 Security modernization. Requires **Node.js 18+**. Vendors **msgpack-c c-7.0.2**. @@ -58,5 +88,6 @@ GitHub Actions tests Node 18/20/22 on Ubuntu, macOS, and Windows 2022. - Pack throw paths free or return pooled sbuffers on every exit. - msgpack-c c-7.0.2 includes unpacker buffer-expansion overflow checks. -[Unreleased]: https://github.com/msgpack/msgpack-node/compare/e04c9b55f98d64512174d6e859b8294b729659a2...HEAD +[Unreleased]: https://github.com/msgpack/msgpack-node/compare/v3.0.0...HEAD +[3.0.0]: https://github.com/msgpack/msgpack-node/compare/e04c9b55f98d64512174d6e859b8294b729659a2...HEAD [2.0.0]: https://github.com/msgpack/msgpack-node/commit/e04c9b55f98d64512174d6e859b8294b729659a2 diff --git a/COVERAGE.md b/COVERAGE.md index 13d2bca..80b8a6f 100644 --- a/COVERAGE.md +++ b/COVERAGE.md @@ -1,4 +1,4 @@ -# Coverage — msgpack 2.0.0 +# Coverage — msgpack 3.0.0 `npm run coverage` runs both halves and fails the build under 95%. @@ -126,12 +126,15 @@ gcovr --root . --filter src/ --exclude deps/ --no-markers --txt-metric branch -- ## What the new tests cover +- `test/bigint.test.js` — `#37` Number-vs-BigInt rule: uint64 of 1 stays + Number, `MAX_SAFE_INTEGER` stays Number, reporter uint64 round-trips as + BigInt, out-of-range BigInt throws, rounded Number bits are not recovered. - `test/coverage-native.test.js` (57 tests) — hand-built wire buffers for every MessagePack format family, including the ones `pack()` never emits (float32, str8/16/32, bin16/32, array32, map16/32, all eight ext forms, negative fixint); a truncation point for every header and payload; `kMaxBytes` / `kMaxContainer` / `kMaxDepth` rejections; `0xc1`; the pack-side - type dispatch (Symbol, BigInt, non-finite numbers, integer edges, undefined, + type dispatch (Symbol, non-finite numbers, integer edges, undefined, zero-argument and multi-argument `pack`); Date failure modes; `toJSON` failure modes and mark cleanup; and a worker that nests 600 packs deep to saturate the thread-local sbuffer pool and reach the "pool is full, free it" diff --git a/README.md b/README.md index 3567dd4..1ab38d5 100644 --- a/README.md +++ b/README.md @@ -2,8 +2,8 @@ and de-serializes JavaScript values with [MessagePack](https://msgpack.org). Packed output is a `Buffer` and is typically much smaller than JSON. -Version 2.0 requires **Node.js 18+**, vendors **msgpack-c c-7.0.2**, and -rejects oversized unpack headers instead of allocating them. See +Version 3.0 requires **Node.js 18+**, vendors **msgpack-c c-7.0.2**, and +unpacks 64-bit integers outside `Number.MAX_SAFE_INTEGER` as `bigint`. See [`SECURITY.md`](SECURITY.md). ### Usage @@ -19,10 +19,10 @@ const oo = msgpack.unpack(b); assert.deepEqual(oo, o); ``` -`pack()` accepts any JSON-like value plus Node `Buffer`s and `Date`s. -`unpack()` consumes a `Buffer` and returns a JavaScript value, or `null` if -the buffer is a truncated (incomplete) MessagePack object. Oversized -array/map/string bombs throw. +`pack()` accepts any JSON-like value plus Node `Buffer`s, `Date`s, and +`bigint` values in the int64/uint64 range. `unpack()` consumes a `Buffer` +and returns a JavaScript value, or `null` if the buffer is a truncated +(incomplete) MessagePack object. Oversized array/map/string bombs throw. A streaming helper wraps a readable socket and emits `msg`, plus `error` when a packet cannot be unpacked (the offending buffer is dropped): @@ -39,13 +39,14 @@ ms.on('error', (e) => { ms.send({ hello: 'world' }); ``` -### Type mapping (2.0) +### Type mapping (3.0) Packing: * `undefined` / `null` → nil * `boolean` → bool -* finite integers → uint/int +* finite integers (`number` or `bigint` in the 64-bit range) → uint/int +* `bigint` outside uint64/int64 → throws * other numbers → float64 * `string` → str (UTF-8) * `Date` → str (ISO 8601, `toISOString()`), at any nesting level @@ -57,15 +58,24 @@ Packing: integer keys, not dropped * functions, circular refs, and nesting deeper than 512 throw +A `number` that is already rounded (for example `18446464814936021000`) packs +on the Number path. Lost bits are not recovered. + Unpacking: * nil → `null` -* bool / int / float → JS boolean / number +* bool / float → JS boolean / number +* int whose magnitude ≤ `Number.MAX_SAFE_INTEGER` → `number` (a uint64 of + `1` is Number `1`; `Number.MAX_SAFE_INTEGER` stays Number) +* int whose magnitude > `Number.MAX_SAFE_INTEGER` → `bigint` * str → `string` * bin → `Buffer` * array / map → Array / Object * ext → throws +So `unpack(pack(1n))` is Number `1`, and `unpack(pack(18446464814936021036n))` +is that same `bigint`. + `unpack.bytes_remaining` is the number of unused trailing bytes after the last successful (or attempted) unpack. Stream uses that to splice leftover data. diff --git a/index.d.ts b/index.d.ts index f768aff..7f1175d 100644 --- a/index.d.ts +++ b/index.d.ts @@ -1,4 +1,4 @@ -// Type definitions for msgpack 2.0.0 +// Type definitions for msgpack 3.0.0 // Project: https://github.com/msgpack/msgpack-node /// @@ -10,12 +10,21 @@ import { EventEmitter } from 'events'; * * A single argument is packed as itself; two or more are packed as an array * of that many elements. + * + * `bigint` values in the int64/uint64 range pack as MessagePack integers + * (smallest family that fits). Values outside that range throw. A `number` + * that has already lost bits below 2^53 stays on the Number path; lost bits + * are not recovered. */ export function pack(...values: any[]): Buffer; /** * Deserialize the first MessagePack value in `buf`. * + * Integers whose magnitude is greater than `Number.MAX_SAFE_INTEGER` return + * as `bigint`. Values that fit stay `number`, even if the wire type is + * uint64 or int64 (a uint64 of 1 is Number 1). + * * Returns `null` when the buffer holds an incomplete value, in which case * `unpack.bytes_remaining` equals `buf.length`. Throws on malformed input or * when a container/string/bin header exceeds the decoder's limits. diff --git a/package-lock.json b/package-lock.json index 6bac0d7..155a1ff 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "msgpack", - "version": "2.0.0", + "version": "3.0.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "msgpack", - "version": "2.0.0", + "version": "3.0.0", "license": "BSD-3-Clause", "dependencies": { "nan": "^2.23.1" diff --git a/package.json b/package.json index 9797ffa..384055b 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "msgpack", "description": "A space-efficient object serialization library for Node.js", - "version": "2.0.0", + "version": "3.0.0", "homepage": "https://github.com/msgpack/msgpack-node", "author": "Peter Griess ", "contributors": [ @@ -34,7 +34,7 @@ "nan": "^2.23.1" }, "scripts": { - "test": "node --test test/cli.test.js test/coverage-native.test.js test/msgpack.test.js test/regression.test.js test/security.test.js test/worker.test.js", + "test": "node --test test/bigint.test.js test/cli.test.js test/coverage-native.test.js test/msgpack.test.js test/regression.test.js test/security.test.js test/worker.test.js", "bench": "node test/benchmark/benchmark.js", "rebuild": "node-gyp rebuild", "coverage": "npm run coverage:js && npm run coverage:native", diff --git a/src/msgpack.cc b/src/msgpack.cc index 5c9c9bb..0c08285 100644 --- a/src/msgpack.cc +++ b/src/msgpack.cc @@ -30,6 +30,10 @@ const int kMaxPackDepth = 512; /* Largest/smallest doubles that survive a cast to uint64_t/int64_t. */ const double kTwoPow64 = 18446744073709551616.0; const double kInt64Min = -9223372036854775808.0; +/* Integers inside this magnitude stay JS Number on unpack, regardless of + * wire width. Outside it they become BigInt so uint64/int64 stay exact. */ +const uint64_t kMaxSafeInteger = 9007199254740991ULL; +const int64_t kMinSafeInteger = -9007199254740991LL; const size_t kSbufferPoolMax = 512; enum ScanStatus { @@ -523,6 +527,22 @@ static void JsToMsgpack(msgpack_packer* pk, v8::Local o, int depth) { } else { rc = msgpack_pack_double(pk, d); } + } else if (o->IsBigInt()) { + /* v8::BigInt, not Number: a JS Number has already lost bits below + * 2^53 and must stay on the double/uint64-from-double path above. */ + v8::Local bi = o.As(); + bool lossless = false; + const int64_t s = bi->Int64Value(&lossless); + if (lossless) { + rc = msgpack_pack_int64(pk, s); + } else { + lossless = false; + const uint64_t u = bi->Uint64Value(&lossless); + if (!lossless) { + throw MsgpackException(Error("cannot pack BigInt outside 64-bit range")); + } + rc = msgpack_pack_uint64(pk, u); + } } else if (o->IsString()) { Nan::Utf8String bytes(o); rc = msgpack_pack_str(pk, bytes.length()); @@ -578,11 +598,17 @@ static v8::Local MsgpackToJs(const msgpack_object* mo) { case MSGPACK_OBJECT_BOOLEAN: return Nan::New(mo->via.boolean); case MSGPACK_OBJECT_POSITIVE_INTEGER: - /* Values that fit in 2^53-1 stay as Number; larger become the - * closest Number (legacy behavior). */ - return Nan::New(static_cast(mo->via.u64)); + /* Wire width does not decide the JS type: a uint64 of 1 is Number 1. + * Only values outside Number.MAX_SAFE_INTEGER become BigInt. */ + if (mo->via.u64 <= kMaxSafeInteger) { + return Nan::New(static_cast(mo->via.u64)); + } + return v8::BigInt::NewFromUnsigned(v8::Isolate::GetCurrent(), mo->via.u64); case MSGPACK_OBJECT_NEGATIVE_INTEGER: - return Nan::New(static_cast(mo->via.i64)); + if (mo->via.i64 >= kMinSafeInteger) { + return Nan::New(static_cast(mo->via.i64)); + } + return v8::BigInt::New(v8::Isolate::GetCurrent(), mo->via.i64); case MSGPACK_OBJECT_FLOAT32: case MSGPACK_OBJECT_FLOAT64: return Nan::New(mo->via.f64); diff --git a/test/bigint.test.js b/test/bigint.test.js new file mode 100644 index 0000000..3f343ff --- /dev/null +++ b/test/bigint.test.js @@ -0,0 +1,102 @@ +'use strict'; + +const { describe, it } = require('node:test'); +const assert = require('node:assert/strict'); +const msgpack = require('../lib/msgpack'); + +function b(...bytes) { + return Buffer.from(bytes); +} + +describe('BigInt 64-bit integers (#37)', () => { + const reporter = 18446464814936021036n; + const reporterWire = b(0xcf, 0xff, 0xff, 0x02, 0x04, 0x00, 0x00, 0xd8, 0x2c); + + it('round-trips a uint64 BigInt that a Number would round', () => { + assert.equal(msgpack.unpack(msgpack.pack(reporter)), reporter); + assert.equal(typeof msgpack.unpack(msgpack.pack(reporter)), 'bigint'); + }); + + it('unpacks the reporter uint64 wire as that BigInt', () => { + assert.equal(msgpack.unpack(reporterWire), reporter); + assert.deepEqual(msgpack.pack(reporter), reporterWire); + }); + + it('unpacks a uint64 of 1 as Number 1, not BigInt', () => { + const wire = b(0xcf, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01); + const got = msgpack.unpack(wire); + assert.equal(typeof got, 'number'); + assert.equal(got, 1); + }); + + it('unpacks Number.MAX_SAFE_INTEGER as Number regardless of wire width', () => { + const fromNumber = msgpack.unpack(msgpack.pack(Number.MAX_SAFE_INTEGER)); + assert.equal(typeof fromNumber, 'number'); + assert.equal(fromNumber, Number.MAX_SAFE_INTEGER); + + const u64 = b(0xcf, 0x00, 0x1f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff); + const fromWire = msgpack.unpack(u64); + assert.equal(typeof fromWire, 'number'); + assert.equal(fromWire, Number.MAX_SAFE_INTEGER); + + const fromBigInt = msgpack.unpack(msgpack.pack(9007199254740991n)); + assert.equal(typeof fromBigInt, 'number'); + assert.equal(fromBigInt, Number.MAX_SAFE_INTEGER); + }); + + it('unpacks a safe BigInt as Number after pack', () => { + assert.equal(msgpack.unpack(msgpack.pack(1n)), 1); + assert.equal(typeof msgpack.unpack(msgpack.pack(1n)), 'number'); + assert.equal(msgpack.pack(1n)[0], 0x01); + assert.equal(msgpack.pack(10n)[0], 0x0a); + }); + + it('encodes BigInt with the smallest integer family that fits', () => { + assert.equal(msgpack.pack(255n)[0], 0xcc); + assert.equal(msgpack.pack(256n)[0], 0xcd); + assert.equal(msgpack.pack(-32n)[0], 0xe0); + assert.equal(msgpack.pack(-33n)[0], 0xd0); + assert.deepEqual( + msgpack.pack(2n ** 64n - 1n), + b(0xcf, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff) + ); + assert.deepEqual( + msgpack.pack(-(2n ** 63n)), + b(0xd3, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00) + ); + assert.equal(msgpack.unpack(msgpack.pack(2n ** 64n - 1n)), 2n ** 64n - 1n); + assert.equal(msgpack.unpack(msgpack.pack(-(2n ** 63n))), -(2n ** 63n)); + }); + + it('throws a catchable error for BigInt outside uint64/int64', () => { + assert.throws(() => msgpack.pack(2n ** 64n), /cannot pack BigInt outside 64-bit range/); + assert.throws( + () => msgpack.pack(-(2n ** 63n) - 1n), + /cannot pack BigInt outside 64-bit range/ + ); + try { + msgpack.pack(2n ** 64n); + assert.fail('expected throw'); + } catch (err) { + assert.ok(err instanceof Error); + assert.match(err.message, /cannot pack BigInt outside 64-bit range/); + } + }); + + it('does not recover lost bits from a Number that is already rounded', () => { + const n = 18446464814936021000; + assert.equal(typeof n, 'number'); + const packed = msgpack.pack(n); + /* Still the Number integer/float path, not the BigInt encoder. */ + assert.notDeepEqual(packed, reporterWire); + assert.notEqual(msgpack.unpack(packed), reporter); + }); + + it('packs nested BigInt values in arrays and objects', () => { + const payload = { n: reporter, a: [1n, reporter] }; + const got = msgpack.unpack(msgpack.pack(payload)); + assert.equal(got.n, reporter); + assert.equal(got.a[0], 1); + assert.equal(got.a[1], reporter); + }); +}); diff --git a/test/coverage-native.test.js b/test/coverage-native.test.js index 4756d0f..5a1c6ec 100644 --- a/test/coverage-native.test.js +++ b/test/coverage-native.test.js @@ -42,9 +42,17 @@ describe('unpack format families', () => { assert.equal(msgpack.unpack(b(0xcc, 0xff)), 255); assert.equal(msgpack.unpack(b(0xcd, 0xff, 0xff)), 65535); assert.equal(msgpack.unpack(b(0xce, 0xff, 0xff, 0xff, 0xff)), 4294967295); + /* uint64 of 1 and of MAX_SAFE_INTEGER stay Number. */ + const u64one = b(0xcf, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01); + assert.equal(typeof msgpack.unpack(u64one), 'number'); + assert.equal(msgpack.unpack(u64one), 1); + const u64safe = b(0xcf, 0x00, 0x1f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff); + assert.equal(typeof msgpack.unpack(u64safe), 'number'); + assert.equal(msgpack.unpack(u64safe), Number.MAX_SAFE_INTEGER); + /* 2^53+1 is outside MAX_SAFE_INTEGER, so BigInt keeps it exact. */ assert.equal( msgpack.unpack(b(0xcf, 0x00, 0x20, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01)), - 9007199254740993 /* nearest double to 2^53+1 */ + 9007199254740993n ); }); @@ -52,9 +60,15 @@ describe('unpack format families', () => { assert.equal(msgpack.unpack(b(0xd0, 0x80)), -128); assert.equal(msgpack.unpack(b(0xd1, 0x80, 0x00)), -32768); assert.equal(msgpack.unpack(b(0xd2, 0x80, 0x00, 0x00, 0x00)), -2147483648); + const i64neg1 = b(0xd3, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff); + assert.equal(typeof msgpack.unpack(i64neg1), 'number'); + assert.equal(msgpack.unpack(i64neg1), -1); + const i64safe = b(0xd3, 0xff, 0xe0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01); + assert.equal(typeof msgpack.unpack(i64safe), 'number'); + assert.equal(msgpack.unpack(i64safe), -Number.MAX_SAFE_INTEGER); assert.equal( - msgpack.unpack(b(0xd3, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff)), - -1 + msgpack.unpack(b(0xd3, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00)), + -(2n ** 63n) ); }); @@ -341,15 +355,20 @@ describe('pack type dispatch', () => { it('packs integers across the signed and unsigned edges', () => { const cases = [ 0, 1, -1, 127, -32, 255, -128, 65535, -32768, - 4294967295, -2147483648, 2 ** 53, -(2 ** 53), + 4294967295, -2147483648, Number.MAX_SAFE_INTEGER, -Number.MAX_SAFE_INTEGER, ]; for (const n of cases) { - assert.equal(msgpack.unpack(msgpack.pack(n)), n, String(n)); + const got = msgpack.unpack(msgpack.pack(n)); + assert.equal(typeof got, 'number', String(n)); + assert.equal(got, n, String(n)); } - /* 2^63 and -2^63 still take the integer path (they fit uint64/int64). */ - assert.equal(msgpack.unpack(msgpack.pack(2 ** 63)), 2 ** 63); - assert.equal(msgpack.unpack(msgpack.pack(-(2 ** 63))), -(2 ** 63)); + /* 2^53, 2^63 and -2^63 take the integer path (they fit uint64/int64) + * but unpack as BigInt because they sit outside MAX_SAFE_INTEGER. */ + assert.equal(msgpack.unpack(msgpack.pack(2 ** 53)), 2n ** 53n); + assert.equal(msgpack.unpack(msgpack.pack(-(2 ** 53))), -(2n ** 53n)); + assert.equal(msgpack.unpack(msgpack.pack(2 ** 63)), 2n ** 63n); + assert.equal(msgpack.unpack(msgpack.pack(-(2 ** 63))), -(2n ** 63n)); /* One ulp below 2^64 is the largest double that survives the uint64 * cast; the next one up must fall through to the double path. */ assert.equal(msgpack.pack(2 ** 64 - 2048)[0], 0xcf); @@ -363,9 +382,8 @@ describe('pack type dispatch', () => { assert.equal(msgpack.unpack(msgpack.pack(false)), false); }); - it('refuses to pack a Symbol or a BigInt', () => { + it('refuses to pack a Symbol', () => { assert.throws(() => msgpack.pack(Symbol('x')), /cannot pack object/); - assert.throws(() => msgpack.pack(10n), /cannot pack object/); }); it('refuses to pack a function', () => { diff --git a/test/regression.test.js b/test/regression.test.js index d272240..934179d 100644 --- a/test/regression.test.js +++ b/test/regression.test.js @@ -31,9 +31,16 @@ describe('regressions', () => { }); it('still packs integers at the edges of the integer path', () => { - for (const n of [0, 1, -1, 2 ** 53, -(2 ** 53), 2 ** 63, -(2 ** 63)]) { + for (const n of [0, 1, -1]) { + assert.equal(typeof msgpack.unpack(msgpack.pack(n)), 'number'); assert.equal(msgpack.unpack(msgpack.pack(n)), n); } + /* Numbers that already sit outside MAX_SAFE_INTEGER pack on the integer + * path when they fit uint64/int64, then unpack as BigInt. */ + assert.equal(msgpack.unpack(msgpack.pack(2 ** 53)), 2n ** 53n); + assert.equal(msgpack.unpack(msgpack.pack(-(2 ** 53))), -(2n ** 53n)); + assert.equal(msgpack.unpack(msgpack.pack(2 ** 63)), 2n ** 63n); + assert.equal(msgpack.unpack(msgpack.pack(-(2 ** 63))), -(2n ** 63n)); }); it('unpacks a map written by Python msgpack (msgpack-node#10)', () => {