diff --git a/.github/workflows/integration-mongodb.yml b/.github/workflows/integration-mongodb.yml index 3df405a75..03223f967 100644 --- a/.github/workflows/integration-mongodb.yml +++ b/.github/workflows/integration-mongodb.yml @@ -51,6 +51,55 @@ jobs: - name: Run MongoDB pytest suite run: devtools/run-mongodb-tests -- --pytest --comprehensive --parallel + # SnapshotEngine consistency check against a disposable replica set. The + # tests build their own per-test scratch databases (snap_cat and + # _data) and drop them; they run in this job because it is the one that + # exports EXTENDDB_TEST_MONGODB_URI to cargo test. + mongodb-snapshot: + runs-on: extenddb_ubuntu-2404_4-core + timeout-minutes: 30 + steps: + - uses: actions/checkout@v6 + - uses: dtolnay/rust-toolchain@stable + - uses: Swatinem/rust-cache@v2 + with: + cache-on-failure: true + - name: Start replica set + # Same bootstrap developers run locally; --mongo-only skips the + # server and prints the URI. No pipe: a bootstrap failure must fail + # this step, not the next one's connection attempt. + run: devtools/run-mongodb-tests --mongo-only --mongo-port 27018 + - name: Start standalone mongod + # No --replSet: a_standalone_server_is_unsupported checks that a + # server which cannot serve snapshot reads answers NotAReplicaSet and + # the open maps it to Unsupported. Published on 127.0.0.1 only. + run: | + docker run -d --name extenddb-snapshot-standalone-27019 \ + -p 127.0.0.1:27019:27017 mongo:7 + for _ in $(seq 1 60); do + if docker exec extenddb-snapshot-standalone-27019 \ + mongosh --quiet --eval 'db.runCommand({ ping: 1 }).ok' 2>/dev/null \ + | grep -qx 1; then + exit 0 + fi + sleep 1 + done + echo "standalone mongod did not answer ping within 60 s" >&2 + docker logs extenddb-snapshot-standalone-27019 >&2 || true + exit 1 + - name: Run snapshot tests + # The tests serialise themselves on the snapshot hooks lock; the + # default parallel runner is what developers get locally and what + # this checks. + env: + EXTENDDB_TEST_MONGODB_URI: mongodb://127.0.0.1:27018/?directConnection=true + EXTENDDB_TEST_MONGODB_STANDALONE_URI: mongodb://127.0.0.1:27019/?directConnection=true + EXTENDDB_TEST_MONGODB_REQUIRED: "1" + run: cargo test -p extenddb-storage-mongodb snapshot_engine + - name: Stop MongoDB containers + if: always() + run: docker rm -f extenddb-runtests-mongo-27018 extenddb-snapshot-standalone-27019 + mongodb-rust: runs-on: extenddb_ubuntu-2404_4-core timeout-minutes: 40 @@ -85,12 +134,13 @@ jobs: mongodb-integration: runs-on: extenddb_ubuntu-2404_4-core timeout-minutes: 40 - needs: [mongodb-pytest, mongodb-rust, mongodb-production-build] + needs: [mongodb-pytest, mongodb-rust, mongodb-production-build, mongodb-snapshot] if: always() steps: - run: | if [ "${{ needs.mongodb-pytest.result }}" != "success" ] || \ [ "${{ needs.mongodb-rust.result }}" != "success" ] || \ - [ "${{ needs.mongodb-production-build.result }}" != "success" ]; then + [ "${{ needs.mongodb-production-build.result }}" != "success" ] || \ + [ "${{ needs.mongodb-snapshot.result }}" != "success" ]; then exit 1 fi diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 8a4818f75..5d9402a2e 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -305,13 +305,18 @@ jobs: # pause writers with a test hook. They build their own throwaway databases; # the connection string is the server, not a database. - name: Run PostgreSQL storage-level tests + # table_snapshot: the SnapshotEngine consistency check; it needs the + # shared page-size hook, compiled only into the crate's own test + # build, and scratch databases it creates and drops itself. env: EXTENDDB_TEST_PG_CONNECTION_STRING: postgresql://postgres:devpass@127.0.0.1:5432 + # table_snapshot fails rather than skips without the string above. + EXTENDDB_TEST_PG_REQUIRED: "1" run: >- cargo test --release -p extenddb-storage-postgres --test vector_control_plane --test put_create_race --test twi_conflict --test stream_shard_retention --test backup_restore - --test gsi_online_build --test gsi_stale_writer + --test gsi_online_build --test gsi_stale_writer --test table_snapshot # The daemonized server logs to syslog; dump it so server-side failures # are diagnosable from the job log. diff --git a/Cargo.lock b/Cargo.lock index 003569679..ede01e9c4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1320,8 +1320,11 @@ version = "0.1.14" dependencies = [ "base64 0.22.1", "bigdecimal", + "flate2", + "md-5 0.10.6", "serde", "serde_json", + "sha2 0.10.9", "thiserror", "time", "uuid", @@ -1449,6 +1452,7 @@ dependencies = [ "extenddb-auth", "extenddb-core", "extenddb-storage", + "extenddb-storage-mongodb", "futures", "mongodb", "proptest", @@ -1477,6 +1481,7 @@ dependencies = [ "extenddb-auth", "extenddb-core", "extenddb-storage", + "extenddb-storage-postgres", "futures", "libc", "pgvector", @@ -1506,6 +1511,7 @@ dependencies = [ "extenddb-auth", "extenddb-core", "extenddb-storage", + "extenddb-storage-sqlite", "futures", "libc", "rand 0.9.4", diff --git a/Cargo.toml b/Cargo.toml index afa4d746c..31e0b2296 100755 --- a/Cargo.toml +++ b/Cargo.toml @@ -80,6 +80,10 @@ dashmap = "6" # Crypto & checksums crc32fast = "1" +# Pinned to the versions already resolved in Cargo.lock. +flate2 = "1.1.9" +md-5 = "0.10.6" +sha2 = "0.10.9" uuid = { version = "1", features = ["v4"] } base64 = "0.22" bigdecimal = { version = "0.4", features = ["serde"] } diff --git a/crates/core/Cargo.toml b/crates/core/Cargo.toml index 971153844..7e04256bf 100755 --- a/crates/core/Cargo.toml +++ b/crates/core/Cargo.toml @@ -13,5 +13,8 @@ serde_json = { workspace = true } thiserror = { workspace = true } base64 = { workspace = true } bigdecimal = { workspace = true } +flate2 = { workspace = true } +md-5 = { workspace = true } +sha2 = { workspace = true } time = { workspace = true } uuid = { workspace = true } diff --git a/crates/core/src/backup_format/data_file.rs b/crates/core/src/backup_format/data_file.rs new file mode 100644 index 000000000..10054b8e9 --- /dev/null +++ b/crates/core/src/backup_format/data_file.rs @@ -0,0 +1,2880 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! Data file encoding and decoding. +//! +//! A data file is gzip-compressed newline-delimited JSON, one +//! `{"Item": }` object per line, the encoding Amazon +//! `DynamoDB` writes for `DYNAMODB_JSON` exports. [`encode_item_line`] is the +//! one encoding of that line; the engine's `ExportTableToPointInTime` writer +//! calls it, so the two paths agree byte for byte by construction. +//! +//! The reader contract, which v1 freezes: the container is a gzip (RFC +//! 1952) stream; several gzip members concatenated are accepted and their +//! decompressed bytes are one stream, so a line may span a member boundary +//! (the reader is flate2's `MultiGzDecoder`; a reader built on a +//! single-member decoder, Go's `gzip.Reader` with `Multistream(false)`, +//! would truncate such a file). The writer emits one member. A line ends in +//! `\n`, with one exception: the last line's `\n` may be absent, and that +//! line is an item like any other, counted in `uncompressed_bytes` without a +//! newline (the writer always ends the last line with one). The object has +//! the one member `Item` (an unknown member is a +//! [`FormatError::Json`]); the item names each top-level attribute once (a +//! repeated name is a [`FormatError::Json`], as `ImportTable` refuses it; +//! nested maps are read by the wire `AttributeValue` deserializer, which +//! this format leaves alone, so a repeated name inside an `M` is last-wins); +//! a repeated type descriptor at the top of an attribute value +//! (`{"S":"x","S":"y"}`) is refused on this path, because the line is +//! parsed from text and the deserializer sees both keys, while the request +//! path, which parses a body to a `serde_json::Value` first, keeps the last +//! (`S("y")`); the writer emits neither shape, so no written file is +//! affected. There are no blank lines. JSON whitespace around the object is +//! JSON's own rule and is accepted, so a `\r` before the newline (a CRLF +//! file) parses; it is a byte of the line and counts toward +//! `uncompressed_bytes`. The writer emits neither. The lines are UTF-8 +//! without a byte order mark. +//! +//! A second reader of the same line format exists outside this module: the +//! engine's `ImportTable` reader for `DYNAMODB_JSON` input +//! (`read_dynamodb_json` in `crates/engine/src/import_export_io.rs`), which +//! reads `ExtendDB`'s own exports. It is laxer than the contract above and is +//! not bound to it: it skips blank and whitespace-only lines and trims +//! Unicode whitespace around each line; it reads an object without an `Item` +//! member as a bare item, and ignores other members beside `Item`; it keeps +//! the last of two top-level attributes with one name (and the last of two +//! `Item` members); it has no line cap, and reads the whole file into +//! memory through a `serde_json::Value` and a clone of the item; its errors +//! name the line but not the cause. Nothing here changes it; a restore reads +//! data files only through [`DataFileDecoder`], which +//! [`super::ExtenddbManifest::reader`] wraps. +//! +//! Checksums: `sha256` and `md5` are computed over the compressed bytes, the +//! object as stored, which is what `extenddb-manifest.json` records per data +//! file and what `manifest-files.json` records as `md5Checksum`, and so is +//! the compressed size. Item and uncompressed byte counts are computed over +//! the line stream. +//! +//! Memory: [`DataFileEncoder`] writes through any [`Write`] and hashes the +//! compressed bytes as they leave the compressor, so a caller streaming to a +//! file or an upload holds the gzip window and one serialized line (a buffer +//! as long as the longest line written so far), not the file. [`DataFileDecoder`] reads from any [`Read`], hashes and counts the +//! compressed bytes as they are consumed, refuses the source as soon as it +//! yields more bytes than recorded, and checks the digests when the stream +//! ends. Decoding a line materializes its item, which takes up to about 128 +//! times the line's size; see [`DataFileDecoder`] for the bound. +//! A caller that must not act on any item before the digests are known (a +//! restore) runs the bytes through [`CompressedHasher`] first, keeping a copy +//! on local disk, and decodes from that copy without hashing it again +//! ([`super::DataFileEntry::decoder_prehashed`], +//! [`super::ExtenddbManifest::reader_prehashed`]); +//! [`DataFileDecoder::verified`] does both steps for a slice already in +//! memory, hashing it once. + +use std::io::{BufRead, BufReader, Read, Write}; + +use base64::Engine; +use base64::engine::general_purpose::STANDARD as BASE64; +use flate2::Compression; +use flate2::read::MultiGzDecoder; +use flate2::write::GzEncoder; +use md5::Md5; +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; + +use super::FormatError; +use super::error::{JsonError, truncated}; +use super::strict_item::item_without_duplicates; +use crate::types::Item; + +/// Longest line, newline included, a data file may carry; the encoder refuses +/// a longer item and the decoder refuses a longer line (a hostile or corrupt +/// gzip object can otherwise allocate gigabytes). A line longer than the cap +/// yields [`FormatError::LineTooLong`] before the whole line is materialized; +/// a line of exactly the cap is accepted. +/// +/// Sizing: the cap must hold the line of every item the default item limit +/// (409,600 bytes, 400 KB as `DynamoDB` sizes items) admits, or +/// `CreateBackup` would refuse a table the service accepted. The largest +/// expansion from item size to line bytes is a list of number sets. A number +/// with one or two significant digits costs 2 bytes of item size +/// ([`crate::types::item_size_bytes`]), and the longest such normalized +/// string is 133 characters (`1.1E-130` is `0.`, 129 zeros, `11`), which a +/// set writes as `"...",`: 136 bytes of line for 2 bytes of item, 68x. A set +/// member has no per-member size overhead, and members are distinct only +/// within one set, so a list of sets repeats the same 90 two-digit values +/// at `E-130` in every set. Measured with `item_size_bytes` and +/// [`encode_item_line`] at the default limit (the test +/// `largest_expansion_at_the_default_item_limit_fits_the_cap` builds it): +/// +/// | item | item bytes | line bytes | ratio | +/// |---|---|---|---| +/// | list of 2,262 `NS` of the 90 values `1.0E-130`..`9.9E-130` | 409,429 | 27,686,918 | 67.62 | +/// | list of 1,134 `NS` of 180 values (`E-130` and `E-129`) | 409,381 | 27,648,092 | 67.54 | +/// | list of 136,531 `N` `1.1E-130` | 409,600 | 19,387,440 | 47.33 | +/// | list of 136,531 `N` `1E-130` (132 characters) | 409,600 | 19,250,909 | 47.00 | +/// | list of 102,398 `N` `-1E-130` (a sign costs a byte) | 409,599 | 14,540,554 | 35.50 | +/// | list of 409,593 empty strings | 409,600 | 3,686,375 | 9.00 | +/// | list of one-member `SS` / list of `S`, each of one control character (`\u0001`) | 409,599 | 3,686,366 / 3,071,978 | 9.00 / 7.50 | +/// | list of one-member `BS`, each of one byte | 409,599 | 3,276,774 | 8.00 | +/// | list of maps (`{"": NULL}`, `{}`) | 409,597 | 1,706,663 / 921,620 | 4.17 / 2.25 | +/// +/// The cap, 32 MiB (33,554,432 bytes), holds the worst case with a 1.21x +/// margin. It is exceeded from an operator item limit +/// (`max_item_size_bytes`) of about 496,000 bytes at the measured 67.62x, +/// 493,000 at the 68x bound; a deployment that raises the limit above that +/// can have items whose line exceeds the cap, and the encoder refuses them +/// at backup time rather than letting the restore find them. +/// +/// The cap bounds the line, not the item a line decodes to; see +/// [`DataFileDecoder`] for the decoder's peak memory. +/// +/// A data file has no blank lines: every line is one item. A blank line is +/// a [`FormatError::Json`] like any other unparseable line. +pub const MAX_LINE_BYTES: usize = 32 * 1024 * 1024; + +/// Serialize one item as the data-file line `{"Item": ...}` plus a newline: +/// the same bytes the engine's `ExportTableToPointInTime` writer produces, so +/// the two paths share one encoding by construction. The line is not checked +/// against [`MAX_LINE_BYTES`]; [`DataFileEncoder::write_item`] does that. +/// [`encode_item_line_into`] writes the same bytes into a buffer the caller +/// keeps. +/// +/// # Errors +/// +/// Returns the serializer's error when the item cannot be serialized. +pub fn encode_item_line(item: &Item) -> Result, serde_json::Error> { + // `serde_json::to_vec`'s initial capacity, so a short line costs one + // allocation as it did through `to_vec`. + let mut line = Vec::with_capacity(128); + encode_item_line_into(item, &mut line)?; + Ok(line) +} + +/// [`encode_item_line`] into `line`, which is cleared first, so a caller +/// encoding many items reuses one allocation. This is the one encoding of a +/// line; [`encode_item_line`] and [`DataFileEncoder::write_item`] call it. +/// After an error `line` holds a partial line. +/// +/// # Errors +/// +/// Returns the serializer's error when the item cannot be serialized. +pub fn encode_item_line_into(item: &Item, line: &mut Vec) -> Result<(), serde_json::Error> { + line.clear(); + serde_json::to_writer(&mut *line, &ItemLineRef { item })?; + line.push(b'\n'); + Ok(()) +} + +/// Lowercase hex encoding of a byte slice. +pub(super) fn hex_encode(bytes: &[u8]) -> String { + use std::fmt::Write as _; + let mut out = String::with_capacity(bytes.len() * 2); + for byte in bytes { + let _ = write!(out, "{byte:02x}"); + } + out +} + +/// Strip the single trailing `\n` from a line. A `\r` before it is JSON +/// whitespace and left to the parser (module doc). +fn strip_trailing_newline(line: &[u8]) -> &[u8] { + line.strip_suffix(b"\n").unwrap_or(line) +} + +/// The single-key wrapper each data file line holds. The item refuses a +/// repeated top-level attribute name (module docs). +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct ItemLine { + #[serde(rename = "Item", deserialize_with = "item_without_duplicates")] + item: Item, +} + +/// Borrowing form of [`ItemLine`] for the writer, so an item is serialized in +/// place rather than cloned into a `serde_json::Value`. +#[derive(Serialize)] +struct ItemLineRef<'a> { + #[serde(rename = "Item")] + item: &'a Item, +} + +/// Checksums and counts produced while encoding a data file. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct DataFileChecksums { + /// Number of items encoded. + pub item_count: u64, + /// Lowercase hex sha256 of the compressed bytes. + pub sha256_hex: String, + /// Base64 MD5 of the compressed bytes, the `md5Checksum` encoding + /// `manifest-files.json` uses. + pub md5_base64: String, + /// Size of the compressed output. + pub compressed_bytes: u64, + /// Total size of the uncompressed lines, newlines included. + pub uncompressed_bytes: u64, +} + +/// What a decoder should verify. Only the members that are `Some` are +/// checked, so a caller holding just the `md5Checksum` from a +/// `manifest-files.json` line can still verify what it has. +#[derive(Debug, Clone, Default)] +pub struct DataFileExpectations { + /// Expected lowercase hex sha256 of the compressed bytes. + pub sha256_hex: Option, + /// Expected base64 MD5 of the compressed bytes. + pub md5_base64: Option, + /// Expected size of the compressed bytes. The decoder refuses the source + /// as soon as it has yielded one byte more, so a file recorded as small + /// cannot stream a long tail of unverified items before the digest + /// check; a shorter source is refused at end of stream. + pub compressed_bytes: Option, + /// Expected item count. + pub item_count: Option, + /// Expected uncompressed byte count. + pub uncompressed_bytes: Option, +} + +impl From<&DataFileChecksums> for DataFileExpectations { + fn from(checksums: &DataFileChecksums) -> Self { + Self { + sha256_hex: Some(checksums.sha256_hex.clone()), + md5_base64: Some(checksums.md5_base64.clone()), + compressed_bytes: Some(checksums.compressed_bytes), + item_count: Some(checksums.item_count), + uncompressed_bytes: Some(checksums.uncompressed_bytes), + } + } +} + +/// Digest of a data file's compressed bytes: what the manifests record and +/// what a reader verifies before trusting the file's contents. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct CompressedDigest { + /// Lowercase hex sha256 of the compressed bytes. + pub sha256_hex: String, + /// Base64 MD5 of the compressed bytes, the `md5Checksum` encoding + /// `manifest-files.json` uses. + pub md5_base64: String, + /// Number of compressed bytes. + pub bytes: u64, +} + +/// Incremental hasher over compressed bytes. Feed it the bytes of a data +/// file as they arrive (from a download, a file read, or the compressor) and +/// call [`CompressedHasher::finish`] for the digest. +#[derive(Default)] +pub struct CompressedHasher { + sha256: Sha256, + md5: Md5, + bytes: u64, +} + +impl CompressedHasher { + /// A hasher with nothing fed yet. + #[must_use] + pub fn new() -> Self { + Self::default() + } + + /// Feed bytes. + pub fn update(&mut self, bytes: &[u8]) { + self.sha256.update(bytes); + self.md5.update(bytes); + self.bytes += bytes.len() as u64; + } + + /// Finish and return the digest. + #[must_use] + pub fn finish(self) -> CompressedDigest { + CompressedDigest { + sha256_hex: hex_encode(&self.sha256.finalize()), + md5_base64: BASE64.encode(self.md5.finalize()), + bytes: self.bytes, + } + } + + /// Digest of a slice, in one call. + #[must_use] + pub fn digest(bytes: &[u8]) -> CompressedDigest { + let mut hasher = Self::new(); + hasher.update(bytes); + hasher.finish() + } +} + +impl DataFileExpectations { + /// Check the size and digest fields that are set against a computed + /// digest: the compressed size first, then sha256, then md5. + /// + /// # Errors + /// + /// Returns [`FormatError::CompressedSizeMismatch`] when the size differs, + /// and [`FormatError::ChecksumMismatch`] naming the first digest that + /// differs (the expected value is shown bounded: it may come from an + /// unvalidated `manifest-files.json` line). + pub fn verify_digest(&self, digest: &CompressedDigest) -> Result<(), FormatError> { + self.verify_compressed_size(digest.bytes)?; + if let Some(expected) = &self.sha256_hex + && expected != &digest.sha256_hex + { + return Err(FormatError::ChecksumMismatch { + subject: "data file sha256", + expected: truncated(expected), + computed: digest.sha256_hex.clone(), + }); + } + if let Some(expected) = &self.md5_base64 + && expected != &digest.md5_base64 + { + return Err(FormatError::ChecksumMismatch { + subject: "data file md5", + expected: truncated(expected), + computed: digest.md5_base64.clone(), + }); + } + Ok(()) + } + + /// Check the compressed size, when it is set, against `bytes`. + fn verify_compressed_size(&self, bytes: u64) -> Result<(), FormatError> { + match self.compressed_bytes { + Some(expected) if expected != bytes => Err(FormatError::CompressedSizeMismatch { + expected, + actual: bytes, + }), + _ => Ok(()), + } + } +} + +/// A [`Write`] that hashes everything written through it before passing it +/// on. The compressor writes into this, so the digest covers the bytes as +/// stored. +/// +/// `inner` is taken out by [`DataFileEncoder::into_inner`], and by the +/// encoder's `Drop` when the encoder is poisoned; the compressor's own `Drop` +/// then writes into nothing and its `BrokenPipe` is ignored, which is the +/// documented flate2 behaviour for a dropped encoder. +struct HashingWriter { + inner: Option, + hasher: CompressedHasher, +} + +impl Write for HashingWriter { + fn write(&mut self, buf: &[u8]) -> std::io::Result { + let Some(inner) = self.inner.as_mut() else { + return Err(std::io::Error::new( + std::io::ErrorKind::BrokenPipe, + "sink already taken", + )); + }; + // `Interrupted` is the one error std defines as "retry"; `write_all` + // already retries it on the item path, so retry it here too and the + // compressor's header and trailer writes see the same sink contract + // (`flush` below does the same). + let written = loop { + match inner.write(buf) { + Err(e) if e.kind() == std::io::ErrorKind::Interrupted => {} + other => break other?, + } + }; + // `Write::write` may return `Ok(0)` for a sink that accepts no more + // bytes. flate2's header and trailer writers loop until their bytes + // are accepted, so passing `Ok(0)` through would spin forever; std's + // `write_all` turns it into `WriteZero` for the same reason. + if written == 0 && !buf.is_empty() { + return Err(std::io::Error::new( + std::io::ErrorKind::WriteZero, + "sink accepted no bytes", + )); + } + self.hasher.update(&buf[..written]); + Ok(written) + } + + fn flush(&mut self) -> std::io::Result<()> { + let Some(inner) = self.inner.as_mut() else { + return Ok(()); + }; + loop { + match inner.flush() { + Err(e) if e.kind() == std::io::ErrorKind::Interrupted => {} + other => return other, + } + } + } +} + +/// A [`Read`] that counts, and unless the digest was verified before the +/// read (`hasher` is `None`) hashes, everything read through it. The +/// decompressor reads from this, so the digest covers exactly the compressed +/// bytes consumed. +/// +/// With a `limit` (the recorded compressed size) it asks the source for at +/// most one byte past the limit, and once it has that byte it fails every +/// read and sets `overrun`; the decoder turns that into +/// [`FormatError::CompressedSizeMismatch`]. One byte past is what tells a +/// longer object from one that ends at the limit. +struct HashingReader { + inner: R, + hasher: Option, + /// Compressed bytes read so far, hashed or not. + bytes: u64, + limit: Option, + overrun: bool, +} + +const OVERRUN: &str = "data file is longer than its recorded compressed size"; + +impl Read for HashingReader { + fn read(&mut self, buf: &mut [u8]) -> std::io::Result { + if self.overrun { + return Err(std::io::Error::other(OVERRUN)); + } + let buf = match self.limit { + Some(limit) => { + let room = limit.saturating_sub(self.bytes).saturating_add(1); + let len = usize::try_from(room).map_or(buf.len(), |room| room.min(buf.len())); + &mut buf[..len] + } + None => buf, + }; + // Retried here, as the encoder retries it on writes: flate2 passes an + // `Interrupted` from its source through `fill_buf`, and the decoder's + // errors are sticky, so a signal during a read would otherwise turn + // a good file into a reported corrupt one. + let read = loop { + match self.inner.read(buf) { + Err(e) if e.kind() == std::io::ErrorKind::Interrupted => {} + other => break other?, + } + }; + if let Some(hasher) = &mut self.hasher { + hasher.update(&buf[..read]); + } + self.bytes += read as u64; + if let Some(limit) = self.limit + && self.bytes > limit + { + self.overrun = true; + return Err(std::io::Error::other(OVERRUN)); + } + Ok(read) + } +} + +/// Streaming encoder: serializes items as `{"Item": ...}` lines into a gzip +/// stream written to `W`, hashing the compressed output as it goes. +/// +/// The gzip level is fixed at [`GZIP_LEVEL`] so two runs of the same build +/// encoding the same items produce the same bytes and checksums (compression +/// is not part of the format contract; the output also depends on the flate2 +/// backend and version, which `Cargo.lock` pins). +/// +/// A write error poisons the encoder: part of a line may have entered the +/// compressor, so the stream is damaged and no later write or `finish` can +/// produce a valid file. Every later call returns [`FormatError::Encode`]; +/// the caller discards the file and starts it again from its first item. +/// Only a successful [`Self::try_finish`] (or [`Self::finish`]) writes the +/// gzip trailer. Dropping an encoder in any other state (fresh, open with +/// items written, poisoned) writes nothing more into the sink: the sink is +/// taken out first, so the compressor's own `Drop` has nowhere to flush its +/// buffered data and trailer. Otherwise an encoder abandoned on a source +/// error or a cancellation would complete a valid gzip file of the items so +/// far, indistinguishable from a whole file without its manifest entry. A +/// caller whose sink needs cleanup (an upload to abort) takes it back with +/// [`Self::into_inner`] before dropping. +pub struct DataFileEncoder { + encoder: GzEncoder>, + /// The line being written, reused from one item to the next: its + /// capacity is the longest line written so far. + line: Vec, + item_count: u64, + uncompressed_bytes: u64, + state: EncoderState, +} + +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +enum EncoderState { + /// Accepting items. + Open, + /// `try_finish` succeeded; the trailer is written and no item may follow. + Finished, + /// A write or finish failed; the stream is damaged. + Poisoned, +} + +/// gzip compression level the encoder uses: flate2's default, pinned so two +/// runs of one build produce identical bytes for identical items. +/// +/// The encoder's own rate at this level, measured on one shared 16-core Xeon +/// Platinum 8124M (no SHA extensions), release build, one thread, writing to +/// `io::sink()`, sha256 and md5 included, item generation subtracted: about +/// 33 MB/s of line bytes for 1M synthetic 318-byte items (compression ratio +/// 0.20) and about 15 MB/s for 1,000 items at the 400 KB limit (a 475 KB line +/// of text, a 5,000-element `L` of `N` and a 500-entry `M`; ratio 0.26). A +/// backup job encodes on one thread, so this is its rate: about 8.5 hours per +/// TB of small-item lines. Not held constant: the host ran other builds (load +/// 1.5 to 2 during these runs), the items are synthetic, and the rate depends +/// on the data's compressibility and on the flate2 backend (`miniz_oxide` +/// here; a different backend changes the output bytes as well as the rate). +/// The level choice rests on ratios between levels, measured with the `gzip` +/// CLI (zlib) on the 318-byte items: level 1 was 1.6x faster than level 6 at +/// a 25% output ratio against 21%, and level 9 was 4x slower for 20%. +pub const GZIP_LEVEL: u32 = 6; + +const POISONED: &str = "encoder poisoned by an earlier write error; discard the file"; +const FINISHED: &str = "encoder already finished; no item may follow the trailer"; + +impl DataFileEncoder { + /// Start a data file written to `sink`. + #[must_use] + pub fn new(sink: W) -> Self { + Self { + encoder: GzEncoder::new( + HashingWriter { + inner: Some(sink), + hasher: CompressedHasher::new(), + }, + Compression::new(GZIP_LEVEL), + ), + line: Vec::new(), + item_count: 0, + uncompressed_bytes: 0, + state: EncoderState::Open, + } + } + + fn check_open(&self) -> Result<(), FormatError> { + match self.state { + EncoderState::Open => Ok(()), + EncoderState::Finished => Err(FormatError::Encode(FINISHED.to_owned())), + EncoderState::Poisoned => Err(FormatError::Encode(POISONED.to_owned())), + } + } + + /// Append one item as a line. + /// + /// # Errors + /// + /// Returns [`FormatError::Json`] when the item fails to serialize and + /// [`FormatError::LineTooLong`] when its line exceeds [`MAX_LINE_BYTES`] + /// (in both cases nothing has been written and the encoder stays usable), + /// and [`FormatError::Encode`] when the compressor or sink fails, after + /// which the encoder is poisoned, or when the encoder is already finished + /// or poisoned. + pub fn write_item(&mut self, item: &Item) -> Result<(), FormatError> { + self.check_open()?; + encode_item_line_into(item, &mut self.line)?; + let line = &self.line; + // The decoder refuses a line over the cap, so a file that contains + // one would verify on write and fail on restore. Refuse it here, with + // the encoder left usable (nothing has entered the compressor). + if line.len() > MAX_LINE_BYTES { + return Err(FormatError::LineTooLong { + cap: MAX_LINE_BYTES, + }); + } + if let Err(e) = self.encoder.write_all(line) { + self.state = EncoderState::Poisoned; + return Err(FormatError::Encode(e.to_string())); + } + self.item_count += 1; + self.uncompressed_bytes += line.len() as u64; + Ok(()) + } + + /// Whether a write or finish error has made this file unusable. False + /// after a successful [`Self::try_finish`]; see [`Self::is_finished`]. + #[must_use] + pub fn is_poisoned(&self) -> bool { + self.state == EncoderState::Poisoned + } + + /// Whether [`Self::try_finish`] has succeeded, so the file is complete + /// and accepts no more items. + #[must_use] + pub fn is_finished(&self) -> bool { + self.state == EncoderState::Finished + } + + /// Number of items written so far. + #[must_use] + pub fn item_count(&self) -> u64 { + self.item_count + } + + /// Uncompressed bytes written so far, newlines included. + /// + /// The rotation rule a writer with a per-file size target + /// (`data_file_target_bytes`) applies: after each [`Self::write_item`], + /// the file closes once this has reached the target. The line that + /// reaches it is in the file, so a file exceeds the target by less than + /// one line, at most [`MAX_LINE_BYTES`], and every file holds at least + /// one item. + #[must_use] + pub fn uncompressed_bytes(&self) -> u64 { + self.uncompressed_bytes + } + + /// Finish the gzip stream and flush the sink, keeping the encoder so the + /// sink can be recovered with [`Self::into_inner`] on failure (an upload + /// handle that must be aborted, for example). Returns the checksums a + /// manifest records. After a successful `try_finish` the encoder accepts + /// no more items. + /// + /// # Errors + /// + /// Returns [`FormatError::Encode`] when the compressor or sink fails, or + /// when the encoder is already finished or poisoned. The encoder is + /// poisoned after a failure. + pub fn try_finish(&mut self) -> Result { + self.check_open()?; + let result = self + .encoder + .try_finish() + .and_then(|()| self.encoder.get_mut().flush()); + if let Err(e) = result { + self.state = EncoderState::Poisoned; + return Err(FormatError::Encode(e.to_string())); + } + // No item may follow the trailer. + self.state = EncoderState::Finished; + let digest = std::mem::take(&mut self.encoder.get_mut().hasher).finish(); + Ok(DataFileChecksums { + item_count: self.item_count, + sha256_hex: digest.sha256_hex, + md5_base64: digest.md5_base64, + compressed_bytes: digest.bytes, + uncompressed_bytes: self.uncompressed_bytes, + }) + } + + /// Give back the sink, finished or not. After a failed `write_item` or + /// `try_finish` the sink holds a partial file the caller must discard. + /// + /// # Panics + /// + /// Never: the sink is present until this call consumes the encoder. + #[must_use] + pub fn into_inner(mut self) -> W { + self.encoder + .get_mut() + .inner + .take() + .expect("the sink is taken exactly once, here") + } + + /// [`Self::try_finish`] then [`Self::into_inner`], for callers whose sink + /// needs no cleanup on failure (a `Vec`, a file removed by path). + /// + /// # Errors + /// + /// Returns [`FormatError::Encode`] when the compressor or sink fails; the + /// sink is dropped without being written to again. + pub fn finish(mut self) -> Result<(W, DataFileChecksums), FormatError> { + let checksums = self.try_finish()?; + Ok((self.into_inner(), checksums)) + } +} + +impl Drop for DataFileEncoder { + /// An encoder that did not finish gives the compressor nothing to write + /// into, so its `Drop` cannot push buffered data and a trailer into a + /// sink the caller abandoned (type doc). A finished encoder has nothing + /// left to write. + fn drop(&mut self) { + if self.state != EncoderState::Finished { + drop(self.encoder.get_mut().inner.take()); + } + } +} + +/// Encode items into an in-memory gzip data file, returning the compressed +/// bytes and the checksums a manifest records for them. Convenience over +/// [`DataFileEncoder`] for callers that hold a whole file anyway (tests, small +/// files); a backup job streams through the encoder instead. +/// +/// # Errors +/// +/// Returns [`FormatError::Json`] when an item fails to serialize and +/// [`FormatError::Encode`] when compression fails. +pub fn encode_data_file(items: I) -> Result<(Vec, DataFileChecksums), FormatError> +where + I: IntoIterator, +{ + let mut encoder = DataFileEncoder::new(Vec::new()); + for item in items { + encoder.write_item(&item)?; + } + encoder.finish() +} + +/// Streaming decoder for a data file read from any [`Read`]. +/// +/// Items stream back one line at a time. The compressed bytes are hashed and +/// counted as they are consumed and the digests, item count and uncompressed +/// byte count are checked when the stream ends, so the last `next_item` call +/// returns the verification result. Until then every item is unverified: a +/// caller that needs the digests checked before it acts on any item uses +/// [`CompressedHasher`] over a local copy first and decodes the copy with +/// [`super::DataFileEntry::decoder_prehashed`], or [`Self::verified`] when +/// the file is already in memory. Two sizes are enforced while streaming, +/// so a file recorded as small cannot stream a long tail of items before +/// the digest refuses it: the compressed size (the source is refused once +/// it yields one byte more than recorded) and the uncompressed size (checked +/// after each line). +/// +/// Memory (the one statement of the decode-memory bound; the other docs and +/// tests point here): the line is read through a loop capped at +/// [`MAX_LINE_BYTES`], so the line buffer never exceeds the cap; one buffer +/// is reused for every line and keeps the longest line's capacity. Parsing the +/// line then materializes its item: the wire `AttributeValue` deserializer +/// builds a `serde_json::Value` of each top-level attribute's value and +/// converts it, consuming it level by level, so the peak is the `Value` plus +/// the item being built, whatever the nesting depth. Peak heap growth while +/// decoding one line, per line byte (requested bytes counted by a global +/// allocator, release build, `serde_json` 1.0.150 without `preserve_order`, +/// which is the default `extenddb` binary and `cargo test -p extenddb-core`; +/// a build that includes the MongoDB backend enables `preserve_order` +/// through `bson`, replaces `serde_json`'s `BTreeMap` with `indexmap`, and +/// measures about two thirds of each figure, 51 for the flat list, so the +/// table is the larger of the two; the `VmHWM` growth on Linux x86-64 with +/// glibc malloc is within 4% of it): +/// +/// | line | peak per line byte | +/// |---|---| +/// | list of empty strings (`{"S":""},`), flat | 76 | +/// | the same list inside 30 more `L` wrappers (32 levels, `DynamoDB`'s nesting limit) | 76 | +/// | the same list inside 60 more (past the limit, so only a forged line) | 76 | +/// | list of one-member maps with empty names chained 30 deep (`{"M":{"":{"M":...}}}`), the worst item found | 115 | +/// | objects with empty names nested to `serde_json`'s recursion limit (`{"":{"":...0}}`), not an item: refused, but only after the `Value` is built | 127 | +/// +/// So one line costs at most about 128 bytes per line byte, about 4 GiB at +/// the cap; a list of empty strings exactly at the cap raises `VmHWM` by +/// 2.44 GiB (the `data_file_decode_peak` probe). At the default item limit +/// the largest measured is the 409,593-empty-string item (a 3.69 MB line, +/// 280 MB); the longest line, the 27.7 MB number-set list, takes 92 MB. A +/// line near the cap comes from a raised `max_item_size_bytes` or a forged +/// file. The figures depend on the allocator and on the `serde_json` and +/// `AttributeValue` code, not on the format. The request path (`PutItem` +/// and every other operation that reads an item) uses the same +/// deserializer, so its cost no longer grows with nesting depth either. +/// +/// Items are checked for JSON shape only. An `N` value that is not a valid +/// `DynamoDB` number (`"abc"`, `"1E999"`, `"1_000"`) decodes as written. The +/// decoder checks the file against its format and its recorded sizes, +/// counts and digests, and [`super::ExtenddbManifest::reader`] adds the key +/// range check; neither applies the target's item limits (size, nesting +/// depth, number validity, key lengths), which the restore applies to each +/// item before writing it. +/// +/// Any error ends the stream: every later `next_item` returns +/// [`FormatError::Decode`] naming the earlier failure, never `Ok(None)`, so a +/// consumer that drops errors cannot mistake a corrupt file for a short one. +/// The [`Iterator`] impl is fused on top of that: it yields the first error +/// once and then `None`, so error-dropping iterator idioms terminate. +pub struct DataFileDecoder { + reader: BufReader>>, + expectations: DataFileExpectations, + items_decoded: u64, + bytes_decoded: u64, + finished: bool, + /// Set by the first error; replayed by every later call. + failed: Option<&'static str>, + /// The current line, cleared before each item and reused, as the + /// encoder reuses its buffer: it keeps the capacity of the longest line + /// read so far (at most [`MAX_LINE_BYTES`]) for the decoder's lifetime. + line: Vec, +} + +impl<'a> DataFileDecoder<&'a [u8]> { + /// Verify the compressed size and checksums against an in-memory file, + /// then open the stream over it. They are checked here, before any item + /// is returned; the end-of-stream checks still run. + /// + /// # Errors + /// + /// Returns [`FormatError::CompressedSizeMismatch`] or + /// [`FormatError::ChecksumMismatch`] when the size or a checksum in + /// `expectations` does not match `compressed`. + pub fn verified( + compressed: &'a [u8], + expectations: DataFileExpectations, + ) -> Result { + expectations.verify_digest(&CompressedHasher::digest(compressed))?; + Ok(Self::prehashed(compressed, expectations)) + } +} + +impl DataFileDecoder { + /// Open a stream over `compressed`. Nothing is verified until the stream + /// ends; see the type docs. The gzip header is read from the source here + /// (flate2 parses it in the constructor), so `new` may block on `R`; a + /// header error surfaces on the first [`Self::next_item`]. + pub fn new(compressed: R, expectations: DataFileExpectations) -> Self { + Self::open(compressed, expectations, Some(CompressedHasher::new())) + } + + /// Open a stream over `compressed` whose digests the caller has already + /// verified against `expectations` (with [`CompressedHasher`] and + /// [`DataFileExpectations::verify_digest`]): the bytes are not hashed + /// again and the digests are not compared, while the compressed size, + /// the uncompressed size, the item count and every line are checked as + /// [`Self::new`] checks them. [`super::DataFileEntry::decoder_prehashed`] + /// states when that is safe; [`Self::verified`] meets the condition by + /// hashing the slice it then decodes. + pub(super) fn prehashed(compressed: R, expectations: DataFileExpectations) -> Self { + Self::open(compressed, expectations, None) + } + + fn open( + compressed: R, + expectations: DataFileExpectations, + hasher: Option, + ) -> Self { + Self { + reader: BufReader::new(MultiGzDecoder::new(HashingReader { + inner: compressed, + hasher, + bytes: 0, + limit: expectations.compressed_bytes, + overrun: false, + })), + expectations, + items_decoded: 0, + bytes_decoded: 0, + finished: false, + failed: None, + line: Vec::new(), + } + } + + /// Decode the next item, or run the end-of-stream checks and return + /// `Ok(None)`. + /// + /// The line is read through a bounded loop that refuses to materialize + /// more than [`MAX_LINE_BYTES`], so a malicious or corrupt gzip member + /// cannot drive unbounded memory. When + /// [`DataFileExpectations::uncompressed_bytes`] is set it is also enforced + /// incrementally: the running total is checked against the expectation as + /// each line is consumed, so an oversized stream is rejected before the + /// whole of it is read rather than only at clean end of stream. + /// + /// # Errors + /// + /// Returns [`FormatError::Decode`] when decompression fails, + /// [`FormatError::LineTooLong`] when a single line reaches the cap, + /// [`FormatError::Json`] when a line does not parse, + /// [`FormatError::CompressedSizeMismatch`] as soon as the source yields + /// more bytes than recorded, and at end of stream + /// [`FormatError::CompressedSizeMismatch`], + /// [`FormatError::ChecksumMismatch`], [`FormatError::ItemCountMismatch`] + /// or [`FormatError::ByteCountMismatch`] when the consumed bytes differ + /// from the expectations (the uncompressed byte count is also enforced + /// mid-stream once it is exceeded). + pub fn next_item(&mut self) -> Result, FormatError> { + if let Some(reason) = self.failed { + return Err(FormatError::Decode(format!( + "decoder failed earlier: {reason}" + ))); + } + if self.finished { + return Ok(None); + } + let result = self.next_item_inner(); + if result.is_err() { + self.finished = true; + self.failed = Some(match &result { + Err(FormatError::ChecksumMismatch { .. }) => "checksum mismatch", + Err(FormatError::CompressedSizeMismatch { .. }) => "compressed size mismatch", + Err(FormatError::ItemCountMismatch { .. }) => "item count mismatch", + Err(FormatError::ByteCountMismatch { .. }) => "byte count mismatch", + Err(FormatError::LineTooLong { .. }) => "line too long", + Err(FormatError::Json(_)) => "unparseable line", + _ => "decode error", + }); + } + result + } + + /// Whether the compressed bytes are being hashed: false for a + /// [`Self::prehashed`] stream, and after the end-of-stream checks. + #[cfg(test)] + fn hashing(&self) -> bool { + self.reader.get_ref().get_ref().hasher.is_some() + } + + /// The error for a failed read: [`FormatError::CompressedSizeMismatch`] + /// when the source ran past its recorded size, otherwise + /// [`FormatError::Decode`]. + fn source_error(&self, e: &std::io::Error) -> FormatError { + let source = self.reader.get_ref().get_ref(); + match source.limit { + Some(expected) if source.overrun => FormatError::CompressedSizeMismatch { + expected, + actual: source.bytes, + }, + _ => FormatError::Decode(e.to_string()), + } + } + + fn next_item_inner(&mut self) -> Result, FormatError> { + // The buffer is taken out for the read, which borrows the reader, + // and put back whatever the read's outcome. + let mut line = std::mem::take(&mut self.line); + line.clear(); + let read = self.read_capped_line(&mut line); + self.line = line; + let read = read?; + if read == 0 { + self.finished = true; + return self.finish_checks().map(|()| None); + } + self.bytes_decoded += read as u64; + // Enforce the recorded uncompressed size incrementally: once the + // running total passes the expectation the stream is oversized, so + // reject it here rather than continuing to read. + if let Some(expected) = self.expectations.uncompressed_bytes + && self.bytes_decoded > expected + { + return Err(FormatError::ByteCountMismatch { + expected, + actual: self.bytes_decoded, + }); + } + // Errors in the line name it (1-based); positions in them are within + // the line, which is parsed on its own. + let line_number = self.items_decoded + 1; + let text = std::str::from_utf8(strip_trailing_newline(&self.line)).map_err(|e| { + FormatError::Decode(format!( + "data file line {line_number}: {e} (a byte index within the line)" + )) + })?; + let parsed: ItemLine = serde_json::from_str(text) + .map_err(|e| FormatError::Json(JsonError::on_data_file_line(e, line_number)))?; + self.items_decoded += 1; + Ok(Some(parsed.item)) + } + + /// End-of-stream verification: compressed size and digests (the size + /// alone for a [`Self::prehashed`] stream) first, then counts. By the time + /// the decompressor reports end of stream it has consumed every byte of + /// the source (`MultiGzDecoder` parses any trailing bytes as a further + /// member and fails on garbage), so the digest covers the whole object. + fn finish_checks(&mut self) -> Result<(), FormatError> { + let source = self.reader.get_mut().get_mut(); + match source.hasher.take() { + Some(hasher) => self.expectations.verify_digest(&hasher.finish())?, + None => self.expectations.verify_compressed_size(source.bytes)?, + } + if let Some(expected) = self.expectations.item_count + && expected != self.items_decoded + { + return Err(FormatError::ItemCountMismatch { + expected, + actual: self.items_decoded, + }); + } + if let Some(expected) = self.expectations.uncompressed_bytes + && expected != self.bytes_decoded + { + return Err(FormatError::ByteCountMismatch { + expected, + actual: self.bytes_decoded, + }); + } + Ok(()) + } + + /// Read one line into `line`, through its newline or, for a last line + /// without one, to the end of the stream (module docs), appending at + /// most [`MAX_LINE_BYTES`] bytes (newline included) before erroring. + /// + /// Returns the number of bytes read, `0` at end of stream. The read walks + /// the `BufReader`'s own buffer with `fill_buf` / `consume`, so reading + /// holds the growing line (capped) plus the reader's fixed buffers, never + /// the whole compressed member expanded at once. This bounds the read + /// only: parsing the line afterwards materializes its item, which can + /// take many times the line's size (type docs). + fn read_capped_line(&mut self, line: &mut Vec) -> Result { + loop { + let available = match self.reader.fill_buf() { + Ok(available) => available, + Err(e) => return Err(self.source_error(&e)), + }; + if available.is_empty() { + return Ok(line.len()); + } + if let Some(nl) = available.iter().position(|&b| b == b'\n') { + let want = nl + 1; + if line.len() + want > MAX_LINE_BYTES { + return Err(FormatError::LineTooLong { + cap: MAX_LINE_BYTES, + }); + } + line.extend_from_slice(&available[..want]); + self.reader.consume(want); + return Ok(line.len()); + } + let take = available.len(); + if line.len() + take > MAX_LINE_BYTES { + return Err(FormatError::LineTooLong { + cap: MAX_LINE_BYTES, + }); + } + line.extend_from_slice(available); + self.reader.consume(take); + } + } +} + +impl Iterator for DataFileDecoder { + type Item = Result; + + /// Fused: once the decoder has failed, `None`. The first error is + /// yielded once by whichever of `next` or [`Self::next_item`] meets it; + /// `next_item` keeps returning it for callers that use it directly, so a + /// caller mixing the two can see it from `next_item` and then `None` + /// from `next`. + fn next(&mut self) -> Option { + if self.failed.is_some() { + return None; + } + self.next_item().transpose() + } +} + +impl std::iter::FusedIterator for DataFileDecoder {} + +/// Decode a whole in-memory data file into a `Vec`, verifying the size and +/// digests before the first item and the counts at the end. For tests and files known +/// to be small; a restore streams through [`DataFileDecoder`] instead and +/// never holds a file's items in memory. +/// +/// # Errors +/// +/// Propagates every error [`DataFileDecoder`] can produce. +pub fn decode_data_file( + compressed: &[u8], + expectations: DataFileExpectations, +) -> Result, FormatError> { + let mut decoder = DataFileDecoder::verified(compressed, expectations)?; + let mut items = Vec::new(); + while let Some(item) = decoder.next_item()? { + items.push(item); + } + Ok(items) +} + +#[cfg(test)] +mod tests { + use std::collections::{BTreeMap, BTreeSet}; + + use super::*; + use crate::types::AttributeValue; + + fn item(entries: Vec<(&str, AttributeValue)>) -> Item { + entries + .into_iter() + .map(|(k, v)| (k.to_owned(), v)) + .collect() + } + + fn string_set(values: &[&str]) -> BTreeSet { + values.iter().map(|s| (*s).to_owned()).collect() + } + + fn byte_set(values: &[&[u8]]) -> BTreeSet> { + values.iter().map(|b| b.to_vec()).collect() + } + + /// A map nested to the given depth, ending in a string leaf. + fn nested_map(depth: usize) -> AttributeValue { + let mut value = AttributeValue::S("leaf".to_owned()); + for level in (0..depth).rev() { + let mut map = BTreeMap::new(); + map.insert(format!("level{level}"), value); + value = AttributeValue::M(map); + } + value + } + + /// Items covering every `DynamoDB` attribute type, with numbers at the + /// 38-significant-digit maximum and in negative form. Number strings are + /// written in the normalized form the `N` deserializer produces, so a + /// round trip compares equal. + fn all_types_items() -> Vec { + vec![ + item(vec![ + ("pk", AttributeValue::S("item-1".to_owned())), + ( + "n_38_digits", + AttributeValue::N("99999999999999999999999999999999999999".to_owned()), + ), + ( + "n_negative", + AttributeValue::N("-12345678901234567890.123456789".to_owned()), + ), + ( + "n_large_magnitude", + AttributeValue::N(format!("1{}", "0".repeat(100))), + ), + ("n_small", AttributeValue::N("0.001".to_owned())), + ("b", AttributeValue::B(vec![0xde, 0xad, 0xbe, 0xef])), + ("bool_true", AttributeValue::Bool(true)), + ("null", AttributeValue::Null), + ]), + item(vec![ + ("pk", AttributeValue::S("item-2".to_owned())), + ("ss", AttributeValue::SS(string_set(&["a", "b", "c"]))), + ( + "ns", + AttributeValue::NS( + ["-1", "0.5", "42"] + .iter() + .map(|s| (*s).to_owned()) + .collect(), + ), + ), + ( + "bs", + AttributeValue::BS(byte_set(&[b"one".as_slice(), b"two".as_slice()])), + ), + ( + "l", + AttributeValue::L(vec![ + AttributeValue::N("1".to_owned()), + AttributeValue::S("x".to_owned()), + AttributeValue::Bool(false), + AttributeValue::Null, + AttributeValue::L(vec![]), + ]), + ), + ("m_depth_32", nested_map(32)), + ]), + ] + } + + #[test] + fn encode_decode_round_trip_all_types() { + let items = all_types_items(); + let (compressed, checksums) = encode_data_file(items.clone()).unwrap(); + assert_eq!(checksums.item_count, 2); + assert_eq!(checksums.compressed_bytes, compressed.len() as u64); + assert!(checksums.uncompressed_bytes > checksums.compressed_bytes / 4); + + let decoded = decode_data_file(&compressed, (&checksums).into()).unwrap(); + assert_eq!(decoded, items); + } + + #[test] + fn exponent_form_numbers_decode_to_normalized_values() { + // Exponent and shorthand forms are valid on the wire; the N + // deserializer normalizes them, so a file holding them decodes to the + // plain forms. + let line = concat!( + r#"{"Item":{"e_neg":{"N":"-2.5e-2"},"e_pos":{"N":"1.5E3"},"#, + r#""pk":{"S":"exp"}}}"#, + "\n", + ); + let mut encoder = GzEncoder::new(Vec::new(), Compression::default()); + encoder.write_all(line.as_bytes()).unwrap(); + let compressed = encoder.finish().unwrap(); + + let decoded = decode_data_file(&compressed, DataFileExpectations::default()).unwrap(); + assert_eq!(decoded.len(), 1); + assert_eq!( + decoded[0].get("e_pos"), + Some(&AttributeValue::N("1500".to_owned())) + ); + assert_eq!( + decoded[0].get("e_neg"), + Some(&AttributeValue::N("-0.025".to_owned())) + ); + } + + #[test] + fn line_encoding_matches_export_handler_shape() { + // One item, then decompress and compare against the exact string the + // engine's export handler would write for it. + let one = item(vec![ + ("a", AttributeValue::N("1".to_owned())), + ("pk", AttributeValue::S("x".to_owned())), + ]); + let (compressed, _) = encode_data_file(vec![one.clone()]).unwrap(); + + let mut reader = BufReader::new(MultiGzDecoder::new(compressed.as_slice())); + let mut text = String::new(); + std::io::Read::read_to_string(&mut reader, &mut text).unwrap(); + + let wrapper = serde_json::json!({ "Item": one }); + let expected = format!("{}\n", serde_json::to_string(&wrapper).unwrap()); + assert_eq!(text, expected); + assert_eq!( + text, + "{\"Item\":{\"a\":{\"N\":\"1\"},\"pk\":{\"S\":\"x\"}}}\n" + ); + } + + #[test] + fn empty_file_round_trip() { + let (compressed, checksums) = encode_data_file(Vec::::new()).unwrap(); + assert_eq!(checksums.item_count, 0); + assert_eq!(checksums.uncompressed_bytes, 0); + let decoded = decode_data_file(&compressed, (&checksums).into()).unwrap(); + assert!(decoded.is_empty()); + } + + #[test] + fn tampered_compressed_bytes_fail_sha256() { + let (mut compressed, checksums) = encode_data_file(all_types_items()).unwrap(); + let last = compressed.len() - 1; + compressed[last] ^= 0xff; + let err = decode_data_file(&compressed, (&checksums).into()).unwrap_err(); + assert!(matches!( + err, + FormatError::ChecksumMismatch { + subject: "data file sha256", + .. + } + )); + } + + #[test] + fn tampered_compressed_bytes_fail_md5_when_only_md5_given() { + let (mut compressed, checksums) = encode_data_file(all_types_items()).unwrap(); + compressed[0] ^= 0x01; + let expectations = DataFileExpectations { + md5_base64: Some(checksums.md5_base64), + ..Default::default() + }; + let err = decode_data_file(&compressed, expectations).unwrap_err(); + assert!(matches!( + err, + FormatError::ChecksumMismatch { + subject: "data file md5", + .. + } + )); + } + + /// Fewer items than recorded and more items than recorded are both + /// refused at end of stream. + #[test] + fn wrong_item_count_is_detected() { + let (compressed, checksums) = encode_data_file(all_types_items()).unwrap(); + assert_eq!(checksums.item_count, 2); + for (recorded, expected_err) in [(3, (3, 2)), (1, (1, 2)), (0, (0, 2))] { + let expectations = DataFileExpectations { + item_count: Some(recorded), + ..Default::default() + }; + let err = decode_data_file(&compressed, expectations).unwrap_err(); + assert!( + matches!( + err, + FormatError::ItemCountMismatch { expected, actual } + if (expected, actual) == expected_err + ), + "recorded {recorded}: {err}" + ); + } + } + + #[test] + fn wrong_uncompressed_byte_count_is_detected() { + let (compressed, checksums) = encode_data_file(all_types_items()).unwrap(); + let expectations = DataFileExpectations { + uncompressed_bytes: Some(checksums.uncompressed_bytes + 1), + ..Default::default() + }; + let err = decode_data_file(&compressed, expectations).unwrap_err(); + assert!(matches!(err, FormatError::ByteCountMismatch { .. })); + } + + #[test] + fn decoder_streams_via_iterator() { + let items = all_types_items(); + let (compressed, checksums) = encode_data_file(items.clone()).unwrap(); + let decoder = DataFileDecoder::verified(&compressed, (&checksums).into()).unwrap(); + let streamed: Result, FormatError> = decoder.collect(); + assert_eq!(streamed.unwrap(), items); + } + + #[test] + fn truncated_gzip_stream_is_an_error() { + let (compressed, _) = encode_data_file(all_types_items()).unwrap(); + let truncated = &compressed[..compressed.len() / 2]; + let err = decode_data_file(truncated, DataFileExpectations::default()).unwrap_err(); + assert!(matches!(err, FormatError::Decode(_))); + } + + #[test] + fn malformed_line_is_an_error() { + let mut encoder = GzEncoder::new(Vec::new(), Compression::default()); + encoder.write_all(b"{\"NotItem\":{}}\n").unwrap(); + let compressed = encoder.finish().unwrap(); + let err = decode_data_file(&compressed, DataFileExpectations::default()).unwrap_err(); + assert!(matches!(err, FormatError::Json(_))); + } + + #[test] + fn md5_checksum_encoding_matches_manifest_files_shape() { + // 16 MD5 bytes encode to 24 base64 characters with padding, the + // shape manifest-files.json carries. + let (_, checksums) = encode_data_file(all_types_items()).unwrap(); + assert_eq!(checksums.md5_base64.len(), 24); + assert!(checksums.md5_base64.ends_with("==")); + assert_eq!(checksums.sha256_hex.len(), 64); + } + + // The line-bomb test reads the process-wide `VmHWM`, so it lives in its + // own integration test binary (`tests/data_file_line_bomb.rs`) where no + // other test can raise the high-water mark first. + + #[test] + fn oversized_stream_is_rejected_before_full_read_when_byte_count_is_known() { + // Many small lines whose running total will exceed a deliberately + // low expectation; the decoder must stop mid-stream, not at EOF. + let items: Vec = (0..1000) + .map(|i| item(vec![("pk", AttributeValue::S(format!("item-{i}")))])) + .collect(); + let (compressed, _) = encode_data_file(items).unwrap(); + let expectations = DataFileExpectations { + uncompressed_bytes: Some(64), + ..Default::default() + }; + let mut decoder = DataFileDecoder::verified(&compressed, expectations).unwrap(); + let mut err = None; + let mut yielded = 0; + // Pull items until the incremental byte-count check fires. + loop { + match decoder.next_item() { + Ok(Some(_)) => yielded += 1, + Ok(None) => break, + Err(e) => { + err = Some(e); + break; + } + } + } + assert!( + matches!( + err, + Some(FormatError::ByteCountMismatch { expected: 64, .. }) + ), + "expected a mid-stream ByteCountMismatch, got {err:?}" + ); + // The end-of-stream check reports the same variant from the same + // input, so the count is what shows the check ran mid-stream: each + // line is about 30 bytes, so the third line passes 64. + assert!(yielded < 10, "{yielded} of 1000 items streamed first"); + } + + #[test] + fn multi_member_gzip_decodes_all_members() { + // Two independent gzip members concatenated: MultiGzDecoder must read + // both, GzDecoder would stop after the first. + let first = all_types_items(); + let second = vec![ + item(vec![("pk", AttributeValue::S("m2-a".to_owned()))]), + item(vec![("pk", AttributeValue::S("m2-b".to_owned()))]), + ]; + let (mut member_one, _) = encode_data_file(first.clone()).unwrap(); + let (member_two, _) = encode_data_file(second.clone()).unwrap(); + member_one.extend_from_slice(&member_two); + + let decoded = decode_data_file(&member_one, DataFileExpectations::default()).unwrap(); + let mut expected = first; + expected.extend(second); + assert_eq!(decoded, expected); + } + + /// The cap is exclusive: a line of exactly `MAX_LINE_BYTES` (newline + /// included) is read and then fails to parse as an item; one byte more is + /// `LineTooLong` before the line is materialized. + #[test] + fn line_cap_boundary() { + // A line of `len` bytes, the last a newline or (an unterminated final + // line) not. + let build = |len: usize, newline: bool| { + let mut encoder = GzEncoder::new(Vec::new(), Compression::default()); + let mut line = vec![b'x'; len]; + if newline { + line[len - 1] = b'\n'; + } + encoder.write_all(&line).unwrap(); + encoder.finish().unwrap() + }; + for newline in [true, false] { + let at_cap = build(MAX_LINE_BYTES, newline); + let err = decode_data_file(&at_cap, DataFileExpectations::default()).unwrap_err(); + assert!( + matches!(err, FormatError::Json(_)), + "newline {newline}: at the cap the line is read: {err}" + ); + let over = build(MAX_LINE_BYTES + 1, newline); + let err = decode_data_file(&over, DataFileExpectations::default()).unwrap_err(); + assert!( + matches!(err, FormatError::LineTooLong { .. }), + "newline {newline}: {err}" + ); + } + } + + /// A sink that only exposes `Write`, like a file, and hands back at most + /// 7 bytes per call so the hashing writer's short-write handling is + /// exercised. + struct CountingSink(Vec, usize); + impl Write for CountingSink { + fn write(&mut self, buf: &[u8]) -> std::io::Result { + let n = buf.len().min(7); + self.0.extend_from_slice(&buf[..n]); + self.1 += 1; + Ok(n) + } + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } + } + + #[test] + fn streaming_encoder_matches_in_memory_encoder() { + let items = all_types_items(); + let (expected_bytes, expected) = encode_data_file(items.clone()).unwrap(); + let mut encoder = DataFileEncoder::new(CountingSink(Vec::new(), 0)); + for item in &items { + encoder.write_item(item).unwrap(); + } + assert_eq!(encoder.item_count(), items.len() as u64); + assert_eq!(encoder.uncompressed_bytes(), expected.uncompressed_bytes); + let (sink, checksums) = encoder.finish().unwrap(); + assert_eq!(sink.0, expected_bytes); + assert_eq!(checksums, expected); + assert!(sink.1 > 1, "short writes were exercised"); + } + + #[test] + fn encoder_output_is_deterministic() { + let (a, ca) = encode_data_file(all_types_items()).unwrap(); + let (b, cb) = encode_data_file(all_types_items()).unwrap(); + assert_eq!(a, b); + assert_eq!(ca, cb); + } + + #[test] + fn compressed_hasher_incremental_equals_one_shot() { + let (compressed, checksums) = encode_data_file(all_types_items()).unwrap(); + let mut hasher = CompressedHasher::new(); + for chunk in compressed.chunks(13) { + hasher.update(chunk); + } + let digest = hasher.finish(); + assert_eq!(digest, CompressedHasher::digest(&compressed)); + assert_eq!(digest.sha256_hex, checksums.sha256_hex); + assert_eq!(digest.md5_base64, checksums.md5_base64); + assert_eq!(digest.bytes, checksums.compressed_bytes); + DataFileExpectations::from(&checksums) + .verify_digest(&digest) + .unwrap(); + } + + #[test] + fn streaming_decoder_verifies_at_end_of_stream() { + let (mut compressed, checksums) = encode_data_file(all_types_items()).unwrap(); + // Flip a byte inside the gzip payload (past the 10-byte header, before + // the 8-byte trailer): the stream still decompresses something, but + // the digest over the stored bytes changes. + let n = compressed.len(); + compressed[n / 2] ^= 0x01; + let mut decoder = DataFileDecoder::new( + std::io::Cursor::new(compressed), + DataFileExpectations::from(&checksums), + ); + // Items are unverified until the end: the decoder either yields + // items, fails to decompress, or fails the digest check at EOF. It + // must not report clean completion. + let mut saw_error = false; + loop { + match decoder.next_item() { + Ok(Some(_)) => {} + Ok(None) => break, + Err(_) => { + saw_error = true; + break; + } + } + } + assert!(saw_error, "a tampered stream must not complete cleanly"); + } + + /// Bytes after the gzip member are parsed as a further member by + /// `MultiGzDecoder`, so garbage is a decode error (not a digest mismatch: + /// the digest check never runs because the stream did not end cleanly). + /// With the compressed size known, the extra bytes are refused as a size + /// mismatch before that (the file is longer than recorded); this test + /// leaves the size out to reach the decompressor's own check. + #[test] + fn trailing_bytes_after_gzip_member_are_a_decode_error() { + for garbage in [ + &b"x"[..], + &b"garbage after the gzip member"[..], + &[0u8; 8][..], + ] { + let (mut compressed, checksums) = encode_data_file(all_types_items()).unwrap(); + compressed.extend_from_slice(garbage); + let recorded = checksums.compressed_bytes; + let err = DataFileDecoder::new( + std::io::Cursor::new(compressed.clone()), + DataFileExpectations::from(&checksums), + ) + .collect::, _>>() + .unwrap_err(); + assert!( + matches!( + err, + FormatError::CompressedSizeMismatch { expected, actual } + if expected == recorded && actual == recorded + 1 + ), + "{err}" + ); + let mut decoder = DataFileDecoder::new( + std::io::Cursor::new(compressed), + DataFileExpectations { + compressed_bytes: None, + ..DataFileExpectations::from(&checksums) + }, + ); + let mut n = 0; + let err = loop { + match decoder.next_item() { + Ok(Some(_)) => n += 1, + Ok(None) => panic!("trailing bytes must not decode cleanly"), + Err(e) => break e, + } + }; + assert!(matches!(err, FormatError::Decode(_)), "{err}"); + assert_eq!(n, all_types_items().len(), "the valid member decoded first"); + // The error is sticky. + assert!(matches!(decoder.next_item(), Err(FormatError::Decode(_)))); + } + } + + #[test] + fn decoder_errors_are_sticky() { + let (mut compressed, checksums) = encode_data_file(all_types_items()).unwrap(); + let n = compressed.len(); + compressed[n / 2] ^= 0x01; + let mut decoder = DataFileDecoder::new( + std::io::Cursor::new(compressed), + DataFileExpectations::from(&checksums), + ); + while let Ok(Some(_)) = decoder.next_item() {} + for _ in 0..3 { + let err = decoder + .next_item() + .expect_err("an error must not turn into Ok(None)"); + assert!(matches!(err, FormatError::Decode(_)), "{err}"); + assert!(err.to_string().contains("failed earlier"), "{err}"); + } + } + + /// The iterator yields the first error once and then ends, so a consumer + /// that drops errors terminates instead of spinning on the sticky error; + /// a consumer that collects into a `Result` still gets the error. + #[test] + fn iterator_is_fused_after_an_error() { + let tampered = || { + let (mut compressed, checksums) = encode_data_file(all_types_items()).unwrap(); + let n = compressed.len(); + compressed[n - 1] ^= 0x01; + DataFileDecoder::new( + std::io::Cursor::new(compressed), + DataFileExpectations::from(&checksums), + ) + }; + let (tx, rx) = std::sync::mpsc::channel(); + std::thread::spawn(move || { + let count = tampered().filter_map(Result::ok).count(); + let _ = tx.send(count); + }); + let count = rx + .recv_timeout(std::time::Duration::from_secs(10)) + .expect("an error-dropping consumer must terminate"); + assert!(count <= all_types_items().len()); + + let collected: Result, FormatError> = tampered().collect(); + assert!(collected.is_err(), "collect still surfaces the error"); + + let mut decoder = tampered(); + let mut errors = 0; + for r in decoder.by_ref() { + if r.is_err() { + errors += 1; + } + } + assert_eq!(errors, 1, "the error is yielded exactly once"); + assert!(decoder.next().is_none()); + assert!(decoder.next_item().is_err(), "next_item stays sticky"); + } + + /// Flaky sink: fails the `fail_on`th write, counts every write attempt. + struct Flaky { + buf: Vec, + calls: usize, + fail_on: usize, + } + impl Write for Flaky { + fn write(&mut self, b: &[u8]) -> std::io::Result { + self.calls += 1; + if self.calls == self.fail_on { + return Err(std::io::Error::other("boom")); + } + self.buf.extend_from_slice(b); + Ok(b.len()) + } + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } + } + + /// Incompressible payloads so the compressor flushes to the sink during + /// `write_item` rather than only at finish. + fn incompressible_items() -> Vec { + (0..4u64) + .map(|i| { + let mut x: u64 = 0x9e37_79b9_7f4a_7c15 ^ i; + let blob: String = (0..300_000) + .map(|_| { + x ^= x << 13; + x ^= x >> 7; + x ^= x << 17; + char::from(b'a' + u8::try_from(x % 26).unwrap()) + }) + .collect(); + item(vec![("pk", AttributeValue::S(blob))]) + }) + .collect() + } + + #[test] + fn encoder_is_poisoned_after_a_sink_error() { + let big = incompressible_items(); + let mut encoder = DataFileEncoder::new(Flaky { + buf: Vec::new(), + calls: 0, + fail_on: 2, + }); + let mut first_err = None; + for it in &big { + if let Err(e) = encoder.write_item(it) { + first_err = Some(e); + break; + } + } + assert!( + matches!(first_err, Some(FormatError::Encode(_))), + "{first_err:?}" + ); + assert!(encoder.is_poisoned()); + assert!(!encoder.is_finished()); + // A retry of the item and a finish are both refused. + let err = encoder.write_item(&big[0]).unwrap_err(); + assert!(err.to_string().contains("poisoned"), "{err}"); + let err = encoder.try_finish().unwrap_err(); + assert!(err.to_string().contains("poisoned"), "{err}"); + // The sink can still be recovered for cleanup. + let sink = encoder.into_inner(); + assert!(sink.calls >= 2); + } + + /// Dropping a poisoned encoder must not push the compressor's buffered + /// data and trailer into the sink: the caller has decided to abandon the + /// file, and for an upload sink that would be tens of kilobytes of + /// multipart data after the abort decision. + #[test] + fn dropping_a_poisoned_encoder_writes_nothing_more() { + use std::sync::{Arc, Mutex}; + struct Shared(Arc>); + impl Write for Shared { + fn write(&mut self, b: &[u8]) -> std::io::Result { + self.0.lock().unwrap().write(b) + } + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } + } + let sink = Arc::new(Mutex::new(Flaky { + buf: Vec::new(), + calls: 0, + fail_on: 2, + })); + let mut encoder = DataFileEncoder::new(Shared(Arc::clone(&sink))); + for it in &incompressible_items() { + if encoder.write_item(it).is_err() { + break; + } + } + assert!(encoder.is_poisoned()); + let calls_before = sink.lock().unwrap().calls; + drop(encoder); + assert_eq!( + sink.lock().unwrap().calls, + calls_before, + "drop of a poisoned encoder wrote into the sink" + ); + } + + /// The encoder and the decoder share the line cap: a line of exactly the + /// cap round-trips, one byte more is refused at write time with the + /// encoder still usable. + #[test] + fn encoder_refuses_a_line_over_the_cap() { + let overhead = encode_item_line(&item(vec![("pk", AttributeValue::S(String::new()))])) + .unwrap() + .len(); + let exactly = item(vec![( + "pk", + AttributeValue::S("x".repeat(MAX_LINE_BYTES - overhead)), + )]); + let over = item(vec![( + "pk", + AttributeValue::S("x".repeat(MAX_LINE_BYTES - overhead + 1)), + )]); + assert_eq!(encode_item_line(&exactly).unwrap().len(), MAX_LINE_BYTES); + let mut encoder = DataFileEncoder::new(Vec::new()); + let err = encoder.write_item(&over).unwrap_err(); + assert!(matches!(err, FormatError::LineTooLong { .. }), "{err}"); + assert!(!encoder.is_poisoned()); + encoder.write_item(&exactly).unwrap(); + let (bytes, checksums) = encoder.finish().unwrap(); + assert_eq!(checksums.item_count, 1); + let decoded = decode_data_file(&bytes, DataFileExpectations::from(&checksums)).unwrap(); + assert_eq!(decoded, vec![exactly]); + } + + /// The streaming decoder checks the digest at end of stream: a file whose + /// items are identical but whose compressed bytes differ (another gzip + /// level) yields every item and then `ChecksumMismatch`, never `Ok(None)`. + #[test] + fn streaming_decoder_digest_check_fires_at_end_of_stream() { + let items = all_types_items(); + let (_, level6) = encode_data_file(items.clone()).unwrap(); + let mut sink = Vec::new(); + { + let mut gz = GzEncoder::new(&mut sink, Compression::new(1)); + for it in &items { + gz.write_all(&encode_item_line(it).unwrap()).unwrap(); + } + gz.finish().unwrap(); + } + assert_ne!(sink.len() as u64, level6.compressed_bytes, "levels differ"); + let mut decoder = DataFileDecoder::new( + std::io::Cursor::new(sink), + DataFileExpectations { + sha256_hex: Some(level6.sha256_hex.clone()), + md5_base64: None, + compressed_bytes: None, + item_count: Some(level6.item_count), + uncompressed_bytes: Some(level6.uncompressed_bytes), + }, + ); + let mut n = 0; + let err = loop { + match decoder.next_item() { + Ok(Some(_)) => n += 1, + Ok(None) => panic!("a digest mismatch must not complete cleanly"), + Err(e) => break e, + } + }; + assert_eq!(n, items.len(), "every item streams before the digest check"); + assert!( + matches!( + err, + FormatError::ChecksumMismatch { + subject: "data file sha256", + .. + } + ), + "{err}" + ); + } + + /// `DataFileEntry::decoder` is the restore's entry point: it validates the + /// key and carries every expectation from the manifest entry. + #[test] + fn data_file_entry_decoder_carries_every_expectation() { + use super::super::{DataFileEntry, data_file_key}; + let items = all_types_items(); + let (bytes, c) = encode_data_file(items.clone()).unwrap(); + let entry = DataFileEntry { + key: data_file_key(1), + item_count: c.item_count, + sha256: c.sha256_hex.clone(), + md5: c.md5_base64.clone(), + bytes: c.compressed_bytes, + uncompressed_bytes: c.uncompressed_bytes, + key_range: None, + }; + let decoded: Vec = entry + .decoder(std::io::Cursor::new(bytes.clone())) + .unwrap() + .collect::>() + .unwrap(); + assert_eq!(decoded, items); + + let run = |e: &DataFileEntry| -> FormatError { + e.decoder(std::io::Cursor::new(bytes.clone())) + .unwrap() + .collect::, _>>() + .unwrap_err() + }; + let mut e = entry.clone(); + e.sha256 = "0".repeat(64); + assert!(matches!( + run(&e), + FormatError::ChecksumMismatch { + subject: "data file sha256", + .. + } + )); + let mut e = entry.clone(); + e.md5 = "AAAAAAAAAAAAAAAAAAAAAA==".to_owned(); + assert!(matches!( + run(&e), + FormatError::ChecksumMismatch { + subject: "data file md5", + .. + } + )); + for bytes in [c.compressed_bytes - 1, c.compressed_bytes + 1] { + let mut e = entry.clone(); + e.bytes = bytes; + assert!( + matches!( + run(&e), + FormatError::CompressedSizeMismatch { expected, .. } if expected == bytes + ), + "{bytes}" + ); + } + let mut e = entry.clone(); + e.item_count += 1; + assert!(matches!(run(&e), FormatError::ItemCountMismatch { .. })); + let mut e = entry.clone(); + e.uncompressed_bytes -= 1; + assert!(matches!(run(&e), FormatError::ByteCountMismatch { .. })); + let mut e = entry; + e.key = "data/../x.json.gz".to_owned(); + assert!(matches!( + e.decoder(std::io::Cursor::new(bytes)).err(), + Some(FormatError::InvalidDataFileKey { .. }) + )); + } + + /// The byte count is enforced while streaming, not only at the end: an + /// oversized stream fails after a few items, not after all of them. + #[test] + fn oversized_stream_fails_before_it_is_fully_read() { + let items: Vec = (0..1000u32) + .map(|i| item(vec![("pk", AttributeValue::N(i.to_string()))])) + .collect(); + let (bytes, c) = encode_data_file(items).unwrap(); + let mut decoder = DataFileDecoder::new( + std::io::Cursor::new(bytes), + DataFileExpectations { + sha256_hex: None, + md5_base64: None, + compressed_bytes: None, + item_count: None, + uncompressed_bytes: Some(64), + }, + ); + let mut yielded = 0; + let err = loop { + match decoder.next_item() { + Ok(Some(_)) => yielded += 1, + Ok(None) => panic!("must not complete"), + Err(e) => break e, + } + }; + assert!( + matches!(err, FormatError::ByteCountMismatch { expected: 64, .. }), + "{err}" + ); + assert!( + yielded < 10, + "{yielded} items streamed before the cap fired" + ); + assert_eq!(c.item_count, 1000); + } + + /// A failure in `try_finish` poisons the encoder like a failed write. + #[test] + fn try_finish_failure_poisons_the_encoder() { + // Fail the last write: the gzip trailer. Count writes on a plain + // encoding first to know which call that is. + let probe = { + let mut e = DataFileEncoder::new(Flaky { + buf: Vec::new(), + calls: 0, + fail_on: usize::MAX, + }); + e.write_item(&all_types_items()[0]).unwrap(); + e.try_finish().unwrap(); + e.into_inner().calls + }; + let mut encoder = DataFileEncoder::new(Flaky { + buf: Vec::new(), + calls: 0, + fail_on: probe, + }); + encoder.write_item(&all_types_items()[0]).unwrap(); + let err = encoder.try_finish().unwrap_err(); + assert!(matches!(err, FormatError::Encode(_)), "{err}"); + assert!(encoder.is_poisoned()); + assert!(!encoder.is_finished()); + assert!(encoder.try_finish().is_err()); + let _ = encoder.into_inner(); + } + + /// `Interrupted` from the sink is retried on every path, so where it + /// lands does not decide whether a file must be re-encoded. + /// The worst expansion the cap's doc names (a list of number sets, 90 + /// two-digit values at `E-130` in each, 2 bytes of item size and 136 + /// bytes of line per member) at the default item limit fits the cap, so + /// the constant and the default limit cannot drift apart silently. The + /// next shape in the doc's table, the list of single numbers, is checked + /// to expand less; the other rows are not checked here. + #[test] + fn largest_expansion_at_the_default_item_limit_fits_the_cap() { + use crate::types::item_size_bytes; + use crate::validation::number::validate_and_normalize_number; + let limit = crate::limits::LimitsConfig::default().max_item_size_bytes; + let set: BTreeSet = (10..100) + .map(|m| validate_and_normalize_number(&format!("{}.{}E-130", m / 10, m % 10)).unwrap()) + .collect(); + assert_eq!(set.len(), 90); + assert!( + set.iter().all(|v| v.len() >= 132), + "the longest two-digit forms" + ); + // As many sets as fit: "pk" + "p" + "l" + the list's 3 bytes, then + // each set costs its members plus 1 as a list element. + let element = crate::types::attribute_value_size(&AttributeValue::NS(set.clone())) + 1; + let fits = (limit - 7) / element; + let build = |n: usize, value: &AttributeValue| { + let mut it = item(vec![("pk", AttributeValue::S("p".into()))]); + it.insert("l".to_owned(), AttributeValue::L(vec![value.clone(); n])); + it + }; + let worst = build(fits, &AttributeValue::NS(set)); + let size = item_size_bytes(&worst); + assert!( + size <= limit && size + element > limit, + "{size} fills the limit" + ); + let line = encode_item_line(&worst).unwrap().len(); + assert!( + line <= MAX_LINE_BYTES, + "{line} bytes > cap {MAX_LINE_BYTES}" + ); + // The doc's figures: 409,429 item bytes, 27,686,918 line bytes. + assert_eq!((size, line), (409_429, 27_686_918)); + // The list of single numbers, the next shape in the table, expands + // less. + let n = validate_and_normalize_number("1.1E-130").unwrap(); + let numbers = build((limit - 7) / 3, &AttributeValue::N(n)); + assert!(item_size_bytes(&numbers) <= limit); + let numbers_line = encode_item_line(&numbers).unwrap().len(); + assert!(numbers_line < line, "{numbers_line}"); + } + + /// Dropping an encoder that did not finish writes nothing more, whether + /// it is fresh or has items in its buffer: an abandoned encoder must not + /// complete a valid gzip file in the sink. + #[test] + fn dropping_an_unfinished_encoder_writes_nothing_more() { + use std::sync::{Arc, Mutex}; + struct Counting(Arc)>>); + impl Write for Counting { + fn write(&mut self, b: &[u8]) -> std::io::Result { + let mut g = self.0.lock().unwrap(); + g.0 += 1; + g.1.extend_from_slice(b); + Ok(b.len()) + } + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } + } + for items in [0usize, 50] { + let sink = Arc::new(Mutex::new((0, Vec::new()))); + let mut encoder = DataFileEncoder::new(Counting(Arc::clone(&sink))); + for it in all_types_items().into_iter().cycle().take(items) { + encoder.write_item(&it).unwrap(); + } + let before = sink.lock().unwrap().0; + drop(encoder); + let (after, bytes) = sink.lock().unwrap().clone(); + assert_eq!(after, before, "{items} items: drop wrote into the sink"); + assert!( + decode_data_file(&bytes, DataFileExpectations::default()).is_err(), + "{items} items: the sink must not hold a complete file" + ); + } + } + + /// `Interrupted` is retried at the last sink write (the trailer, which + /// std's `write_all` does not cover) and on `flush`; the output is the + /// same bytes as an uninterrupted run. + #[test] + fn interrupted_trailer_write_and_flush_are_retried() { + struct Interrupting { + buf: Vec, + writes: usize, + interrupt_write: usize, + flush_interrupts: usize, + } + impl Write for Interrupting { + fn write(&mut self, b: &[u8]) -> std::io::Result { + self.writes += 1; + if self.writes == self.interrupt_write { + return Err(std::io::Error::from(std::io::ErrorKind::Interrupted)); + } + self.buf.extend_from_slice(b); + Ok(b.len()) + } + fn flush(&mut self) -> std::io::Result<()> { + if self.flush_interrupts > 0 { + self.flush_interrupts -= 1; + return Err(std::io::Error::from(std::io::ErrorKind::Interrupted)); + } + Ok(()) + } + } + let (expected, checksums) = encode_data_file(all_types_items()).unwrap(); + // Count the writes of a plain run, then interrupt the last one. + let mut probe = Interrupting { + buf: Vec::new(), + writes: 0, + interrupt_write: usize::MAX, + flush_interrupts: 0, + }; + { + let mut encoder = DataFileEncoder::new(&mut probe); + for it in all_types_items() { + encoder.write_item(&it).unwrap(); + } + encoder.try_finish().unwrap(); + } + let last = probe.writes; + let mut encoder = DataFileEncoder::new(Interrupting { + buf: Vec::new(), + writes: 0, + interrupt_write: last, + flush_interrupts: 3, + }); + for it in all_types_items() { + encoder.write_item(&it).unwrap(); + } + let (sink, got) = encoder.finish().unwrap(); + assert_eq!(sink.buf, expected); + assert_eq!(got, checksums); + } + + /// A source that returns `Interrupted` is read through, as `read_to_end` + /// would; the decoder's sticky error is for real failures. + #[test] + fn interrupted_source_is_retried() { + struct Interrupting<'a> { + data: &'a [u8], + calls: usize, + } + impl Read for Interrupting<'_> { + fn read(&mut self, buf: &mut [u8]) -> std::io::Result { + self.calls += 1; + if self.calls.is_multiple_of(2) { + return Err(std::io::Error::from(std::io::ErrorKind::Interrupted)); + } + let n = buf.len().min(self.data.len()).min(7); + buf[..n].copy_from_slice(&self.data[..n]); + self.data = &self.data[n..]; + Ok(n) + } + } + let (compressed, checksums) = encode_data_file(all_types_items()).unwrap(); + let decoder = DataFileDecoder::new( + Interrupting { + data: &compressed, + calls: 0, + }, + (&checksums).into(), + ); + let items: Vec = decoder.collect::>().unwrap(); + assert_eq!(items, all_types_items()); + } + + /// The reader contract: an unknown member beside `Item` is refused; a + /// CRLF line is JSON whitespace and parses, its `\r` counted as a byte. + #[test] + fn unknown_members_are_refused_and_crlf_is_whitespace() { + let crlf = b"{\"Item\":{\"pk\":{\"S\":\"a\"}}}\r\n"; + let mut sink = Vec::new(); + { + let mut gz = GzEncoder::new(&mut sink, Compression::new(GZIP_LEVEL)); + gz.write_all(crlf).unwrap(); + gz.finish().unwrap(); + } + let expectations = DataFileExpectations { + uncompressed_bytes: Some(crlf.len() as u64), + ..DataFileExpectations::default() + }; + assert_eq!(decode_data_file(&sink, expectations).unwrap().len(), 1); + for body in [ + &b"{\"Item\":{\"pk\":{\"S\":\"a\"}},\"Extra\":1}\n"[..], + &b"{\"Extra\":1,\"Item\":{\"pk\":{\"S\":\"a\"}}}\n"[..], + ] { + let mut sink = Vec::new(); + { + let mut gz = GzEncoder::new(&mut sink, Compression::new(GZIP_LEVEL)); + gz.write_all(body).unwrap(); + gz.finish().unwrap(); + } + let err = decode_data_file(&sink, DataFileExpectations::default()).unwrap_err(); + assert!(matches!(err, FormatError::Json(_)), "{body:?}: {err}"); + } + } + + /// The end-of-stream checks run on a zero-item stream too. + #[test] + fn empty_file_with_wrong_expectations_fails() { + let (compressed, checksums) = encode_data_file(Vec::::new()).unwrap(); + let mut count = DataFileExpectations::from(&checksums); + count.item_count = Some(5); + assert!(matches!( + decode_data_file(&compressed, count).unwrap_err(), + FormatError::ItemCountMismatch { .. } + )); + let mut bytes = DataFileExpectations::from(&checksums); + bytes.uncompressed_bytes = Some(5); + assert!(matches!( + decode_data_file(&compressed, bytes).unwrap_err(), + FormatError::ByteCountMismatch { .. } + )); + let mut digest = DataFileExpectations::from(&checksums); + digest.sha256_hex = Some("0".repeat(64)); + assert!(matches!( + decode_data_file(&compressed, digest).unwrap_err(), + FormatError::ChecksumMismatch { .. } + )); + } + + /// A sink whose `flush` fails (a full disk under a `BufWriter`) fails the + /// finish and poisons the encoder; it is not a clean finish. + #[test] + fn failing_flush_poisons_the_encoder() { + struct FlushFails(Vec); + impl Write for FlushFails { + fn write(&mut self, b: &[u8]) -> std::io::Result { + self.0.extend_from_slice(b); + Ok(b.len()) + } + fn flush(&mut self) -> std::io::Result<()> { + Err(std::io::Error::other("disk full")) + } + } + let mut encoder = DataFileEncoder::new(FlushFails(Vec::new())); + for it in all_types_items() { + encoder.write_item(&it).unwrap(); + } + let err = encoder.try_finish().unwrap_err(); + assert!(matches!(err, FormatError::Encode(_)), "{err}"); + assert!(encoder.is_poisoned()); + } + + #[test] + fn interrupted_sink_is_retried() { + struct Interrupting { + buf: Vec, + interrupts_left: usize, + } + impl Write for Interrupting { + fn write(&mut self, b: &[u8]) -> std::io::Result { + if self.interrupts_left > 0 { + self.interrupts_left -= 1; + return Err(std::io::Error::from(std::io::ErrorKind::Interrupted)); + } + self.buf.extend_from_slice(b); + Ok(b.len()) + } + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } + } + let (expected, checksums) = encode_data_file(all_types_items()).unwrap(); + let mut encoder = DataFileEncoder::new(Interrupting { + buf: Vec::new(), + interrupts_left: 5, + }); + for it in all_types_items() { + encoder.write_item(&it).unwrap(); + } + let (sink, got) = encoder.finish().unwrap(); + assert_eq!(sink.buf, expected); + assert_eq!(got, checksums); + } + + /// Invalid UTF-8 inside a line is a decode error, not a lossy item. + #[test] + fn invalid_utf8_line_is_a_decode_error() { + let mut sink = Vec::new(); + { + let mut gz = GzEncoder::new(&mut sink, Compression::new(GZIP_LEVEL)); + gz.write_all(b"{\"Item\":{\"pk\":{\"S\":\"\xff\xfe\"}}}\n") + .unwrap(); + gz.finish().unwrap(); + } + let mut decoder = + DataFileDecoder::new(std::io::Cursor::new(sink), DataFileExpectations::default()); + let err = decoder.next_item().unwrap_err(); + assert!(matches!(err, FormatError::Decode(_)), "{err}"); + } + + /// A final line without a newline is accepted: the service's export + /// files end with one, but a reader that refused its absence would turn a + /// hand-concatenated file into a missing item. + #[test] + fn unterminated_final_line_is_an_item() { + let items = all_types_items(); + let mut sink = Vec::new(); + { + let mut gz = GzEncoder::new(&mut sink, Compression::new(GZIP_LEVEL)); + let mut all = Vec::new(); + for it in &items { + all.extend(encode_item_line(it).unwrap()); + } + all.pop(); + gz.write_all(&all).unwrap(); + gz.finish().unwrap(); + } + let decoded: Vec = + DataFileDecoder::new(std::io::Cursor::new(sink), DataFileExpectations::default()) + .collect::>() + .unwrap(); + assert_eq!(decoded, items); + } + + #[test] + fn sink_accepting_no_bytes_is_an_error_not_a_hang() { + struct Zero; + impl Write for Zero { + fn write(&mut self, _b: &[u8]) -> std::io::Result { + Ok(0) + } + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } + } + let (tx, rx) = std::sync::mpsc::channel(); + std::thread::spawn(move || { + let mut encoder = DataFileEncoder::new(Zero); + let r = encoder.write_item(&all_types_items()[0]); + let _ = tx.send(r.map_err(|e| e.to_string())); + }); + let outcome = rx + .recv_timeout(std::time::Duration::from_secs(10)) + .expect("write_item returned within the deadline"); + let err = outcome.expect_err("Ok(0) sink is an error"); + assert!(err.contains("accepted no bytes"), "{err}"); + } + + #[test] + fn try_finish_keeps_the_sink_and_refuses_further_items() { + let mut encoder = DataFileEncoder::new(Vec::new()); + for it in all_types_items() { + encoder.write_item(&it).unwrap(); + } + let checksums = encoder.try_finish().unwrap(); + assert!(encoder.is_finished()); + assert!(!encoder.is_poisoned(), "a clean finish is not a failure"); + let err = encoder.write_item(&all_types_items()[0]).unwrap_err(); + assert!(err.to_string().contains("finished"), "{err}"); + let bytes = encoder.into_inner(); + assert_eq!(bytes.len() as u64, checksums.compressed_bytes); + let (expected_bytes, expected) = encode_data_file(all_types_items()).unwrap(); + assert_eq!(bytes, expected_bytes); + assert_eq!(checksums, expected); + } + + #[test] + fn streaming_decoder_clean_stream_completes_with_counts_checked() { + let (compressed, checksums) = encode_data_file(all_types_items()).unwrap(); + let mut decoder = DataFileDecoder::new( + std::io::Cursor::new(compressed), + DataFileExpectations::from(&checksums), + ); + let mut n = 0; + while let Some(_item) = decoder.next_item().unwrap() { + n += 1; + } + assert_eq!(n, all_types_items().len()); + // A second call after completion stays at the end. + assert!(decoder.next_item().unwrap().is_none()); + } + + #[test] + fn verified_decoder_rejects_before_first_item() { + let (mut compressed, checksums) = encode_data_file(all_types_items()).unwrap(); + let n = compressed.len(); + compressed[n / 2] ^= 0x01; + let err = DataFileDecoder::verified(&compressed, DataFileExpectations::from(&checksums)) + .err() + .expect("digest checked before any item"); + assert!(matches!(err, FormatError::ChecksumMismatch { .. }), "{err}"); + } + + /// Items that compress well but not trivially, so the gzip level changes + /// the output (at a handful of short items levels 6 and 9 agree). + fn semi_compressible_items() -> Vec { + (0..2000u32) + .map(|i| { + item(vec![ + ("pk", AttributeValue::S(format!("item-{i:05}"))), + ("v", AttributeValue::N(((i * 7919) % 10007).to_string())), + ( + "s", + AttributeValue::S("lorem ipsum dolor sit amet ".repeat((i % 17) as usize)), + ), + ]) + }) + .collect() + } + + /// The encoder's bytes for a fixed input are pinned: a change of + /// `GZIP_LEVEL`, a timestamp or name in the gzip header, or another + /// compressor backend changes the digest. Two runs of one build already + /// agree (`encoder_output_is_deterministic`); this pins the bytes across + /// builds, as `Cargo.lock` pins the `flate2` and `miniz_oxide` versions. + #[test] + fn encoder_output_bytes_are_pinned() { + let (bytes, checksums) = encode_data_file(semi_compressible_items()).unwrap(); + // The gzip header: magic, deflate, no flags, mtime 0, XFL 0, OS 255 + // (flate2's defaults; level 6 sets no XFL bit). + assert_eq!(&bytes[..10], &[0x1f, 0x8b, 8, 0, 0, 0, 0, 0, 0, 0xff]); + assert_eq!( + (checksums.compressed_bytes, checksums.sha256_hex.as_str()), + (PINNED_LEN, PINNED_SHA256), + "the encoder's output changed" + ); + } + + const PINNED_LEN: u64 = 17_507; + const PINNED_SHA256: &str = "b5d05ccbf2851b5103069dab1870b41470025bda0af1b91f4ba5eaf4a57bad32"; + + /// A file recorded as 1 compressed byte is refused before it yields more + /// than a handful of items, however large it is and whatever the other + /// expectations say (here none but the size, and an uncompressed size of + /// `u64::MAX`). + #[test] + fn compressed_size_is_enforced_while_reading() { + /// Counts the bytes it hands out and the reads asked of it. + struct CountingSource { + inner: std::io::Cursor>, + stats: std::rc::Rc>, + } + impl Read for CountingSource { + fn read(&mut self, buf: &mut [u8]) -> std::io::Result { + let n = self.inner.read(buf)?; + let (bytes, reads) = self.stats.get(); + self.stats.set((bytes + n as u64, reads + 1)); + Ok(n) + } + } + let items = incompressible_items(); + let (bytes, c) = encode_data_file(items.clone()).unwrap(); + assert!(c.compressed_bytes > 256 * 1024, "{}", c.compressed_bytes); + for (recorded, max_yielded) in [(1, 0), (c.compressed_bytes / 3, 2)] { + let stats = std::rc::Rc::new(std::cell::Cell::new((0, 0))); + let source = CountingSource { + inner: std::io::Cursor::new(bytes.clone()), + stats: stats.clone(), + }; + let mut decoder = DataFileDecoder::new( + source, + DataFileExpectations { + compressed_bytes: Some(recorded), + uncompressed_bytes: Some(u64::MAX), + ..DataFileExpectations::default() + }, + ); + let mut yielded = 0; + let err = loop { + match decoder.next_item() { + Ok(Some(_)) => yielded += 1, + Ok(None) => panic!("{recorded}: must not complete"), + Err(e) => break e, + } + }; + assert!( + matches!( + err, + FormatError::CompressedSizeMismatch { expected, actual } + if expected == recorded && actual == recorded + 1 + ), + "{recorded}: {err}" + ); + assert!( + yielded <= max_yielded, + "{recorded}: {yielded} of {} items streamed first", + items.len() + ); + // The source handed out exactly one byte past the record. + let (handed_out, reads) = stats.get(); + assert_eq!(handed_out, recorded + 1, "{recorded}"); + // Sticky, and the reader asked the source for no more. + assert!(decoder.next_item().is_err()); + assert_eq!(stats.get(), (handed_out, reads), "{recorded}"); + } + } + + /// Once [`HashingReader`] has seen a byte past its limit it fails every + /// read without asking its source again (its own contract, below the + /// decoder's stickiness). + #[test] + fn hashing_reader_fails_every_read_after_an_overrun() { + struct Source(u64); + impl Read for Source { + fn read(&mut self, buf: &mut [u8]) -> std::io::Result { + self.0 += 1; + buf.fill(0); + Ok(buf.len()) + } + } + let mut reader = HashingReader { + inner: Source(0), + hasher: Some(CompressedHasher::new()), + bytes: 0, + limit: Some(10), + overrun: false, + }; + let mut buf = [0u8; 64]; + let err = reader.read(&mut buf).unwrap_err(); + assert_eq!(err.to_string(), OVERRUN); + assert_eq!((reader.inner.0, reader.bytes), (1, 11)); + for _ in 0..3 { + assert_eq!(reader.read(&mut buf).unwrap_err().to_string(), OVERRUN); + } + assert_eq!((reader.inner.0, reader.bytes), (1, 11)); + } + + /// The streaming loop, not just `verified`: many small items, a file far + /// larger than the recorded size, and the error arrives after a few + /// items of a thousand. + #[test] + fn compressed_size_overrun_stops_a_long_stream_early() { + let items: Vec = (0..20_000u32) + .map(|i| { + let mut x = u64::from(i).wrapping_mul(0x9e37_79b9_7f4a_7c15); + x ^= x >> 29; + item(vec![("pk", AttributeValue::S(format!("{x:016x}{i}")))]) + }) + .collect(); + let (bytes, c) = encode_data_file(items).unwrap(); + let recorded = c.compressed_bytes / 4; + let mut decoder = DataFileDecoder::new( + std::io::Cursor::new(bytes), + DataFileExpectations { + compressed_bytes: Some(recorded), + ..DataFileExpectations::default() + }, + ); + let mut yielded = 0u64; + let err = loop { + match decoder.next_item() { + Ok(Some(_)) => yielded += 1, + Ok(None) => panic!("must not complete"), + Err(e) => break e, + } + }; + assert!( + matches!(err, FormatError::CompressedSizeMismatch { expected, .. } if expected == recorded), + "{err}" + ); + assert!( + yielded < c.item_count / 2, + "{yielded} of {} items streamed before the size check", + c.item_count + ); + } + + /// A source shorter than recorded is refused at end of stream, a source + /// of exactly the recorded size passes, and `verified` checks the size + /// before the first item. + #[test] + fn compressed_size_mismatch_at_end_of_stream_and_in_verified() { + let (bytes, c) = encode_data_file(all_types_items()).unwrap(); + let with = |compressed_bytes| DataFileExpectations { + compressed_bytes: Some(compressed_bytes), + ..DataFileExpectations::default() + }; + let streamed = |e: DataFileExpectations| { + DataFileDecoder::new(std::io::Cursor::new(bytes.clone()), e) + .collect::, _>>() + }; + assert_eq!(streamed(with(c.compressed_bytes)).unwrap().len(), 2); + let err = streamed(with(c.compressed_bytes + 1)).unwrap_err(); + assert!( + matches!( + err, + FormatError::CompressedSizeMismatch { expected, actual } + if expected == c.compressed_bytes + 1 && actual == c.compressed_bytes + ), + "{err}" + ); + let err = streamed(with(c.compressed_bytes - 1)).unwrap_err(); + assert!( + matches!(err, FormatError::CompressedSizeMismatch { .. }), + "{err}" + ); + for recorded in [c.compressed_bytes - 1, c.compressed_bytes + 1] { + let err = DataFileDecoder::verified(&bytes, with(recorded)) + .err() + .expect("size checked before any item"); + assert!( + matches!(err, FormatError::CompressedSizeMismatch { .. }), + "{err}" + ); + } + assert_eq!( + DataFileExpectations::from(&c).compressed_bytes, + Some(c.compressed_bytes) + ); + } + + /// A data line that names one top-level attribute twice is refused, as + /// `ImportTable` refuses it, whichever value comes first; a repeated + /// name inside a nested map is the wire deserializer's (last wins), not + /// this format's. + #[test] + fn repeated_top_level_attribute_is_refused() { + let gz = |body: &[u8]| { + let mut sink = Vec::new(); + let mut enc = GzEncoder::new(&mut sink, Compression::new(GZIP_LEVEL)); + enc.write_all(body).unwrap(); + enc.finish().unwrap(); + sink + }; + for body in [ + &b"{\"Item\":{\"pk\":{\"S\":\"a\"},\"pk\":{\"S\":\"b\"}}}\n"[..], + &b"{\"Item\":{\"pk\":{\"S\":\"a\"},\"v\":{\"N\":\"1\"},\"v\":{\"S\":\"x\"}}}\n"[..], + ] { + let err = decode_data_file(&gz(body), DataFileExpectations::default()).unwrap_err(); + assert!(matches!(err, FormatError::Json(_)), "{err}"); + assert!(err.to_string().contains("more than once"), "{err}"); + } + let nested = b"{\"Item\":{\"pk\":{\"S\":\"a\"},\"m\":{\"M\":{\"x\":{\"S\":\"1\"},\"x\":{\"S\":\"2\"}}}}}\n"; + let items = decode_data_file(&gz(nested), DataFileExpectations::default()).unwrap(); + assert_eq!( + items[0].get("m"), + Some(&AttributeValue::M( + [("x".to_owned(), AttributeValue::S("2".to_owned()))] + .into_iter() + .collect() + )) + ); + } + + /// serde's messages quote the input without a bound; the `Json` + /// variant's message is bounded whatever the line holds. + #[test] + fn json_errors_on_data_lines_are_bounded() { + let long = "Q".repeat(1_000_000); + for body in [ + format!("{{\"Item\":{{\"pk\":{{\"{long}\":\"a\"}}}}}}\n"), + format!("{{\"Item\":{{\"pk\":{{\"SS\":[\"{long}\",\"{long}\"]}}}}}}\n"), + format!("{{\"{long}\":1}}\n"), + format!( + "{{\"Item\":{{\"pk\":{{\"S\":\"a\"}},\"{}\":{{\"S\":\"a\"}},\"{}\":{{\"S\":\"b\"}}}}}}\n", + "\u{1}".repeat(500), + "\u{1}".repeat(500) + ), + ] { + let mut sink = Vec::new(); + { + let mut gz = GzEncoder::new(&mut sink, Compression::new(GZIP_LEVEL)); + gz.write_all(body.as_bytes()).unwrap(); + gz.finish().unwrap(); + } + let err = decode_data_file(&sink, DataFileExpectations::default()).unwrap_err(); + assert!(matches!(err, FormatError::Json(_)), "{err}"); + let text = err.to_string(); + assert!(text.len() < 200, "{} bytes: {}", text.len(), &text[..200]); + assert!(!text.contains('\u{1}'), "control characters are escaped"); + } + } + + /// A recorded digest comes from an unvalidated `manifest-files.json` + /// line, so the mismatch message carries a bounded, escaped rendering of + /// it: a 1 MB expectation full of control characters (a newline, an ANSI + /// escape) gives a short message with none in it. + #[test] + fn checksum_mismatch_bounds_and_escapes_the_expected_value() { + let hostile = "x\n\u{1b}[2J\u{1b}[31mFAKE LOG LINE\r\u{7}".repeat(40_000); + assert!(hostile.len() > 1_000_000); + let (compressed, checksums) = encode_data_file(all_types_items()).unwrap(); + for expectations in [ + DataFileExpectations { + md5_base64: Some(hostile.clone()), + ..Default::default() + }, + DataFileExpectations { + sha256_hex: Some(hostile.clone()), + ..Default::default() + }, + ] { + let err = DataFileDecoder::verified(&compressed, expectations) + .err() + .expect("a mismatch"); + assert!(matches!(err, FormatError::ChecksumMismatch { .. }), "{err}"); + let text = err.to_string(); + assert!(text.len() < 300, "{} bytes", text.len()); + assert!( + !text.chars().any(char::is_control), + "control characters are escaped: {text:?}" + ); + assert!(text.contains(r#""x\n\u{1b}[2J"#), "{text}"); + } + // The computed value is the decoder's own and stays as it is. + let err = DataFileDecoder::verified( + &compressed, + DataFileExpectations { + md5_base64: Some("AAAAAAAAAAAAAAAAAAAAAA==".to_owned()), + ..Default::default() + }, + ) + .err() + .expect("a mismatch"); + assert_eq!( + err.to_string(), + format!( + "data file md5 checksum mismatch: expected \"AAAAAAAAAAAAAAAAAAAAAA==\", \ + computed {}", + checksums.md5_base64 + ) + ); + } + + /// `Debug` is how `unwrap`, `expect` and a `tracing` `?err` field print + /// an error, so it is bounded like `Display`: a 1 MB attribute + /// descriptor that serde quotes in full. + #[test] + fn json_error_debug_is_bounded() { + let long = "Q".repeat(1_000_000); + for body in [ + format!("{{\"Item\":{{\"pk\":{{\"{long}\":\"a\"}}}}}}\n"), + format!("{{\"{long}\":1}}\n"), + ] { + let mut sink = Vec::new(); + { + let mut gz = GzEncoder::new(&mut sink, Compression::new(GZIP_LEVEL)); + gz.write_all(body.as_bytes()).unwrap(); + gz.finish().unwrap(); + } + let err = decode_data_file(&sink, DataFileExpectations::default()).unwrap_err(); + assert!(matches!(err, FormatError::Json(_)), "{err}"); + let debug = format!("{err:?}"); + assert!( + debug.len() < 300, + "{} bytes: {}", + debug.len(), + &debug[..300] + ); + assert!(debug.contains("QQQQ"), "{debug}"); + } + // A manifest's error (no data-file line) is bounded the same way. + let err = FormatError::from( + serde_json::from_str::(&format!("{{\"{long}\":1}}")) + .err() + .unwrap(), + ); + assert!(format!("{err:?}").len() < 300); + assert!(err.to_string().len() < 300); + } + + /// Each line is parsed on its own, so serde's position is within the + /// line; the error names the line in the file, 1-based, so an operator + /// can find the item in a large file. + #[test] + fn line_errors_name_the_line_in_the_file() { + let good = encode_item_line(&item(vec![("pk", AttributeValue::S("a".into()))])).unwrap(); + let gzip = |lines: &[&[u8]]| { + let mut sink = Vec::new(); + let mut gz = GzEncoder::new(&mut sink, Compression::new(GZIP_LEVEL)); + for line in lines { + gz.write_all(line).unwrap(); + } + gz.finish().unwrap(); + sink + }; + let bad_json = br#"{"Item":{"pk":{"Q":"a"}}} +"#; + let err = decode_data_file( + &gzip(&[&good, &good, bad_json, &good]), + DataFileExpectations::default(), + ) + .unwrap_err(); + let FormatError::Json(json) = &err else { + panic!("{err}") + }; + assert_eq!(json.data_file_line(), Some(3)); + assert_eq!((json.line(), json.column()), (1, 23)); + assert_eq!( + err.to_string(), + "JSON error: data file line 3: unknown AttributeValue type descriptor: Q \ + at column 23 of the line" + ); + // Invalid UTF-8 on line 2. + let bad_utf8 = b"{\"Item\":{\"pk\":{\"S\":\"\xff\"}}}\n"; + let err = decode_data_file(&gzip(&[&good, bad_utf8]), DataFileExpectations::default()) + .unwrap_err(); + assert!(matches!(err, FormatError::Decode(_)), "{err}"); + assert!( + err.to_string() + .starts_with("data file decode error: data file line 2: invalid utf-8"), + "{err}" + ); + // A blank first line is line 1. + let err = decode_data_file(&gzip(&[b"\n"]), DataFileExpectations::default()).unwrap_err(); + assert!(err.to_string().contains("data file line 1: "), "{err}"); + // A descriptor name ending in ` at line 3 column 4` sets serde's + // position; no column is shown for it, and the name is kept whole. + let forged = br#"{"Item":{"a":{"L":[{"X at line 3 column 4":1}]}}} +"#; + let err = + decode_data_file(&gzip(&[&good, forged]), DataFileExpectations::default()).unwrap_err(); + let FormatError::Json(json) = &err else { + panic!("{err}") + }; + assert_eq!((json.line(), json.column()), (3, 4)); + assert_eq!( + err.to_string(), + "JSON error: data file line 2: unknown AttributeValue type descriptor: X \ + at line 3 column 4" + ); + } + + /// A prehashed decoder does not hash the bytes and does not compare the + /// digests (a digest that does not match is not noticed), and enforces + /// the compressed size, the uncompressed size and the item count as the + /// hashing decoder does. `verified` hashes once, before the stream. + #[test] + fn prehashed_decoder_skips_the_digest_and_keeps_every_other_check() { + let items = all_types_items(); + let (bytes, checksums) = encode_data_file(items.clone()).unwrap(); + let entry = crate::backup_format::DataFileEntry::from_checksums( + crate::backup_format::data_file_key(1), + &checksums, + None, + ); + let decode = |entry: &crate::backup_format::DataFileEntry| { + let mut decoder = entry.decoder_prehashed(&bytes[..]).unwrap(); + let mut out = Vec::new(); + loop { + assert!(!decoder.hashing(), "a prehashed decoder hashes"); + match decoder.next_item() { + Ok(Some(item)) => out.push(item), + Ok(None) => return Ok(out), + Err(e) => return Err(e), + } + } + }; + assert_eq!(decode(&entry).unwrap(), items); + // Wrong digests: not compared, so the file reads. + let mut wrong_digests = entry.clone(); + wrong_digests.sha256 = "0".repeat(64); + wrong_digests.md5 = "AAAAAAAAAAAAAAAAAAAAAA==".to_owned(); + assert_eq!(decode(&wrong_digests).unwrap(), items); + // The hashing decoder refuses the same entry. + assert!(matches!( + wrong_digests + .decoder(&bytes[..]) + .unwrap() + .collect::, _>>(), + Err(FormatError::ChecksumMismatch { .. }) + )); + // Every other expectation is still enforced. + let mut short = entry.clone(); + short.bytes -= 1; + assert!(matches!( + decode(&short), + Err(FormatError::CompressedSizeMismatch { .. }) + )); + let mut long = entry.clone(); + long.bytes += 1; + assert!(matches!( + decode(&long), + Err(FormatError::CompressedSizeMismatch { .. }) + )); + let mut count = entry.clone(); + count.item_count += 1; + assert!(matches!( + decode(&count), + Err(FormatError::ItemCountMismatch { .. }) + )); + let mut size = entry.clone(); + size.uncompressed_bytes -= 1; + assert!(matches!( + decode(&size), + Err(FormatError::ByteCountMismatch { .. }) + )); + // One byte more than the stream holds is found at the end of the + // stream, where the hashing decoder compares its digests; the + // incremental check cannot see it. + let mut size = entry.clone(); + size.uncompressed_bytes += 1; + let mut decoder = size.decoder_prehashed(&bytes[..]).unwrap(); + for _ in 0..items.len() { + decoder.next_item().unwrap().unwrap(); + } + assert!(matches!( + decoder.next_item(), + Err(FormatError::ByteCountMismatch { + expected, + actual + }) if expected == entry.uncompressed_bytes + 1 && actual == entry.uncompressed_bytes + )); + // `verified` checks the digest itself, then streams without hashing. + let decoder = + DataFileDecoder::verified(&bytes, DataFileExpectations::from(&checksums)).unwrap(); + assert!(!decoder.hashing()); + assert!( + DataFileDecoder::verified(&bytes, wrong_digests.expectations()).is_err(), + "verified compares the digest" + ); + // The hashing decoder hashes until its end-of-stream checks. + let decoder = entry.decoder(&bytes[..]).unwrap(); + assert!(decoder.hashing()); + } + + /// The encoder reuses one line buffer: a short line after a long one is + /// written whole and alone, and `encode_item_line_into` clears the + /// buffer it is given. + #[test] + fn encoder_line_buffer_is_cleared_between_items() { + let long = item(vec![("pk", AttributeValue::S("x".repeat(500)))]); + let short = item(vec![("pk", AttributeValue::S("y".to_owned()))]); + let (bytes, checksums) = encode_data_file([long.clone(), short.clone()]).unwrap(); + assert_eq!( + decode_data_file(&bytes, DataFileExpectations::from(&checksums)).unwrap(), + vec![long.clone(), short.clone()] + ); + let expected = + encode_item_line(&long).unwrap().len() + encode_item_line(&short).unwrap().len(); + assert_eq!(checksums.uncompressed_bytes, expected as u64); + let mut line = b"left over".to_vec(); + encode_item_line_into(&short, &mut line).unwrap(); + assert_eq!(line, encode_item_line(&short).unwrap()); + } +} diff --git a/crates/core/src/backup_format/ddb_export.rs b/crates/core/src/backup_format/ddb_export.rs new file mode 100644 index 000000000..5698b0d3d --- /dev/null +++ b/crates/core/src/backup_format/ddb_export.rs @@ -0,0 +1,678 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! The two manifests Amazon `DynamoDB` writes for `ExportTableToPointInTime` +//! with `DYNAMODB_JSON` output: `manifest-summary.json` (one JSON document) +//! and `manifest-files.json` (JSON lines, one object per data file). +//! +//! Field sets and encodings follow the service's documented output format: +//! . +//! Notably: `version` is the string `"2020-06-30"` for full exports, +//! timestamps are ISO 8601 strings, `s3SseKmsKeyId` is serialized as an +//! explicit `null` when absent, `exportType` appears only on incremental +//! exports, and each files-manifest line carries `itemCount`, `md5Checksum` +//! (base64 of the data file's MD5), `etag`, and `dataFileS3Key`. +//! +//! `dataFileS3Key` is the object's full key in the bucket, not a path +//! relative to the manifest. For a full export to a prefix the service +//! writes `/AWSDynamoDB//data/` (measured on +//! `DynamoDB`). The documentation's full-export example differs from that: +//! it has `s3Prefix` `2020-Nov` and keys starting at +//! `AWSDynamoDB//data/`, without the prefix (its +//! incremental-export example carries the prefix). The checked-in fixture +//! under `testdata/ddb-export/` copies the documentation's full-export +//! example, so it pins the field set and encodings, not the key shape. An +//! `ExtendDB` writer records the full store key, +//! `/data/`, where `` is the backup prefix +//! `//` ([`super::BackupArn::store_prefix`]), +//! preceded by the store's configured key prefix when it has one; +//! [`super::BackupDocuments::build`] derives it from the backup ARN and the +//! store's key prefix, once for both manifests. `ImportTable` does not read this manifest (it lists the prefix it +//! is given), so the field is informational; `etag` is optional here because +//! a filesystem store has none. +//! +//! `manifest-summary.json` for an `ExtendDB` backup, which +//! [`super::BackupDocuments::build`] derives from the +//! [`super::ExtenddbManifest`] (the summary has no reader in `ExtendDB`; it +//! is written for tools that read the service's export layout). `` +//! below is the backup's key prefix in the store with every leading and +//! trailing `/` dropped ([`trim_key_prefix`]): +//! +//! | member | value | +//! |---|---| +//! | `version` | `2020-06-30` ([`EXPORT_SUMMARY_VERSION`]) | +//! | `exportArn` | the backup ARN (`backup_arn`); there is no export | +//! | `startTime`, `endTime`, `exportTime` | the snapshot instant, `backup_creation_date_time`, in RFC 3339 UTC with millisecond precision (`2025-09-10T10:26:40.123Z`); the backup is a snapshot, so the three instants are one | +//! | `tableArn`, `tableId` | `source_table.table_arn`, `source_table.table_id` | +//! | `s3Bucket` | the bucket name on an S3 store; the root path on a filesystem store | +//! | `s3Prefix` | ``: the backup prefix, `/
/`, preceded by the store's configured key prefix when it has one | +//! | `s3SseAlgorithm` | `null`: the store's encryption is the store's configuration, not the format's; the service writes `AES256` or `aws:kms` | +//! | `s3SseKmsKeyId` | `null` | +//! | `manifestFilesS3Key` | `/manifest-files.json`; `manifest-files.json` when `` is empty | +//! | `billedSizeBytes` | `backup_size_bytes` (the compressed size; nothing is billed) | +//! | `itemCount` | the sum of the data files' `item_count` | +//! | `outputFormat` | `DYNAMODB_JSON` ([`DYNAMODB_JSON_OUTPUT_FORMAT`]) | +//! | `exportType` | absent (a full export) | + +use serde::{Deserialize, Serialize}; + +use super::FormatError; +use super::layout::{DATA_PREFIX, FILES_MANIFEST_FILE, backup_id_epoch_millis}; +use super::manifest::{DataFileEntry, ExtenddbManifest}; + +/// The `version` value the service writes for full exports. +pub const EXPORT_SUMMARY_VERSION: &str = "2020-06-30"; + +/// The `outputFormat` value for `DynamoDB` JSON exports. +pub const DYNAMODB_JSON_OUTPUT_FORMAT: &str = "DYNAMODB_JSON"; + +/// The `manifest-summary.json` document. +/// +/// Unknown fields are tolerated so a newer service revision (for example the +/// incremental-export `exportFromTime`/`exportToTime`/`outputView` members) +/// still parses. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ExportSummaryManifest { + /// Manifest schema version, `"2020-06-30"` for full exports. + pub version: String, + /// ARN of the export. + pub export_arn: String, + /// When the export started, ISO 8601. + pub start_time: String, + /// When the export finished, ISO 8601. + pub end_time: String, + /// ARN of the exported table. + pub table_arn: String, + /// Unique id of the exported table. + pub table_id: String, + /// Point in time the export represents, ISO 8601. + pub export_time: String, + /// Destination bucket. + pub s3_bucket: String, + /// Destination prefix. + pub s3_prefix: String, + /// Server-side encryption algorithm of the written objects (`AES256` or + /// `aws:kms` from the service); `null` for an `ExtendDB` backup (module + /// docs). Never skipped: the member is present in every document. + pub s3_sse_algorithm: Option, + /// KMS key id when SSE-KMS was used; the service writes an explicit + /// `null` otherwise, so this member is never skipped. + pub s3_sse_kms_key_id: Option, + /// Key of the `manifest-files.json` object. + pub manifest_files_s3_key: String, + /// Billed size of the export. + pub billed_size_bytes: i64, + /// Total item count across all data files. + pub item_count: i64, + /// Output format, `"DYNAMODB_JSON"` here. + pub output_format: String, + /// Present only for incremental exports, where the service writes + /// `"INCREMENTAL_EXPORT"`; absent for full exports. + #[serde(skip_serializing_if = "Option::is_none", default)] + pub export_type: Option, +} + +impl ExportSummaryManifest { + /// The summary an `ExtendDB` backup writes, with the member values the + /// module docs table gives: `bucket` is the bucket name (S3) or root + /// path (filesystem), `prefix` the backup's key prefix in the store + /// (every leading and trailing `/` dropped, [`trim_key_prefix`]). The + /// instants are + /// taken from the backup id in `manifest.backup_arn`, which + /// [`ExtenddbManifest::validate`] has checked equals + /// `backup_creation_date_time`. + /// + /// # Errors + /// + /// [`FormatError::InvalidArn`] when `backup_arn` does not parse, + /// [`FormatError::InvalidManifest`] when its id carries no instant or + /// the item counts overflow; neither happens for a manifest that passed + /// [`ExtenddbManifest::validate`] unless its counts sum past `i64::MAX`. + pub(super) fn for_backup( + manifest: &ExtenddbManifest, + bucket: &str, + prefix: &str, + ) -> Result { + let arn = manifest.backup_arn()?; + let millis = backup_id_epoch_millis(arn.backup_id()).ok_or_else(|| { + FormatError::InvalidManifest(format!( + "backup id {:?} does not carry the backup's creation instant", + arn.backup_id() + )) + })?; + let instant = rfc3339_millis(millis)?; + let item_count = manifest + .data_files + .iter() + .try_fold(0u64, |sum, f| sum.checked_add(f.item_count)) + .and_then(|sum| i64::try_from(sum).ok()) + .ok_or_else(|| { + FormatError::InvalidManifest("data file item counts overflow".to_owned()) + })?; + let prefix = trim_key_prefix(prefix); + Ok(Self { + version: EXPORT_SUMMARY_VERSION.to_owned(), + export_arn: manifest.backup_arn.clone(), + start_time: instant.clone(), + end_time: instant.clone(), + table_arn: manifest.source_table.table_arn.clone(), + table_id: manifest.source_table.table_id.clone(), + export_time: instant, + s3_bucket: bucket.to_owned(), + s3_prefix: prefix.to_owned(), + s3_sse_algorithm: None, + s3_sse_kms_key_id: None, + manifest_files_s3_key: store_key(prefix, FILES_MANIFEST_FILE), + billed_size_bytes: manifest.backup_size_bytes, + item_count, + output_format: DYNAMODB_JSON_OUTPUT_FORMAT.to_owned(), + export_type: None, + }) + } + + /// Serialize to the compact JSON document the file holds. + /// + /// # Errors + /// + /// Returns [`FormatError::Json`] when serialization fails. + pub fn to_json(&self) -> Result { + Ok(serde_json::to_string(self)?) + } + + /// Parse a `manifest-summary.json` document. + /// + /// # Errors + /// + /// Returns [`FormatError::Json`] when the document does not parse. + pub fn from_json(s: &str) -> Result { + Ok(serde_json::from_str(s)?) + } +} + +/// One line of `manifest-files.json`: a single exported data file. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ExportFileEntry { + /// Number of items in the data file. + pub item_count: u64, + /// Base64 MD5 of the data file's bytes. + pub md5_checksum: String, + /// `ETag` of the data file object as the store reported it, without + /// quotes. The service always writes one; an `ExtendDB` writer omits the + /// member when the store (filesystem) has no `ETag`. + #[serde(skip_serializing_if = "Option::is_none", default)] + pub etag: Option, + /// Key of the data file object. + pub data_file_s3_key: String, +} + +impl ExportFileEntry { + /// The `manifest-files.json` line for a data file the + /// `extenddb-manifest.json` records as `entry`: `itemCount` and + /// `md5Checksum` are the entry's, so the two manifests cannot disagree; + /// `dataFileS3Key` is `store_key_prefix` (every leading and trailing + /// `/` dropped, [`trim_key_prefix`]) followed by `/` and the entry's key + /// (`/data/`, module docs), and `etag` is what the store + /// reported for the object, `None` on a store without one. + #[must_use] + pub(super) fn from_data_file( + entry: &DataFileEntry, + store_key_prefix: &str, + etag: Option, + ) -> Self { + debug_assert!(entry.key.starts_with(DATA_PREFIX), "{}", entry.key); + Self { + item_count: entry.item_count, + md5_checksum: entry.md5.clone(), + etag, + data_file_s3_key: store_key(store_key_prefix, &entry.key), + } + } +} + +/// A store key prefix as the export manifests carry it: every leading and +/// trailing `/` dropped, so `a//` is `a`, `/a/` is `a` and `//` is empty. +/// `s3Prefix`, `manifestFilesS3Key` and `dataFileS3Key` all start from this +/// form, and [`super::BackupDocuments::build`] states how it meets the +/// store's own normalization of a configured prefix. +pub(super) fn trim_key_prefix(prefix: &str) -> &str { + prefix.trim_matches('/') +} + +/// The store key of `relative` (a key relative to the backup prefix) under +/// `prefix`: `/` with `prefix` in [`trim_key_prefix`]'s +/// form, and `relative` alone when that is empty. Both export manifests +/// build their keys with it. +fn store_key(prefix: &str, relative: &str) -> String { + let prefix = trim_key_prefix(prefix); + if prefix.is_empty() { + relative.to_owned() + } else { + format!("{prefix}/{relative}") + } +} + +/// `millis` since the Unix epoch as RFC 3339 UTC with millisecond precision +/// (`2025-09-10T10:26:40.123Z`), the form the summary manifest's instants +/// take. +fn rfc3339_millis(millis: u64) -> Result { + let nanos = i128::from(millis) * 1_000_000; + let instant = time::OffsetDateTime::from_unix_timestamp_nanos(nanos) + .map_err(|e| FormatError::InvalidManifest(format!("backup instant {millis} ms: {e}")))?; + let format = time::format_description::parse( + "[year]-[month]-[day]T[hour]:[minute]:[second].[subsecond digits:3]Z", + ) + .expect("a constant format description parses"); + instant + .format(&format) + .map_err(|e| FormatError::InvalidManifest(format!("backup instant {millis} ms: {e}"))) +} + +/// Serialize a files manifest: one compact JSON object per line, each line +/// terminated with a newline, matching the JSON lines format the service +/// writes. +/// +/// # Errors +/// +/// Returns [`FormatError::Json`] when serialization fails. +pub(super) fn write_files_manifest(entries: &[ExportFileEntry]) -> Result { + let mut out = String::new(); + for entry in entries { + out.push_str(&serde_json::to_string(entry)?); + out.push('\n'); + } + Ok(out) +} + +/// Parse a `manifest-files.json` document (JSON lines). Blank lines are +/// tolerated, including the absence of a trailing newline. +/// +/// # Errors +/// +/// Returns [`FormatError::Json`] when any non-blank line does not parse. +pub fn parse_files_manifest(s: &str) -> Result, FormatError> { + let mut entries = Vec::new(); + for line in s.lines() { + if line.trim().is_empty() { + continue; + } + entries.push(serde_json::from_str(line)?); + } + Ok(entries) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn sample_summary() -> ExportSummaryManifest { + ExportSummaryManifest { + version: EXPORT_SUMMARY_VERSION.to_owned(), + export_arn: "arn:aws:dynamodb:us-east-1:123456789012:table/ProductCatalog/export/01234567890123-a1b2c3d4".to_owned(), + start_time: "2020-11-04T07:28:34.028Z".to_owned(), + end_time: "2020-11-04T07:33:43.897Z".to_owned(), + table_arn: "arn:aws:dynamodb:us-east-1:123456789012:table/ProductCatalog".to_owned(), + table_id: "12345a12-abcd-123a-ab12-1234abc12345".to_owned(), + export_time: "2020-11-04T07:28:34.028Z".to_owned(), + s3_bucket: "ddb-productcatalog-export".to_owned(), + s3_prefix: "2020-Nov".to_owned(), + s3_sse_algorithm: Some("AES256".to_owned()), + s3_sse_kms_key_id: None, + manifest_files_s3_key: "AWSDynamoDB/01693685827463-2d8752fd/manifest-files.json".to_owned(), + billed_size_bytes: 0, + item_count: 8, + output_format: DYNAMODB_JSON_OUTPUT_FORMAT.to_owned(), + export_type: None, + } + } + + #[test] + fn summary_serde_round_trip() { + let summary = sample_summary(); + let json = summary.to_json().unwrap(); + let parsed = ExportSummaryManifest::from_json(&json).unwrap(); + assert_eq!(parsed, summary); + } + + #[test] + fn summary_uses_documented_member_names() { + let json = sample_summary().to_json().unwrap(); + for member in [ + "\"version\"", + "\"exportArn\"", + "\"startTime\"", + "\"endTime\"", + "\"tableArn\"", + "\"tableId\"", + "\"exportTime\"", + "\"s3Bucket\"", + "\"s3Prefix\"", + "\"s3SseAlgorithm\"", + "\"s3SseKmsKeyId\"", + "\"manifestFilesS3Key\"", + "\"billedSizeBytes\"", + "\"itemCount\"", + "\"outputFormat\"", + ] { + assert!(json.contains(member), "missing member {member} in {json}"); + } + } + + #[test] + fn summary_writes_explicit_null_kms_key() { + let json = sample_summary().to_json().unwrap(); + assert!(json.contains("\"s3SseKmsKeyId\":null")); + } + + /// `s3SseAlgorithm` is present as `null` when absent, never skipped, + /// and parses back. + #[test] + fn summary_writes_explicit_null_sse_algorithm() { + let mut summary = sample_summary(); + summary.s3_sse_algorithm = None; + let json = summary.to_json().unwrap(); + assert!(json.contains("\"s3SseAlgorithm\":null"), "{json}"); + assert_eq!(ExportSummaryManifest::from_json(&json).unwrap(), summary); + } + + const V1: &[u8] = include_bytes!("../../testdata/extenddb-manifest/v1.json"); + + /// Every member of the summary an `ExtendDB` backup writes is the value + /// the module docs table gives, derived from the manifest. + #[test] + fn summary_for_backup_follows_the_documented_table() { + let manifest = ExtenddbManifest::from_json_bytes(V1).unwrap(); + let summary = ExportSummaryManifest::for_backup( + &manifest, + "backups-bucket", + "123456789012/Music/01757500000123-0badcafe/", + ) + .unwrap(); + assert_eq!(summary.version, "2020-06-30"); + assert_eq!(summary.export_arn, manifest.backup_arn); + // 1757500000.123 s is 2025-09-10T10:26:40.123Z. + assert_eq!(summary.start_time, "2025-09-10T10:26:40.123Z"); + assert_eq!(summary.end_time, summary.start_time); + assert_eq!(summary.export_time, summary.start_time); + assert_eq!(summary.table_arn, manifest.source_table.table_arn); + assert_eq!(summary.table_id, manifest.source_table.table_id); + assert_eq!(summary.s3_bucket, "backups-bucket"); + assert_eq!( + summary.s3_prefix, + "123456789012/Music/01757500000123-0badcafe" + ); + assert_eq!(summary.s3_sse_algorithm, None); + assert_eq!(summary.s3_sse_kms_key_id, None); + assert_eq!( + summary.manifest_files_s3_key, + "123456789012/Music/01757500000123-0badcafe/manifest-files.json" + ); + assert_eq!(summary.billed_size_bytes, manifest.backup_size_bytes); + assert_eq!(summary.billed_size_bytes, 61_828); + assert_eq!(summary.item_count, 100_000); + assert_eq!( + u64::try_from(summary.item_count).unwrap(), + manifest + .data_files + .iter() + .map(|f| f.item_count) + .sum::() + ); + assert_eq!(summary.output_format, "DYNAMODB_JSON"); + assert_eq!(summary.export_type, None); + // Without a trailing slash on the prefix the keys are the same. + let again = ExportSummaryManifest::for_backup( + &manifest, + "backups-bucket", + "123456789012/Music/01757500000123-0badcafe", + ) + .unwrap(); + assert_eq!(again, summary); + // An empty prefix (a store rooted at the backup) gives the bare + // file name, as `ExportFileEntry::from_data_file` gives the bare + // data file key, not a key with a leading `/`. + for empty in ["", "/"] { + let bare = + ExportSummaryManifest::for_backup(&manifest, "backups-bucket", empty).unwrap(); + assert_eq!(bare.s3_prefix, ""); + assert_eq!(bare.manifest_files_s3_key, "manifest-files.json"); + let line = ExportFileEntry::from_data_file(&manifest.data_files[0], empty, None); + assert_eq!(line.data_file_s3_key, manifest.data_files[0].key); + } + // The written document has every member, `exportType` excepted. + let json = summary.to_json().unwrap(); + assert!(json.contains("\"s3SseAlgorithm\":null"), "{json}"); + assert!(!json.contains("exportType"), "{json}"); + assert_eq!(ExportSummaryManifest::from_json(&json).unwrap(), summary); + } + + /// The instant formatter: whole seconds keep three fraction digits, and + /// the epoch and a far instant format. + #[test] + fn rfc3339_millis_has_millisecond_precision() { + assert_eq!(rfc3339_millis(0).unwrap(), "1970-01-01T00:00:00.000Z"); + assert_eq!( + rfc3339_millis(1_757_500_000_000).unwrap(), + "2025-09-10T10:26:40.000Z" + ); + assert_eq!( + rfc3339_millis(1_757_500_000_123).unwrap(), + "2025-09-10T10:26:40.123Z" + ); + assert_eq!( + rfc3339_millis(99_999_999_999_999).unwrap(), + "5138-11-16T09:46:39.999Z" + ); + } + + /// The `manifest-files.json` line is derived from the manifest's entry, + /// so `md5Checksum` and `itemCount` agree with `md5` and `item_count` by + /// construction, and `dataFileS3Key` is the full store key. + #[test] + fn file_entry_from_data_file_agrees_with_the_manifest_entry() { + let manifest = ExtenddbManifest::from_json_bytes(V1).unwrap(); + let prefix = "123456789012/Music/01757500000123-0badcafe/"; + for (entry, etag) in manifest + .data_files + .iter() + .zip([Some("etag-1".to_owned()), None]) + { + let line = ExportFileEntry::from_data_file(entry, prefix, etag.clone()); + assert_eq!(line.item_count, entry.item_count); + assert_eq!(line.md5_checksum, entry.md5); + assert_eq!(line.etag, etag); + assert_eq!( + line.data_file_s3_key, + format!("123456789012/Music/01757500000123-0badcafe/{}", entry.key) + ); + assert!(line.data_file_s3_key.ends_with(".json.gz")); + // The same key whether or not the prefix ends in `/`, and with a + // store key prefix in front of the backup prefix. + let no_slash = + ExportFileEntry::from_data_file(entry, prefix.trim_end_matches('/'), None); + assert_eq!(no_slash.data_file_s3_key, line.data_file_s3_key); + let rooted = ExportFileEntry::from_data_file(entry, &format!("root/{prefix}"), None); + assert_eq!( + rooted.data_file_s3_key, + format!("root/{}", line.data_file_s3_key) + ); + let bare = ExportFileEntry::from_data_file(entry, "", None); + assert_eq!(bare.data_file_s3_key, entry.key); + } + // The counts the two manifests carry are the same numbers. + let lines: Vec = manifest + .data_files + .iter() + .map(|e| ExportFileEntry::from_data_file(e, prefix, None)) + .collect(); + let summary = ExportSummaryManifest::for_backup(&manifest, "b", prefix).unwrap(); + assert_eq!( + lines.iter().map(|l| l.item_count).sum::(), + u64::try_from(summary.item_count).unwrap() + ); + let text = write_files_manifest(&lines).unwrap(); + assert_eq!(parse_files_manifest(&text).unwrap(), lines); + } + + #[test] + fn summary_omits_export_type_for_full_exports() { + let json = sample_summary().to_json().unwrap(); + assert!(!json.contains("exportType")); + } + + #[test] + fn summary_carries_export_type_for_incremental_exports() { + let mut summary = sample_summary(); + summary.export_type = Some("INCREMENTAL_EXPORT".to_owned()); + let json = summary.to_json().unwrap(); + assert!(json.contains("\"exportType\":\"INCREMENTAL_EXPORT\"")); + let parsed = ExportSummaryManifest::from_json(&json).unwrap(); + assert_eq!(parsed, summary); + } + + #[test] + fn summary_parses_documented_example() { + // The full-export example from the documented output format. + let doc = r#"{ + "version": "2020-06-30", + "exportArn": "arn:aws:dynamodb:us-east-1:123456789012:table/ProductCatalog/export/01234567890123-a1b2c3d4", + "startTime": "2020-11-04T07:28:34.028Z", + "endTime": "2020-11-04T07:33:43.897Z", + "tableArn": "arn:aws:dynamodb:us-east-1:123456789012:table/ProductCatalog", + "tableId": "12345a12-abcd-123a-ab12-1234abc12345", + "exportTime": "2020-11-04T07:28:34.028Z", + "s3Bucket": "ddb-productcatalog-export", + "s3Prefix": "2020-Nov", + "s3SseAlgorithm": "AES256", + "s3SseKmsKeyId": null, + "manifestFilesS3Key": "AWSDynamoDB/01693685827463-2d8752fd/manifest-files.json", + "billedSizeBytes": 0, + "itemCount": 8, + "outputFormat": "DYNAMODB_JSON" +}"#; + let parsed = ExportSummaryManifest::from_json(doc).unwrap(); + assert_eq!(parsed, sample_summary()); + } + + #[test] + fn summary_tolerates_incremental_only_members() { + // Incremental exports add members full-export readers do not model. + let doc = r#"{ + "version": "2023-08-01", + "exportArn": "arn:aws:dynamodb:us-east-1:123456789012:table/t/export/x", + "startTime": "2023-09-19T04:20:18.000Z", + "endTime": "2023-09-19T04:40:24.780Z", + "tableArn": "arn:aws:dynamodb:us-east-1:123456789012:table/t", + "tableId": "b116b490-6460-4d4a-9a6b-5d360abf4fb3", + "exportFromTime": "2023-09-18T17:00:00.000Z", + "exportToTime": "2023-09-19T04:00:00.000Z", + "exportTime": "2023-09-19T04:00:00.000Z", + "s3Bucket": "b", + "s3Prefix": "p", + "s3SseAlgorithm": "AES256", + "s3SseKmsKeyId": null, + "manifestFilesS3Key": "p/AWSDynamoDB/x/manifest-files.json", + "billedSizeBytes": 20901239349, + "itemCount": 169928274, + "outputFormat": "DYNAMODB_JSON", + "outputView": "NEW_AND_OLD_IMAGES", + "exportType": "INCREMENTAL_EXPORT" +}"#; + let parsed = ExportSummaryManifest::from_json(doc).unwrap(); + assert_eq!(parsed.export_type.as_deref(), Some("INCREMENTAL_EXPORT")); + assert_eq!(parsed.item_count, 169_928_274); + } + + #[test] + fn file_entry_serde_round_trip() { + let entry = ExportFileEntry { + item_count: 8, + md5_checksum: "sQMSpEILNgoQmarvDFonGQ==".to_owned(), + etag: Some("af83d6f217c19b8b0fff8023d8ca4716-1".to_owned()), + data_file_s3_key: "AWSDynamoDB/01693685827463-2d8752fd/data/asdl123dasas.json.gz" + .to_owned(), + }; + let json = serde_json::to_string(&entry).unwrap(); + let parsed: ExportFileEntry = serde_json::from_str(&json).unwrap(); + assert_eq!(parsed, entry); + } + + #[test] + fn files_manifest_write_parse_round_trip() { + let entries = vec![ + ExportFileEntry { + item_count: 8, + md5_checksum: "sQMSpEILNgoQmarvDFonGQ==".to_owned(), + etag: Some("af83d6f217c19b8b0fff8023d8ca4716-1".to_owned()), + data_file_s3_key: "data/000001.json.gz".to_owned(), + }, + ExportFileEntry { + item_count: 0, + md5_checksum: "1B2M2Y8AsgTpgAmY7PhCfg==".to_owned(), + etag: Some("d41d8cd98f00b204e9800998ecf8427e".to_owned()), + data_file_s3_key: "data/000002.json.gz".to_owned(), + }, + ]; + let text = write_files_manifest(&entries).unwrap(); + assert_eq!(text.lines().count(), 2); + assert!(text.ends_with('\n')); + let parsed = parse_files_manifest(&text).unwrap(); + assert_eq!(parsed, entries); + } + + #[test] + fn files_manifest_parses_documented_example_line() { + // The files-manifest example from the documented output format, + // reflowed onto one line as the service writes it. + let line = r#"{"itemCount":8,"md5Checksum":"sQMSpEILNgoQmarvDFonGQ==","etag":"af83d6f217c19b8b0fff8023d8ca4716-1","dataFileS3Key":"AWSDynamoDB/01693685827463-2d8752fd/data/asdl123dasas.json.gz"}"#; + let parsed = parse_files_manifest(line).unwrap(); + assert_eq!(parsed.len(), 1); + assert_eq!(parsed[0].item_count, 8); + assert_eq!(parsed[0].md5_checksum, "sQMSpEILNgoQmarvDFonGQ=="); + // Writing back reproduces the line byte for byte, plus the newline. + let written = write_files_manifest(&parsed).unwrap(); + assert_eq!(written, format!("{line}\n")); + } + + #[test] + fn files_manifest_tolerates_blank_lines() { + let text = + "\n{\"itemCount\":1,\"md5Checksum\":\"m\",\"etag\":\"e\",\"dataFileS3Key\":\"k\"}\n\n"; + let parsed = parse_files_manifest(text).unwrap(); + assert_eq!(parsed.len(), 1); + } + + /// Every leading and trailing `/` of the prefix is dropped once, in one + /// place, so `s3Prefix`, `manifestFilesS3Key` and `dataFileS3Key` start + /// from the same prefix: `a//` is `a`, `/a/` is `a`, and `//` is the + /// empty prefix. + #[test] + fn prefix_with_repeated_trailing_slashes_gives_one_prefix_everywhere() { + let manifest = ExtenddbManifest::from_json_bytes(V1).unwrap(); + let entry = &manifest.data_files[0]; + for (prefix, trimmed) in [ + ("a//", "a"), + ("a/b///", "a/b"), + ("//", ""), + ("a", "a"), + ("/a/", "a"), + ("//a/b", "a/b"), + ("/", ""), + ] { + let summary = ExportSummaryManifest::for_backup(&manifest, "b", prefix).unwrap(); + let line = ExportFileEntry::from_data_file(entry, prefix, None); + assert_eq!(summary.s3_prefix, trimmed, "{prefix}"); + let (files_key, data_key) = if trimmed.is_empty() { + (FILES_MANIFEST_FILE.to_owned(), entry.key.clone()) + } else { + ( + format!("{trimmed}/{FILES_MANIFEST_FILE}"), + format!("{trimmed}/{}", entry.key), + ) + }; + assert_eq!(summary.manifest_files_s3_key, files_key, "{prefix}"); + assert_eq!(line.data_file_s3_key, data_key, "{prefix}"); + } + } +} diff --git a/crates/core/src/backup_format/documents.rs b/crates/core/src/backup_format/documents.rs new file mode 100644 index 000000000..53353f07e --- /dev/null +++ b/crates/core/src/backup_format/documents.rs @@ -0,0 +1,447 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! The documents a backup job writes besides its data files, built together +//! from one input: [`BackupDocuments::build`]. + +use super::FormatError; +use super::ddb_export::{ + ExportFileEntry, ExportSummaryManifest, trim_key_prefix, write_files_manifest, +}; +use super::layout::{ + BackupArn, EXTENDDB_MANIFEST_FILE, FILES_MANIFEST_FILE, MANIFEST_CHECKSUM_FILE, + SUMMARY_MANIFEST_FILE, backup_creation_date_time, +}; +use super::manifest::{ + CURRENT_FORMAT_VERSION, DataFileEntry, ExtenddbManifest, ItemOrder, SnapshotDescriptor, + SourceTableSchema, manifest_checksum, +}; + +/// The members of `extenddb-manifest.json` a backup job knows before it +/// writes its data files: every member except `format_version` (the +/// version this build writes), `backup_creation_date_time` (the instant in +/// the backup id), `backup_size_bytes` and `data_files` (from the files), +/// which [`BackupDocuments::build`] derives. +#[derive(Debug, Clone, PartialEq)] +pub struct ManifestHeader { + /// The backup's ARN; its backup id carries the creation instant. + pub backup_arn: BackupArn, + /// See [`ExtenddbManifest::backup_name`]. + pub backup_name: String, + /// See [`ExtenddbManifest::source_backend`]. + pub source_backend: String, + /// See [`ExtenddbManifest::source_extenddb_version`]. + pub source_extenddb_version: String, + /// See [`ExtenddbManifest::snapshot`]. + pub snapshot: SnapshotDescriptor, + /// See [`ExtenddbManifest::item_order`]. + pub item_order: ItemOrder, + /// See [`ExtenddbManifest::source_table`]. + pub source_table: SourceTableSchema, +} + +/// `extenddb-manifest.json`, its sidecar, `manifest-files.json` and +/// `manifest-summary.json` for one backup, built by [`Self::build`] from one +/// header, one list of data files with their etags and one key prefix, so +/// the four agree: one item count, one `md5` per file, one prefix for every +/// key. +/// +/// Write order: the data files first, then the three documents of +/// [`Self::documents_before_manifest`] in the order it returns them +/// (`manifest-files.json`, `manifest-summary.json`, +/// `extenddb-manifest.sha256`), then whatever the job checks before it +/// commits (that its claim on the backup still holds), then +/// [`Self::manifest_document`], `extenddb-manifest.json`, last. +/// [`Self::documents`] is the same four in that order for a caller with no +/// step between. The manifest's presence is the commit record; a prefix +/// with a manifest but no sidecar, or whose sidecar does not match, is +/// corrupt and is refused. The manifest is validated before any document is +/// built, so a job that writes in this order never stores the export +/// manifests of a manifest the reader refuses. +#[derive(Debug, Clone)] +pub struct BackupDocuments { + manifest: ExtenddbManifest, + manifest_bytes: Vec, + sidecar: String, + files_manifest: String, + summary: String, + key_prefix: String, +} + +impl BackupDocuments { + /// Build the documents of a backup whose data files are `data_files`, + /// in the order a restore applies them, each with the `ETag` the store + /// reported when the file was written under the backup's prefix (`None` + /// on a store without one). + /// + /// Derived here and nowhere else: `backup_creation_date_time` from the + /// backup id; `backup_size_bytes` as the sum of the files' `bytes`; the + /// key prefix, `store_key_prefix` (the store's configured key prefix, + /// empty when it has none; leading and trailing `/` dropped, + /// [`trim_key_prefix`]) followed by [`BackupArn::store_prefix`], which + /// is `s3Prefix` and the start of `manifestFilesS3Key` and of every + /// `dataFileS3Key`. The store normalizes its configured prefix on its + /// own (leading `/` dropped, one trailing `/` dropped, then each `/` + /// separated component checked, an empty one refused), so for every + /// prefix the store accepts the two rules give the same path and the + /// keys here are the keys the store wrote. `bucket` is the summary's + /// `s3Bucket`: the bucket name on S3, the root path on a filesystem + /// store. The manifest is validated once ([`ExtenddbManifest::validate`], + /// the reader's rule set). + /// + /// # Errors + /// + /// [`FormatError::InvalidManifest`] when the backup id carries no + /// instant (a 13-digit or bare id) or the sizes overflow, the errors of + /// [`ExtenddbManifest::validate`], and [`FormatError::Json`] when a + /// document fails to serialize. + pub fn build( + header: ManifestHeader, + data_files: Vec<(DataFileEntry, Option)>, + store_key_prefix: &str, + bucket: &str, + ) -> Result { + let (data_files, etags): (Vec, Vec>) = + data_files.into_iter().unzip(); + let ManifestHeader { + backup_arn, + backup_name, + source_backend, + source_extenddb_version, + snapshot, + item_order, + source_table, + } = header; + let backup_creation_date_time = backup_creation_date_time(backup_arn.backup_id()) + .ok_or_else(|| { + FormatError::InvalidManifest(format!( + "backup id {:?} does not carry the backup's creation instant", + backup_arn.backup_id() + )) + })?; + let backup_size_bytes = data_files + .iter() + .try_fold(0u64, |sum, f| sum.checked_add(f.bytes)) + .and_then(|sum| i64::try_from(sum).ok()) + .ok_or_else(|| FormatError::InvalidManifest("data file sizes overflow".to_owned()))?; + let backup_prefix = backup_arn.store_prefix(); + let backup_prefix = trim_key_prefix(&backup_prefix); + let key_prefix = match trim_key_prefix(store_key_prefix) { + "" => backup_prefix.to_owned(), + store => format!("{store}/{backup_prefix}"), + }; + let manifest = ExtenddbManifest { + format_version: CURRENT_FORMAT_VERSION, + backup_arn: backup_arn.to_arn_string(), + backup_name, + backup_creation_date_time, + backup_size_bytes, + source_backend, + source_extenddb_version, + snapshot, + item_order, + source_table, + data_files, + }; + let manifest_bytes = manifest.to_json_bytes()?; + let sidecar = manifest_checksum(&manifest_bytes); + let lines: Vec = manifest + .data_files + .iter() + .zip(etags) + .map(|(entry, etag)| ExportFileEntry::from_data_file(entry, &key_prefix, etag)) + .collect(); + let files_manifest = write_files_manifest(&lines)?; + let summary = + ExportSummaryManifest::for_backup(&manifest, bucket, &key_prefix)?.to_json()?; + Ok(Self { + manifest, + manifest_bytes, + sidecar, + files_manifest, + summary, + key_prefix, + }) + } + + /// The manifest the documents describe. + #[must_use] + pub fn manifest(&self) -> &ExtenddbManifest { + &self.manifest + } + + /// The bytes of `extenddb-manifest.json`, which the sidecar hashes. + #[must_use] + pub fn manifest_bytes(&self) -> &[u8] { + &self.manifest_bytes + } + + /// The content of `extenddb-manifest.sha256`. + #[must_use] + pub fn sidecar(&self) -> &str { + &self.sidecar + } + + /// The content of `manifest-files.json`. + #[must_use] + pub fn files_manifest(&self) -> &str { + &self.files_manifest + } + + /// The content of `manifest-summary.json`. + #[must_use] + pub fn summary(&self) -> &str { + &self.summary + } + + /// The backup's key prefix in the store, without a leading or trailing + /// `/`: the summary's `s3Prefix`. + #[must_use] + pub fn key_prefix(&self) -> &str { + &self.key_prefix + } + + /// The three documents written before the manifest, as (name under the + /// backup prefix, bytes) in the order they are written (type docs): + /// `manifest-files.json`, `manifest-summary.json`, + /// `extenddb-manifest.sha256`. + #[must_use] + pub fn documents_before_manifest(&self) -> [(&'static str, &[u8]); 3] { + [ + (FILES_MANIFEST_FILE, self.files_manifest.as_bytes()), + (SUMMARY_MANIFEST_FILE, self.summary.as_bytes()), + (MANIFEST_CHECKSUM_FILE, self.sidecar.as_bytes()), + ] + } + + /// The document written last, `extenddb-manifest.json`, as (name under + /// the backup prefix, bytes): the commit record (type docs). + #[must_use] + pub fn manifest_document(&self) -> (&'static str, &[u8]) { + (EXTENDDB_MANIFEST_FILE, &self.manifest_bytes) + } + + /// The four documents as (name under the backup prefix, bytes), in the + /// order they are written: [`Self::documents_before_manifest`] then + /// [`Self::manifest_document`]. + #[must_use] + pub fn documents(&self) -> [(&'static str, &[u8]); 4] { + let [files, summary, sidecar] = self.documents_before_manifest(); + [files, summary, sidecar, self.manifest_document()] + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::backup_format::{ + ExportSummaryManifest, parse_files_manifest, verify_manifest_checksum, + }; + + const V1: &[u8] = include_bytes!("../../testdata/extenddb-manifest/v1.json"); + const V1_SIDECAR: &str = include_str!("../../testdata/extenddb-manifest/v1.sha256"); + + fn header(manifest: &ExtenddbManifest) -> ManifestHeader { + ManifestHeader { + backup_arn: manifest.backup_arn().unwrap(), + backup_name: manifest.backup_name.clone(), + source_backend: manifest.source_backend.clone(), + source_extenddb_version: manifest.source_extenddb_version.clone(), + snapshot: manifest.snapshot.clone(), + item_order: manifest.item_order, + source_table: manifest.source_table.clone(), + } + } + + /// `manifest`'s data files paired with `etags`, as `build` takes them. + fn with_etags( + manifest: &ExtenddbManifest, + etags: &[Option], + ) -> Vec<(DataFileEntry, Option)> { + assert_eq!(manifest.data_files.len(), etags.len()); + manifest + .data_files + .iter() + .cloned() + .zip(etags.iter().cloned()) + .collect() + } + + /// `manifest`'s data files with no etags. + fn no_etags(manifest: &ExtenddbManifest) -> Vec<(DataFileEntry, Option)> { + manifest + .data_files + .iter() + .cloned() + .map(|entry| (entry, None)) + .collect() + } + + /// Built from the golden manifest's header and files, the manifest is + /// the fixture's bytes and the sidecar its sidecar, and the two export + /// manifests agree with it and with each other: one prefix, derived + /// from the ARN, for every key; one `md5` and item count per file, each + /// etag on its file. + #[test] + fn the_four_documents_agree() { + let golden = ExtenddbManifest::from_json_bytes(V1).unwrap(); + let etags = [Some("etag-1".to_owned()), None]; + let docs = BackupDocuments::build( + header(&golden), + with_etags(&golden, &etags), + "root/", + "backups-bucket", + ) + .unwrap(); + assert_eq!(docs.manifest_bytes(), V1); + assert_eq!(docs.sidecar(), V1_SIDECAR); + assert_eq!(docs.manifest(), &golden); + verify_manifest_checksum(docs.manifest_bytes(), docs.sidecar().as_bytes()).unwrap(); + let prefix = "root/123456789012/Music/01757500000123-0badcafe"; + assert_eq!(docs.key_prefix(), prefix); + + let lines = parse_files_manifest(docs.files_manifest()).unwrap(); + assert_eq!(lines.len(), golden.data_files.len()); + for ((line, entry), etag) in lines.iter().zip(&golden.data_files).zip(&etags) { + assert_eq!(line.item_count, entry.item_count); + assert_eq!(line.md5_checksum, entry.md5); + assert_eq!(&line.etag, etag); + assert_eq!(line.data_file_s3_key, format!("{prefix}/{}", entry.key)); + } + let summary = ExportSummaryManifest::from_json(docs.summary()).unwrap(); + assert_eq!(summary.export_arn, golden.backup_arn); + assert_eq!(summary.s3_bucket, "backups-bucket"); + assert_eq!(summary.s3_prefix, prefix); + assert_eq!( + summary.manifest_files_s3_key, + format!("{prefix}/{FILES_MANIFEST_FILE}") + ); + assert_eq!(summary.billed_size_bytes, golden.backup_size_bytes); + assert_eq!( + u64::try_from(summary.item_count).unwrap(), + lines.iter().map(|l| l.item_count).sum::() + ); + assert_eq!(summary.start_time, "2025-09-10T10:26:40.123Z"); + + // The write order: the three documents, then the manifest, and the + // four together are the same in that order. + let before: Vec<&str> = docs + .documents_before_manifest() + .iter() + .map(|(name, _)| *name) + .collect(); + assert_eq!( + before, + [ + FILES_MANIFEST_FILE, + SUMMARY_MANIFEST_FILE, + MANIFEST_CHECKSUM_FILE + ] + ); + assert_eq!(docs.manifest_document(), (EXTENDDB_MANIFEST_FILE, V1)); + let all: Vec<(&str, &[u8])> = docs + .documents_before_manifest() + .into_iter() + .chain([docs.manifest_document()]) + .collect(); + assert_eq!(docs.documents().to_vec(), all); + assert_eq!(docs.documents()[2].1, V1_SIDECAR.as_bytes()); + } + + /// The summary's `itemCount` is the sum of the files' counts, not the + /// source table's `item_count`, which is the catalog's figure and can + /// differ from the snapshot's. + #[test] + fn summary_item_count_is_the_sum_of_the_files() { + let golden = ExtenddbManifest::from_json_bytes(V1).unwrap(); + let sum: u64 = golden.data_files.iter().map(|f| f.item_count).sum(); + let mut header = header(&golden); + header.source_table.item_count = i64::try_from(sum).unwrap() + 7; + let docs = BackupDocuments::build(header, no_etags(&golden), "", "b").unwrap(); + let summary = ExportSummaryManifest::from_json(docs.summary()).unwrap(); + assert_eq!(u64::try_from(summary.item_count).unwrap(), sum); + assert_ne!(summary.item_count, docs.manifest().source_table.item_count); + } + + /// The prefix is one input: the store's key prefix in any spelling + /// gives one key prefix, and every key of both export manifests starts + /// with it. Leading `/` are dropped as trailing ones are, so a + /// configured `/root/` gives the keys the store writes under `root/`. + #[test] + fn one_prefix_for_every_key() { + let golden = ExtenddbManifest::from_json_bytes(V1).unwrap(); + let backup = "123456789012/Music/01757500000123-0badcafe"; + for (store, expected) in [ + ("", backup.to_owned()), + ("/", backup.to_owned()), + ("//", backup.to_owned()), + ("root", format!("root/{backup}")), + ("root//", format!("root/{backup}")), + ("/root/", format!("root/{backup}")), + ("/a", format!("a/{backup}")), + ("//a//", format!("a/{backup}")), + ("a/b/", format!("a/b/{backup}")), + ("/a/b", format!("a/b/{backup}")), + ] { + let docs = + BackupDocuments::build(header(&golden), no_etags(&golden), store, "b").unwrap(); + assert_eq!(docs.key_prefix(), expected, "{store:?}"); + let summary = ExportSummaryManifest::from_json(docs.summary()).unwrap(); + assert_eq!(summary.s3_prefix, expected); + assert!( + summary + .manifest_files_s3_key + .starts_with(&format!("{expected}/")) + ); + for line in parse_files_manifest(docs.files_manifest()).unwrap() { + assert!( + line.data_file_s3_key + .starts_with(&format!("{expected}/data/")), + "{store:?}: {}", + line.data_file_s3_key + ); + } + } + } + + /// What the derived fields cannot be: a backup id without an instant + /// and a file list the reader refuses are each refused before any + /// document exists, and the size sum is refused where it leaves `i64`, + /// not only where it leaves `u64`. + #[test] + fn inputs_the_reader_would_refuse_build_nothing() { + let golden = ExtenddbManifest::from_json_bytes(V1).unwrap(); + let mut thirteen = header(&golden); + thirteen.backup_arn = BackupArn::new( + "aws", + "us-east-1", + "123456789012", + "Music", + "1757500000123-0badcafe", + ) + .unwrap(); + let err = BackupDocuments::build(thirteen, no_etags(&golden), "", "b").unwrap_err(); + assert!( + matches!(&err, FormatError::InvalidManifest(m) if m.contains("instant")), + "{err}" + ); + // Files out of order: the second file's range before the first's. + let mut files = no_etags(&golden); + files.reverse(); + assert!(BackupDocuments::build(header(&golden), files, "", "b").is_err()); + // A size total that overflows `u64`, and one that is `2^63`: in + // `u64` but not in `i64`, so the sum itself is refused, not the + // manifest it would have made. + for (first, second) in [(u64::MAX, golden.data_files[1].bytes), (1 << 63, 0)] { + let mut files = no_etags(&golden); + files[0].0.bytes = first; + files[1].0.bytes = second; + let err = BackupDocuments::build(header(&golden), files, "", "b").unwrap_err(); + assert!( + matches!(&err, FormatError::InvalidManifest(m) + if m == "data file sizes overflow"), + "{first} + {second}: {err}" + ); + } + } +} diff --git a/crates/core/src/backup_format/error.rs b/crates/core/src/backup_format/error.rs new file mode 100644 index 000000000..0f1d3d32d --- /dev/null +++ b/crates/core/src/backup_format/error.rs @@ -0,0 +1,335 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! Error type for backup format encoding, decoding, and verification. +//! +//! Every message that echoes an untrusted field (a manifest member, a data +//! file key, an attribute name or value, serde's own text about the input) +//! carries at most a bounded slice of it: [`truncated`] for fields this module +//! formats, and [`JsonError`]'s `Display` and `Debug` for serde's messages, +//! which quote the input (an unknown `item_order` variant, a duplicate set +//! member list) without a bound of their own. + +use std::fmt::Write as _; + +use thiserror::Error; + +/// Longest form of one untrusted field an error message carries: the bytes +/// of its `{:?}` rendering, escapes included and the two quotes not. +pub(super) const SHOWN_FIELD_BYTES: usize = 80; + +/// The prefix kept of a field longer than [`SHOWN_FIELD_BYTES`], in the same +/// measure; the rest is replaced by its length, so the `{:?}` rendering of +/// the result stays within `SHOWN_FIELD_BYTES + 2` bytes. +const SHOWN_PREFIX_BYTES: usize = 48; + +/// Longest serde message [`FormatError::Json`] displays, before its line and +/// column. +const JSON_MESSAGE_BYTES: usize = 120; + +/// An untrusted string for an error message: the string itself when an +/// upper bound of its `{:?}` rendering is at most [`SHOWN_FIELD_BYTES`] +/// bytes, otherwise a prefix and the total length +/// (`data/00000000... <2000010 bytes>`). Call sites format the result with +/// `{:?}`. The rendering is measured, not the raw length, because escaping +/// expands a control character up to tenfold (`\u{7f}` for one byte). The +/// measure is each character's `char::escape_debug`, which also escapes `'` +/// where `str`'s `Debug` does not, so a string of apostrophes is cut at 80 +/// bytes although it would render in fewer. +pub(super) fn truncated(s: &str) -> String { + let mut shown = 0; + let mut prefix = 0; + for c in s.chars() { + // `char::escape_debug` escapes at least what `str`'s `Debug` does + // (it also escapes `'`), so this is an upper bound; it is measured + // in bytes, since an unescaped character can take up to four. + shown += c.escape_debug().map(char::len_utf8).sum::(); + if shown > SHOWN_FIELD_BYTES { + return format!("{}... <{} bytes>", &s[..prefix], s.len()); + } + if shown <= SHOWN_PREFIX_BYTES { + prefix += c.len_utf8(); + } + } + s.to_owned() +} + +/// A message that quotes untrusted text without escaping it (serde's): at +/// most `max` bytes, control characters escaped, the remainder replaced by +/// its length. +fn bounded_text(s: &str, max: usize) -> String { + let mut out = String::new(); + for c in s.chars() { + let before = out.len(); + if c.is_control() { + let _ = write!(out, "{}", c.escape_debug()); + } else { + out.push(c); + } + if out.len() > max { + out.truncate(before); + let _ = write!(out, "... <{} bytes>", s.len()); + break; + } + } + out +} + +/// A `serde_json` error with its message bounded: the payload of +/// [`FormatError::Json`]. +/// +/// serde's message quotes the input without a bound, so neither `Display` +/// nor `Debug` prints it whole: both carry at most [`JSON_MESSAGE_BYTES`] of +/// it, control characters escaped, then the position. The inner error is not +/// exposed (nor as `source()`), so a reporter that walks the chain or +/// debug-prints the error (`unwrap`, `expect`, a `tracing` `?err` field) +/// cannot print it whole either; [`Self::line`] and [`Self::column`] give the +/// position. +/// +/// An error in a data-file line names the line: [`Self::data_file_line`] is +/// its 1-based number in the file, and serde's position is within that line +/// (each line is parsed on its own). serde takes a position from the end of +/// a message raised inside the deserializer (` at line 3 column 4`), and a +/// type descriptor or attribute name quoted at the end of such a message can +/// carry that text, so serde's line is not always 1. The column is shown, +/// as `at column N of the line`, only when serde's line is 1; otherwise the +/// message is shown as serde rendered it, with no position of this type's. +pub struct JsonError { + inner: serde_json::Error, + data_file_line: Option, +} + +impl JsonError { + /// An error in line `line` (1-based) of a data file. + pub(super) fn on_data_file_line(inner: serde_json::Error, line: u64) -> Self { + Self { + inner, + data_file_line: Some(line), + } + } + + /// serde's 1-based line within the document, or 0 when serde gave no + /// position. Within a data-file line it is 1 unless the message carried + /// a position of its own (type docs). + #[must_use] + pub fn line(&self) -> usize { + self.inner.line() + } + + /// serde's 1-based column within the line, or 0 when serde gave no + /// position. + #[must_use] + pub fn column(&self) -> usize { + self.inner.column() + } + + /// The 1-based number of the data-file line that failed to parse, or + /// `None` for a document that is not a data file (a manifest). + #[must_use] + pub fn data_file_line(&self) -> Option { + self.data_file_line + } + + /// serde's message bounded, without the position unless `keep_position`. + fn bounded_message(&self, keep_position: bool) -> String { + let full = self.inner.to_string(); + let message = if self.inner.line() == 0 || keep_position { + full.as_str() + } else { + let position = format!(" at line {} column {}", self.line(), self.column()); + full.strip_suffix(&position).unwrap_or(&full) + }; + bounded_text(message, JSON_MESSAGE_BYTES) + } +} + +impl From for JsonError { + fn from(inner: serde_json::Error) -> Self { + Self { + inner, + data_file_line: None, + } + } +} + +impl std::fmt::Display for JsonError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match (self.data_file_line, self.line()) { + (None, 0) => write!(f, "{}", self.bounded_message(false)), + (None, line) => write!( + f, + "{} at line {line} column {}", + self.bounded_message(false), + self.column() + ), + (Some(n), 1) => write!( + f, + "data file line {n}: {} at column {} of the line", + self.bounded_message(false), + self.column() + ), + (Some(n), _) => write!(f, "data file line {n}: {}", self.bounded_message(true)), + } + } +} + +impl std::fmt::Debug for JsonError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("JsonError") + .field("message", &self.bounded_message(false)) + .field("line", &self.line()) + .field("column", &self.column()) + .field("data_file_line", &self.data_file_line) + .finish_non_exhaustive() + } +} + +/// Errors produced while reading, writing, or verifying the backup format. +#[derive(Debug, Error)] +pub enum FormatError { + /// The manifest declares a format version this build does not read: + /// 0, or later than `supported`. A build reads every version from 1 to + /// the one it writes. + #[error( + "unsupported backup format version {found}; this build reads format versions 1 to {supported}" + )] + UnsupportedFormatVersion { + /// Version found in the manifest. + found: u32, + /// The latest version this build reads (`CURRENT_FORMAT_VERSION`); + /// it reads every version from 1 to this one. + supported: u32, + }, + + /// A checksum did not match the expected value. + #[error("{subject} checksum mismatch: expected {expected:?}, computed {computed}")] + ChecksumMismatch { + /// What was checksummed, for the error message. + subject: &'static str, + /// The recorded checksum, cut by [`truncated`] where it is untrusted; + /// displayed with `{:?}`, so a control character in it is escaped. + expected: String, + /// The checksum computed from the bytes at hand. + computed: String, + }, + + /// The number of items decoded differs from the number recorded. + #[error("item count mismatch: expected {expected}, decoded {actual}")] + ItemCountMismatch { + /// The recorded item count. + expected: u64, + /// The number of items actually decoded. + actual: u64, + }, + + /// The uncompressed byte count differs from the number recorded. + #[error("uncompressed byte count mismatch: expected {expected}, decoded {actual}")] + ByteCountMismatch { + /// The recorded uncompressed byte count. + expected: u64, + /// The number of bytes actually decoded. + actual: u64, + }, + + /// The number of compressed bytes read from a data file differs from the + /// size recorded for it. Raised as soon as the reader has consumed more + /// than the recorded size (then `actual` is the count when it stopped, + /// one more than `expected`), and at end of stream when it consumed + /// fewer. + #[error("compressed size mismatch: expected {expected} bytes, read {actual}")] + CompressedSizeMismatch { + /// The recorded compressed size. + expected: u64, + /// The number of compressed bytes read. + actual: u64, + }, + + /// A backup ARN did not have the expected shape. + #[error("invalid backup ARN: {0}")] + InvalidArn(String), + + /// JSON serialization or deserialization failed: a manifest that does + /// not parse, or a data file line that is not one `{"Item": ...}` object + /// (the message then names the line). + /// + /// The message carries serde's text cut to a bound (serde quotes the + /// input, unbounded), in `Display` and `Debug` alike; see [`JsonError`]. + /// The manifest's nested members reuse the request wire types, whose + /// enum deserializers word their errors as the API does, so a bad value + /// in one of them reads like a request validation error (`1 validation + /// error detected: Value 'MANHATTAN' at 'distanceFunction' failed to + /// satisfy constraint ...`, `'keySchema.1.member.keyType'`): request-path + /// text about a stored document, in a restore log. Decoupling the types + /// would change the format's authority; the text is left as is. + #[error("JSON error: {0}")] + Json(JsonError), + + /// A single decompressed data-file line exceeded the maximum allowed + /// length, so decoding stopped before materializing it in full. + #[error( + "data file line exceeded the {cap}-byte maximum length; \ + refusing to decompress further to bound memory" + )] + LineTooLong { + /// The maximum line length, in bytes. + cap: usize, + }, + + /// A `DataFileEntry.key` did not have the required + /// `data/.json.gz` shape. + #[error("invalid data file key {key:?}: {reason}")] + InvalidDataFileKey { + /// The offending key. + key: String, + /// Why the key was rejected. + reason: &'static str, + }, + + /// A `DataFileEntry` other than its key is malformed: a digest of the + /// wrong shape, a zero size or item count, or a `key_range` missing from + /// a key-ordered manifest or present in an unordered one. + #[error("invalid data file entry {key:?}: {reason}")] + InvalidDataFileEntry { + /// The entry's key. + key: String, + /// Why the entry was rejected. + reason: &'static str, + }, + + /// A key-ordered data file's items do not ascend strictly (refused by + /// the writer's `KeyRangeTracker` and the reader's `KeyRangeCheck`), or + /// do not match the file's `key_range`: the first or last item is not + /// the recorded endpoint (refused by `KeyRangeCheck`). + #[error("data file items break key order: {0}")] + KeyOrderViolation(String), + + /// Writing to the gzip encoder failed. + #[error("data file encode error: {0}")] + Encode(String), + + /// Gzip decompression failed or the stream was malformed. + #[error("data file decode error: {0}")] + Decode(String), + + /// The key schema has no order the format can define (multi-part keys, + /// or a key attribute without a definition). + #[error("key schema has no format-defined order: {0}")] + UnorderableKeySchema(String), + + /// An item's key attributes do not match the key schema. + #[error("invalid key: {0}")] + InvalidKey(String), + + /// A manifest is inconsistent: its identity fields disagree, its data + /// file list has overlapping or unsorted key ranges or a size total that + /// does not match, or its checksum sidecar is not in the one accepted + /// shape. + #[error("invalid manifest: {0}")] + InvalidManifest(String), +} + +impl From for FormatError { + fn from(e: serde_json::Error) -> Self { + Self::Json(e.into()) + } +} diff --git a/crates/core/src/backup_format/golden_tests.rs b/crates/core/src/backup_format/golden_tests.rs new file mode 100644 index 000000000..068a13804 --- /dev/null +++ b/crates/core/src/backup_format/golden_tests.rs @@ -0,0 +1,1411 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! Golden-file tests against the checked-in `DynamoDB` export under +//! `testdata/ddb-export/` and the `ExtendDB` manifest under +//! `testdata/extenddb-manifest/`. +//! +//! The export fixture is constructed, not captured from the live service: it +//! follows the documented S3 export output format (member names, the +//! `2020-06-30` summary version, timestamps, the explicit `null` for +//! `s3SseKmsKeyId`) and was generated with this crate's own encoder, so what +//! it proves is that the reader accepts a document of that shape and that the +//! writer's output does not drift, not that it matches bytes the service +//! wrote. The README in that directory records the details. + +use std::path::PathBuf; + +use super::ddb_export::write_files_manifest; +use super::{DataFileExpectations, ExportSummaryManifest, decode_data_file, parse_files_manifest}; +use crate::types::AttributeValue; + +fn fixture_dir() -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("testdata/ddb-export") +} + +fn read_fixture(name: &str) -> Vec { + let path = fixture_dir().join(name); + std::fs::read(&path).unwrap_or_else(|e| panic!("cannot read {}: {e}", path.display())) +} + +#[test] +fn golden_summary_manifest_parses() { + let raw = String::from_utf8(read_fixture("manifest-summary.json")).unwrap(); + let summary = ExportSummaryManifest::from_json(&raw).unwrap(); + assert_eq!(summary.version, "2020-06-30"); + assert_eq!( + summary.export_arn, + "arn:aws:dynamodb:us-east-1:123456789012:table/ProductCatalog/export/01693685827463-2d8752fd" + ); + assert_eq!(summary.table_id, "12345a12-abcd-123a-ab12-1234abc12345"); + assert_eq!(summary.s3_sse_algorithm.as_deref(), Some("AES256")); + assert_eq!(summary.s3_sse_kms_key_id, None); + assert_eq!(summary.item_count, 3); + assert_eq!(summary.output_format, "DYNAMODB_JSON"); + assert_eq!(summary.export_type, None); + + // The serializer reproduces the document byte for byte. + assert_eq!(summary.to_json().unwrap(), raw); +} + +#[test] +fn golden_files_manifest_parses_and_reserializes_byte_for_byte() { + let raw = String::from_utf8(read_fixture("manifest-files.json")).unwrap(); + let entries = parse_files_manifest(&raw).unwrap(); + assert_eq!(entries.len(), 1); + assert_eq!(entries[0].item_count, 3); + assert_eq!(entries[0].md5_checksum, "1jUj2wzOC+iuxJN+LKy6Aw=="); + assert_eq!( + entries[0].data_file_s3_key, + "AWSDynamoDB/01693685827463-2d8752fd/data/ka2sswm5ha4uejfqcmnjcbg6ru.json.gz" + ); + + assert_eq!(write_files_manifest(&entries).unwrap(), raw); +} + +#[test] +fn golden_data_file_decodes_with_manifest_checksum() { + let files_raw = String::from_utf8(read_fixture("manifest-files.json")).unwrap(); + let entry = &parse_files_manifest(&files_raw).unwrap()[0]; + + let compressed = read_fixture("data/ka2sswm5ha4uejfqcmnjcbg6ru.json.gz"); + let expectations = DataFileExpectations { + md5_base64: Some(entry.md5_checksum.clone()), + item_count: Some(entry.item_count), + ..Default::default() + }; + let items = decode_data_file(&compressed, expectations).unwrap(); + assert_eq!(items.len(), 3); + + assert_eq!( + items[0].get("Id"), + Some(&AttributeValue::N("103".to_owned())) + ); + assert_eq!( + items[0].get("Title"), + Some(&AttributeValue::S("Book 103 Title".to_owned())) + ); + assert_eq!( + items[0].get("Authors"), + Some(&AttributeValue::SS( + ["Author1", "Author2"] + .iter() + .map(|s| (*s).to_owned()) + .collect() + )) + ); + assert_eq!( + items[0].get("InPublication"), + Some(&AttributeValue::Bool(false)) + ); + assert_eq!( + items[1].get("Binary"), + Some(&AttributeValue::B(vec![0xde, 0xad, 0xbe, 0xef])) + ); + match items[2].get("Meta") { + Some(AttributeValue::M(map)) => { + assert_eq!(map.get("a"), Some(&AttributeValue::Null)); + assert_eq!( + map.get("list"), + Some(&AttributeValue::L(vec![ + AttributeValue::N("1".to_owned()), + AttributeValue::S("x".to_owned()), + ])) + ); + } + other => panic!("unexpected Meta value: {other:?}"), + } +} + +#[test] +fn golden_data_file_reencodes_to_identical_lines() { + // Decoding the fixture and re-encoding its items must reproduce the same + // uncompressed lines: the fixture's line encoding and the encoder's agree + // byte for byte. (The gzip container itself is not compared, since + // compressor output is implementation-defined.) + use std::io::Read; + + let compressed = read_fixture("data/ka2sswm5ha4uejfqcmnjcbg6ru.json.gz"); + let items = decode_data_file(&compressed, DataFileExpectations::default()).unwrap(); + + let (reencoded, checksums) = super::encode_data_file(items).unwrap(); + assert_eq!(checksums.item_count, 3); + + let mut fixture_lines = String::new(); + flate2::read::GzDecoder::new(compressed.as_slice()) + .read_to_string(&mut fixture_lines) + .unwrap(); + let mut reencoded_lines = String::new(); + flate2::read::GzDecoder::new(reencoded.as_slice()) + .read_to_string(&mut reencoded_lines) + .unwrap(); + assert_eq!(reencoded_lines, fixture_lines); + assert_eq!(checksums.uncompressed_bytes, fixture_lines.len() as u64); +} + +/// The frozen `extenddb-manifest.json` for format version 1, three times: +/// `v1.json` with every optional member set and every list non-empty, +/// `v1-minimal.json` with every optional member absent (written as `null`) +/// and every list empty, and `v1-unordered.json` with the shapes the other +/// two lack (an unordered backup's `"key_range":null` files, and the absent +/// state of every optional nested member `v1.json` sets: `OnDemandThroughput` +/// with both limits `null`, an `SseDescription` without `SSEType` and +/// `KMSMasterKeyArn`, a GSI without `ProvisionedThroughput`, a +/// `StreamSpecification` without `StreamViewType`, a `time_to_live` without +/// `AttributeName`; and a vector index's `SearchSchema` and `Projection`, an +/// LSI's `INCLUDE` projection with `NonKeyAttributes`, which `v1.json` +/// lacks). The nested members reuse the `DynamoDB` +/// wire types, so a change to one of those types (a rename, a +/// `skip_serializing_if` toggle, a new required member) changes the on-disk +/// format without any other test noticing; these fixtures make such a +/// change fail here, in the present and in the absent state. A new optional +/// member that is omitted when absent leaves every fixture's bytes as they +/// were, and so does a new or renamed enum value no fixture uses; +/// `nested_types_have_the_pinned_members` and `enum_values_are_pinned` fail +/// those instead. They are +/// permanent read tests: every later build parses them, and they are never +/// regenerated once v1 is released (see the testdata README and +/// `ExtenddbManifest`'s evolution rule). +mod extenddb_manifest_golden { + use super::super::manifest::manifest_checksum; + use super::super::{ExtenddbManifest, ItemOrder, verify_manifest_checksum}; + use crate::types::{ProjectionType, SearchSchemaElementType, TimeToLiveStatus}; + + const MANIFEST: &[u8] = include_bytes!("../../testdata/extenddb-manifest/v1.json"); + const SIDECAR: &str = include_str!("../../testdata/extenddb-manifest/v1.sha256"); + const MINIMAL: &[u8] = include_bytes!("../../testdata/extenddb-manifest/v1-minimal.json"); + const MINIMAL_SIDECAR: &str = + include_str!("../../testdata/extenddb-manifest/v1-minimal.sha256"); + const UNORDERED: &[u8] = include_bytes!("../../testdata/extenddb-manifest/v1-unordered.json"); + const UNORDERED_SIDECAR: &str = + include_str!("../../testdata/extenddb-manifest/v1-unordered.sha256"); + + #[test] + fn golden_unordered_manifest_parses_and_reserializes_byte_for_byte() { + let manifest = + ExtenddbManifest::from_json_bytes(UNORDERED).expect("unordered golden parses"); + let again = manifest.to_json_bytes().unwrap(); + assert_eq!( + std::str::from_utf8(&again).unwrap(), + std::str::from_utf8(UNORDERED).unwrap(), + "the on-disk shape of format version 1 changed; bump the version" + ); + verify_manifest_checksum(UNORDERED, UNORDERED_SIDECAR.as_bytes()).unwrap(); + assert_eq!(manifest_checksum(UNORDERED), UNORDERED_SIDECAR); + } + + #[test] + fn golden_unordered_manifest_has_the_shapes_the_others_lack() { + let manifest = ExtenddbManifest::from_json_bytes(UNORDERED).unwrap(); + assert_eq!(manifest.item_order, ItemOrder::Unordered); + assert!(manifest.data_files.len() > 1); + assert!(manifest.data_files.iter().all(|f| f.key_range.is_none())); + let t = &manifest.source_table; + let on_demand = t.on_demand_throughput.as_ref().unwrap(); + assert!(on_demand.max_read_request_units.is_none()); + assert!(on_demand.max_write_request_units.is_none()); + let sse = t.sse_description.as_ref().unwrap(); + assert!(sse.sse_type.is_none()); + assert!(sse.kms_master_key_arn.is_none()); + let gsi = &t.global_secondary_indexes[0]; + assert!(gsi.provisioned_throughput.is_none()); + let vector = &t.vector_indexes.as_ref().unwrap()[0]; + let schema = vector.search_schema.as_ref().unwrap(); + assert!( + schema + .iter() + .any(|e| e.element_type == SearchSchemaElementType::Hash) + ); + assert!( + schema + .iter() + .any(|e| e.element_type == SearchSchemaElementType::InlineFilter) + ); + assert!( + vector + .projection + .as_ref() + .unwrap() + .non_key_attributes + .is_some() + ); + let lsi = &t.local_secondary_indexes[0]; + assert_eq!(lsi.projection.projection_type, ProjectionType::Include); + assert!(lsi.projection.non_key_attributes.is_some()); + let stream = t.stream_specification.as_ref().unwrap(); + assert!(!stream.stream_enabled); + assert!(stream.stream_view_type.is_none()); + let ttl = t.time_to_live.as_ref().unwrap(); + assert_eq!(ttl.time_to_live_status, TimeToLiveStatus::Disabled); + assert!(ttl.attribute_name.is_none()); + // The members are in the bytes as written: the nested wire types' + // absent members as `null` (`OnDemandThroughput`) or omitted (the + // others). + let text = std::str::from_utf8(UNORDERED).unwrap(); + for member in [ + "\"item_order\":\"unordered\"", + "\"key_range\":null", + "\"on_demand_throughput\":{\"MaxReadRequestUnits\":null,\"MaxWriteRequestUnits\":null}", + "\"sse_description\":{\"Status\":\"DISABLED\"}", + "\"Projection\":{\"ProjectionType\":\"KEYS_ONLY\"}}]", + "\"SearchSchema\":[", + "\"SearchSchemaElementType\":\"HASH\"", + "\"SearchSchemaElementType\":\"INLINE_FILTER\"", + "\"NonKeyAttributes\":[", + "\"ProjectionType\":\"INCLUDE\"", + "\"provisioned_throughput\":null", + "\"stream_specification\":{\"StreamEnabled\":false}", + // The wire type omits an absent `AttributeName`. + "\"time_to_live\":{\"TimeToLiveStatus\":\"DISABLED\"}", + ] { + assert!(text.contains(member), "{member}"); + } + } + + #[test] + fn golden_minimal_manifest_parses_and_reserializes_byte_for_byte() { + let manifest = ExtenddbManifest::from_json_bytes(MINIMAL).expect("minimal golden parses"); + let again = manifest.to_json_bytes().unwrap(); + assert_eq!( + std::str::from_utf8(&again).unwrap(), + std::str::from_utf8(MINIMAL).unwrap(), + "the on-disk shape of format version 1 with absent members changed; bump the version" + ); + verify_manifest_checksum(MINIMAL, MINIMAL_SIDECAR.as_bytes()).unwrap(); + assert_eq!(manifest_checksum(MINIMAL), MINIMAL_SIDECAR); + } + + #[test] + fn golden_minimal_manifest_has_every_optional_member_absent() { + let manifest = ExtenddbManifest::from_json_bytes(MINIMAL).unwrap(); + let t = &manifest.source_table; + assert!(t.provisioned_throughput.is_none()); + assert!(t.on_demand_throughput.is_none()); + assert!(t.table_class.is_none()); + assert!(t.sse_description.is_none()); + assert!(t.vector_indexes.is_none()); + assert!(t.stream_specification.is_none()); + assert!(t.time_to_live.is_none()); + assert!(t.global_secondary_indexes.is_empty()); + assert!(t.local_secondary_indexes.is_empty()); + assert!(manifest.data_files.is_empty()); + // The members are in the bytes as `null` / `[]`, not omitted. + let text = std::str::from_utf8(MINIMAL).unwrap(); + for member in [ + "\"provisioned_throughput\":null", + "\"on_demand_throughput\":null", + "\"table_class\":null", + "\"sse_description\":null", + "\"vector_indexes\":null", + "\"stream_specification\":null", + "\"time_to_live\":null", + "\"global_secondary_indexes\":[]", + "\"local_secondary_indexes\":[]", + "\"data_files\":[]", + ] { + assert!(text.contains(member), "{member}"); + } + } + + #[test] + fn golden_manifest_parses_and_reserializes_byte_for_byte() { + let manifest = ExtenddbManifest::from_json_bytes(MANIFEST).expect("golden parses"); + let again = manifest.to_json_bytes().unwrap(); + assert_eq!( + std::str::from_utf8(&again).unwrap(), + std::str::from_utf8(MANIFEST).unwrap(), + "the on-disk shape of format version 1 changed; bump the version" + ); + } + + #[test] + fn golden_sidecar_verifies() { + verify_manifest_checksum(MANIFEST, SIDECAR.as_bytes()).unwrap(); + assert_eq!(manifest_checksum(MANIFEST), SIDECAR); + } + + #[test] + fn golden_manifest_covers_every_optional_member() { + // The fixture sets every top-level optional member of the source + // table so a serialization change to any of them is caught. Nested + // optionals inside the index and vector shapes (an LSI's + // `NonKeyAttributes`, the vector index's `SearchSchema` and + // `Projection`) are absent here and present in `v1-unordered.json`, + // so a rename of any of them fails that fixture; the nested + // optionals set here (`MaxReadRequestUnits`, `MaxWriteRequestUnits`, + // `SSEType`, `KMSMasterKeyArn`, a GSI's `ProvisionedThroughput`, + // `StreamViewType`, `AttributeName`) are absent there. + let manifest = ExtenddbManifest::from_json_bytes(MANIFEST).unwrap(); + let t = &manifest.source_table; + assert!(t.provisioned_throughput.is_some()); + assert!(t.on_demand_throughput.is_some()); + assert!(t.table_class.is_some()); + assert!(t.sse_description.is_some()); + assert!(!t.global_secondary_indexes.is_empty()); + assert!(!t.local_secondary_indexes.is_empty()); + assert!(t.vector_indexes.as_ref().is_some_and(|v| !v.is_empty())); + let stream = t.stream_specification.as_ref().unwrap(); + assert!(stream.stream_enabled); + assert!(stream.stream_view_type.is_some()); + let ttl = t.time_to_live.as_ref().unwrap(); + assert_eq!(ttl.time_to_live_status, TimeToLiveStatus::Enabled); + assert!(ttl.attribute_name.is_some()); + assert!(manifest.data_files.iter().all(|f| f.key_range.is_some())); + let text = std::str::from_utf8(MANIFEST).unwrap(); + for member in [ + "\"stream_specification\":{\"StreamEnabled\":true,\"StreamViewType\":\"NEW_AND_OLD_IMAGES\"}", + "\"time_to_live\":{\"TimeToLiveStatus\":\"ENABLED\",\"AttributeName\":\"expires_at\"}", + ] { + assert!(text.contains(member), "{member}"); + } + } + + /// Unknown members are tolerated at every level of the document, not + /// only at the top: a later writer can add a member to the top level, + /// to `snapshot`, to `source_table`, to any nested wire type (a key + /// schema element, an attribute definition, a throughput, an index, a + /// projection, a vector index and its search schema, the stream and TTL + /// descriptions), to a `data_files[]` entry and to a `key_range`, and + /// this reader parses the document to the same value. Every JSON object + /// in the two fixtures that cover all of those shapes gets one injected + /// member in turn, except the key items under `key_range.first` and + /// `.last`, where a new name is a new key attribute (refused as a wrong + /// key shape, correctly) and a new descriptor is a second type (refused + /// by the `AttributeValue` deserializer, correctly). + #[test] + fn unknown_members_are_tolerated_at_every_level() { + fn objects(value: &serde_json::Value, path: &str, out: &mut Vec) { + match value { + serde_json::Value::Object(map) => { + out.push(path.to_owned()); + for (k, v) in map { + objects(v, &format!("{path}/{k}"), out); + } + } + serde_json::Value::Array(items) => { + for (i, v) in items.iter().enumerate() { + objects(v, &format!("{path}/{i}"), out); + } + } + _ => {} + } + } + fn inject(value: &mut serde_json::Value, path: &str) { + let mut cursor = value; + for segment in path.split('/').filter(|s| !s.is_empty()) { + cursor = match cursor { + serde_json::Value::Object(map) => map.get_mut(segment).unwrap(), + serde_json::Value::Array(items) => { + &mut items[segment.parse::().unwrap()] + } + other => panic!("{path}: {other}"), + }; + } + cursor + .as_object_mut() + .unwrap() + .insert("added_in_a_later_version".to_owned(), serde_json::json!(42)); + } + let mut injected = 0; + for fixture in [MANIFEST, UNORDERED] { + let original = ExtenddbManifest::from_json_bytes(fixture).unwrap(); + let doc: serde_json::Value = serde_json::from_slice(fixture).unwrap(); + let mut paths = Vec::new(); + objects(&doc, "", &mut paths); + for path in paths { + if path.contains("/key_range/first") || path.contains("/key_range/last") { + continue; + } + let mut doc = doc.clone(); + inject(&mut doc, &path); + let parsed = ExtenddbManifest::from_json_bytes(&serde_json::to_vec(&doc).unwrap()) + .unwrap_or_else(|e| panic!("unknown member at {path:?}: {e}")); + assert_eq!(parsed, original, "unknown member at {path:?}"); + injected += 1; + } + } + // The levels the contract names are among those visited. + let covered = |needle: &str| { + let doc: serde_json::Value = serde_json::from_slice(MANIFEST).unwrap(); + let mut paths = Vec::new(); + objects(&doc, "", &mut paths); + let doc: serde_json::Value = serde_json::from_slice(UNORDERED).unwrap(); + objects(&doc, "", &mut paths); + paths.iter().any(|p| p.ends_with(needle)) + }; + for level in [ + "", + "/snapshot", + "/source_table", + "/source_table/key_schema/0", + "/source_table/attribute_definitions/0", + "/source_table/provisioned_throughput", + "/source_table/on_demand_throughput", + "/source_table/sse_description", + "/source_table/global_secondary_indexes/0", + "/source_table/global_secondary_indexes/0/Projection", + "/source_table/global_secondary_indexes/0/ProvisionedThroughput", + "/source_table/local_secondary_indexes/0", + "/source_table/local_secondary_indexes/0/Projection", + "/source_table/vector_indexes/0", + "/source_table/vector_indexes/0/VectorAttribute", + "/source_table/vector_indexes/0/SearchSchema/0", + "/source_table/vector_indexes/0/Projection", + "/source_table/stream_specification", + "/source_table/time_to_live", + "/data_files/0", + "/data_files/0/key_range", + ] { + assert!(covered(level), "no object at {level:?} in the fixtures"); + } + assert!(injected >= 30, "{injected}"); + } +} + +#[cfg(test)] +mod generate_golden_manifest { + #![allow(clippy::too_many_lines)] + //! Run with `EXTENDDB_WRITE_GOLDEN=1 cargo test -p extenddb-core + //! write_golden_manifest -- --ignored` to write the fixtures of the + //! current format version, `v{CURRENT_FORMAT_VERSION}*.json`. Once a + //! version is released its fixtures are never rewritten; a version bump + //! writes new ones beside them. + use super::super::manifest::manifest_checksum; + use super::super::{ + CURRENT_FORMAT_VERSION, DataFileEntry, ExtenddbManifest, ItemOrder, KeyRange, + SnapshotDescriptor, SourceTableSchema, + }; + use crate::types::{ + AttributeDefinition, AttributeValue, BillingMode, DistanceFunction, GsiInput, Item, + KeySchemaElement, KeyType, LsiInput, OnDemandThroughput, Projection, ProjectionType, + ProvisionedThroughput, ScalarAttributeType, SearchSchemaElement, SearchSchemaElementType, + SseDescription, SseType, StreamSpecification, StreamViewType, TimeToLiveDescription, + TimeToLiveStatus, VectorAttribute, VectorIndexSpecification, + }; + + fn key(a: &str, t: &str) -> Item { + [ + ("Artist".to_owned(), AttributeValue::S(a.to_owned())), + ("SongTitle".to_owned(), AttributeValue::S(t.to_owned())), + ] + .into_iter() + .collect() + } + + pub(super) fn golden() -> ExtenddbManifest { + ExtenddbManifest { + format_version: CURRENT_FORMAT_VERSION, + backup_arn: + "arn:aws:dynamodb:us-east-1:123456789012:table/Music/backup/01757500000123-0badcafe" + .to_owned(), + backup_name: "music-nightly".to_owned(), + backup_creation_date_time: 1_757_500_000.123, + backup_size_bytes: 61_728 + 100, + source_backend: "postgres".to_owned(), + source_extenddb_version: "0.1.13".to_owned(), + snapshot: SnapshotDescriptor { + kind: "postgres_repeatable_read".to_owned(), + marker: "1234:1234:".to_owned(), + }, + item_order: ItemOrder::KeyOrdered, + source_table: SourceTableSchema { + table_name: "Music".to_owned(), + table_id: "12345a12-abcd-123a-ab12-1234abc12345".to_owned(), + table_arn: "arn:aws:dynamodb:us-east-1:123456789012:table/Music".to_owned(), + table_creation_date_time: 1_757_400_000.0, + key_schema: vec![ + KeySchemaElement { + attribute_name: "Artist".to_owned(), + key_type: KeyType::Hash, + }, + KeySchemaElement { + attribute_name: "SongTitle".to_owned(), + key_type: KeyType::Range, + }, + ], + attribute_definitions: vec![ + AttributeDefinition { + attribute_name: "Artist".to_owned(), + attribute_type: ScalarAttributeType::S, + }, + AttributeDefinition { + attribute_name: "SongTitle".to_owned(), + attribute_type: ScalarAttributeType::S, + }, + AttributeDefinition { + attribute_name: "Genre".to_owned(), + attribute_type: ScalarAttributeType::S, + }, + ], + billing_mode: BillingMode::Provisioned, + provisioned_throughput: Some(ProvisionedThroughput { + read_capacity_units: 5, + write_capacity_units: 5, + }), + on_demand_throughput: Some(OnDemandThroughput { + max_read_request_units: Some(100), + max_write_request_units: Some(50), + }), + table_class: Some("STANDARD".to_owned()), + sse_description: Some(SseDescription { + status: "ENABLED".to_owned(), + sse_type: Some(SseType::KMS), + kms_master_key_arn: Some( + "arn:aws:kms:us-east-1:123456789012:key/abcd".to_owned(), + ), + }), + global_secondary_indexes: vec![GsiInput { + index_name: "GenreIndex".to_owned(), + key_schema: vec![KeySchemaElement { + attribute_name: "Genre".to_owned(), + key_type: KeyType::Hash, + }], + projection: Projection { + projection_type: ProjectionType::Include, + non_key_attributes: Some(vec!["Year".to_owned()]), + }, + provisioned_throughput: Some(ProvisionedThroughput { + read_capacity_units: 1, + write_capacity_units: 1, + }), + }], + local_secondary_indexes: vec![LsiInput { + index_name: "TitleIndex".to_owned(), + key_schema: vec![ + KeySchemaElement { + attribute_name: "Artist".to_owned(), + key_type: KeyType::Hash, + }, + KeySchemaElement { + attribute_name: "SongTitle".to_owned(), + key_type: KeyType::Range, + }, + ], + projection: Projection { + projection_type: ProjectionType::KeysOnly, + non_key_attributes: None, + }, + }], + vector_indexes: Some(vec![VectorIndexSpecification { + index_name: "embeddings".to_owned(), + dimensions: 1024, + distance_function: DistanceFunction::Cosine, + vector_attribute: VectorAttribute { + attribute_name: "vec".to_owned(), + }, + search_schema: None, + projection: None, + }]), + stream_specification: Some(StreamSpecification { + stream_enabled: true, + stream_view_type: Some(StreamViewType::NewAndOldImages), + }), + time_to_live: Some(TimeToLiveDescription { + time_to_live_status: TimeToLiveStatus::Enabled, + attribute_name: Some("expires_at".to_owned()), + }), + item_count: 100_000, + table_size_bytes: 123_456, + }, + data_files: vec![ + DataFileEntry { + key: "data/000001.json.gz".to_owned(), + item_count: 50_000, + sha256: "0".repeat(64), + md5: "1B2M2Y8AsgTpgAmY7PhCfg==".to_owned(), + bytes: 61_728, + uncompressed_bytes: 300_000, + key_range: Some(KeyRange { + first: key("A", "a"), + last: key("M", "z"), + }), + }, + DataFileEntry { + key: "data/000002.json.gz".to_owned(), + item_count: 50_000, + sha256: "1".repeat(64), + md5: "1B2M2Y8AsgTpgAmY7PhCfg==".to_owned(), + bytes: 100, + uncompressed_bytes: 300_000, + key_range: Some(KeyRange { + first: key("N", "a"), + last: key("Z", "z"), + }), + }, + ], + } + } + + /// The minimal manifest: an empty hash-only table with every optional + /// member absent and every list empty. + fn minimal() -> ExtenddbManifest { + ExtenddbManifest { + format_version: CURRENT_FORMAT_VERSION, + backup_arn: + "arn:aws:dynamodb:us-east-1:123456789012:table/Empty/backup/01757500000123-0badcafe" + .to_owned(), + backup_name: "empty".to_owned(), + backup_creation_date_time: 1_757_500_000.123, + backup_size_bytes: 0, + source_backend: "sqlite".to_owned(), + source_extenddb_version: "0.1.13".to_owned(), + snapshot: SnapshotDescriptor { + kind: "sqlite_wal_read_transaction".to_owned(), + marker: String::new(), + }, + item_order: ItemOrder::Unordered, + source_table: SourceTableSchema { + table_name: "Empty".to_owned(), + table_id: "00000000-0000-4000-8000-000000000000".to_owned(), + table_arn: "arn:aws:dynamodb:us-east-1:123456789012:table/Empty".to_owned(), + table_creation_date_time: 1_757_400_000.0, + key_schema: vec![KeySchemaElement { + attribute_name: "pk".to_owned(), + key_type: KeyType::Hash, + }], + attribute_definitions: vec![AttributeDefinition { + attribute_name: "pk".to_owned(), + attribute_type: ScalarAttributeType::S, + }], + billing_mode: BillingMode::PayPerRequest, + provisioned_throughput: None, + on_demand_throughput: None, + table_class: None, + sse_description: None, + global_secondary_indexes: vec![], + local_secondary_indexes: vec![], + vector_indexes: None, + stream_specification: None, + time_to_live: None, + item_count: 0, + table_size_bytes: 0, + }, + data_files: vec![], + } + } + + /// An unordered backup (MongoDB) of an on-demand table: data files + /// without key ranges, an `OnDemandThroughput` with both limits absent, + /// an `SseDescription` without `SSEType` or `KMSMasterKeyArn`, a GSI + /// without `ProvisionedThroughput`, a vector index with a search schema + /// and a projection, an LSI with an `INCLUDE` projection, a disabled + /// stream without `StreamViewType` and a disabled TTL without + /// `AttributeName`: the absent states of the nested members `v1.json` + /// sets. + pub(super) fn unordered() -> ExtenddbManifest { + let element = |name: &str, key_type| KeySchemaElement { + attribute_name: name.to_owned(), + key_type, + }; + let file = |index: u32, bytes: u64| DataFileEntry { + key: crate::backup_format::data_file_key(index), + item_count: 1_000, + sha256: "2".repeat(64), + md5: "1B2M2Y8AsgTpgAmY7PhCfg==".to_owned(), + bytes, + uncompressed_bytes: 64_000, + key_range: None, + }; + ExtenddbManifest { + format_version: CURRENT_FORMAT_VERSION, + backup_arn: + "arn:aws:dynamodb:eu-west-1:123456789012:table/Events/backup/01757500000123-0badcafe" + .to_owned(), + backup_name: "events-weekly".to_owned(), + backup_creation_date_time: 1_757_500_000.123, + backup_size_bytes: 9_000 + 8_000, + source_backend: "mongodb".to_owned(), + source_extenddb_version: "0.1.14".to_owned(), + snapshot: SnapshotDescriptor { + kind: "mongodb_snapshot_read_concern".to_owned(), + marker: "1757500000:7".to_owned(), + }, + item_order: ItemOrder::Unordered, + source_table: SourceTableSchema { + table_name: "Events".to_owned(), + table_id: "6f1c2a0e-1111-4222-8333-444455556666".to_owned(), + table_arn: "arn:aws:dynamodb:eu-west-1:123456789012:table/Events".to_owned(), + table_creation_date_time: 1_757_400_000.5, + key_schema: vec![ + element("tenant", KeyType::Hash), + element("at", KeyType::Range), + ], + attribute_definitions: vec![ + AttributeDefinition { + attribute_name: "tenant".to_owned(), + attribute_type: ScalarAttributeType::S, + }, + AttributeDefinition { + attribute_name: "at".to_owned(), + attribute_type: ScalarAttributeType::N, + }, + AttributeDefinition { + attribute_name: "kind".to_owned(), + attribute_type: ScalarAttributeType::S, + }, + ], + billing_mode: BillingMode::PayPerRequest, + provisioned_throughput: None, + // Both limits absent: the wire type writes each as `null`. + on_demand_throughput: Some(OnDemandThroughput { + max_read_request_units: None, + max_write_request_units: None, + }), + table_class: Some("STANDARD_INFREQUENT_ACCESS".to_owned()), + // A description without `SSEType` or `KMSMasterKeyArn`: the + // wire type omits both. + sse_description: Some(SseDescription { + status: "DISABLED".to_owned(), + sse_type: None, + kms_master_key_arn: None, + }), + // The GSI of an on-demand table has no `ProvisionedThroughput`. + global_secondary_indexes: vec![GsiInput { + index_name: "ByKindGlobal".to_owned(), + key_schema: vec![element("kind", KeyType::Hash)], + projection: Projection { + projection_type: ProjectionType::KeysOnly, + non_key_attributes: None, + }, + provisioned_throughput: None, + }], + local_secondary_indexes: vec![LsiInput { + index_name: "ByKind".to_owned(), + key_schema: vec![ + element("tenant", KeyType::Hash), + element("kind", KeyType::Range), + ], + projection: Projection { + projection_type: ProjectionType::Include, + non_key_attributes: Some(vec!["payload".to_owned(), "at".to_owned()]), + }, + }], + vector_indexes: Some(vec![VectorIndexSpecification { + index_name: "similar".to_owned(), + dimensions: 384, + distance_function: DistanceFunction::Euclidean, + vector_attribute: VectorAttribute { + attribute_name: "embedding".to_owned(), + }, + search_schema: Some(vec![ + SearchSchemaElement { + attribute_name: "tenant".to_owned(), + element_type: SearchSchemaElementType::Hash, + }, + SearchSchemaElement { + attribute_name: "kind".to_owned(), + element_type: SearchSchemaElementType::InlineFilter, + }, + ]), + projection: Some(Projection { + projection_type: ProjectionType::Include, + non_key_attributes: Some(vec!["payload".to_owned()]), + }), + }]), + // A disabled stream, as `DescribeTable` reports it: no + // `StreamViewType`. + stream_specification: Some(StreamSpecification { + stream_enabled: false, + stream_view_type: None, + }), + // TTL reported without an attribute name, as `DescribeTimeToLive` + // reports a table whose TTL is disabled. + time_to_live: Some(TimeToLiveDescription { + time_to_live_status: TimeToLiveStatus::Disabled, + attribute_name: None, + }), + item_count: 2_000, + table_size_bytes: 128_000, + }, + data_files: vec![file(1, 9_000), file(2, 8_000)], + } + } + + /// Writes the current version's fixtures from `golden()`, `minimal()` + /// and `unordered()`. Ignored by default so an ordinary `cargo test` + /// never writes checked-in files; run it with + /// `EXTENDDB_WRITE_GOLDEN=1 cargo test -p extenddb-core write_golden_manifest -- --ignored` + /// after a version bump, then review the diff. It refuses to overwrite a + /// fixture that exists: a released version's fixtures are never + /// rewritten, only added beside (`ExtenddbManifest`'s evolution rule). + /// Before format version 1 is released, a change to its shape is made + /// by writing into another directory (`EXTENDDB_WRITE_GOLDEN=`; the checked-in files are `include_bytes!`'d, so they + /// cannot be deleted first) and moving the files over the old ones. + #[test] + #[ignore = "writes the checked-in fixtures; run explicitly"] + fn write_golden_manifest() { + use std::io::Write as _; + // `--ignored` and `--include-ignored` would otherwise write the + // checked-in fixtures as a side effect of an unrelated run. + let Some(target) = std::env::var_os("EXTENDDB_WRITE_GOLDEN") else { + eprintln!("set EXTENDDB_WRITE_GOLDEN=1 to write the fixtures"); + return; + }; + let target = std::path::PathBuf::from(target); + let dir = if target.is_dir() { + target + } else { + std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("testdata/extenddb-manifest") + }; + let v = format!("v{CURRENT_FORMAT_VERSION}"); + for (name, manifest) in [ + (v.clone(), golden()), + (format!("{v}-minimal"), minimal()), + (format!("{v}-unordered"), unordered()), + ] { + let bytes = manifest.to_json_bytes().unwrap(); + for (path, content) in [ + (dir.join(format!("{name}.json")), bytes.clone()), + ( + dir.join(format!("{name}.sha256")), + manifest_checksum(&bytes).into_bytes(), + ), + ] { + // `create_new` fails if the file exists: the fixtures of a + // version are written once. + let mut file = std::fs::File::create_new(&path).unwrap_or_else(|e| { + panic!( + "refusing to write {}: {e}; a released version's fixtures are never \ + rewritten (write into another directory with \ + EXTENDDB_WRITE_GOLDEN= and move the files over only if version \ + {CURRENT_FORMAT_VERSION} has not been released)", + path.display() + ) + }); + file.write_all(&content).unwrap(); + } + } + } + + /// The writer's constructors and the checked-in fixtures of the current + /// version are two copies of one shape; this binds them, so an edit to + /// `golden()`, `minimal()` or `unordered()` that the writer would turn + /// into different fixture bytes fails here first. Written against + /// version 1; when the version is bumped the three constructors move to + /// the new version's shape and the comparison moves with them. + #[test] + fn writer_constructors_match_the_checked_in_fixtures() { + const V1: &[u8] = include_bytes!("../../testdata/extenddb-manifest/v1.json"); + const V1_MINIMAL: &[u8] = + include_bytes!("../../testdata/extenddb-manifest/v1-minimal.json"); + const V1_UNORDERED: &[u8] = + include_bytes!("../../testdata/extenddb-manifest/v1-unordered.json"); + assert_eq!( + CURRENT_FORMAT_VERSION, 1, + "the constructors now describe version {CURRENT_FORMAT_VERSION}: compare them with \ + its fixtures here and keep the v1 files as parse tests" + ); + for (name, manifest, fixture) in [ + ("v1.json", golden(), V1), + ("v1-minimal.json", minimal(), V1_MINIMAL), + ("v1-unordered.json", unordered(), V1_UNORDERED), + ] { + let bytes = manifest.to_json_bytes().unwrap(); + assert_eq!( + std::str::from_utf8(&bytes).unwrap(), + std::str::from_utf8(fixture).unwrap(), + "{name}: the writer's constructor and the checked-in fixture differ" + ); + } + } +} + +/// The members each type of the manifest writes, and the values each enum +/// in it reads and writes, pinned per type. The golden fixtures pin the +/// bytes of their inputs, but a new optional member that is omitted when +/// absent (`skip_serializing_if`) leaves those bytes unchanged, and an enum +/// value no fixture uses (a renamed `AES256`, a new `TimeToLiveStatus`) +/// changes no fixture either; this test fails both, until the list here is +/// changed, which is a format version bump (`ExtenddbManifest`'s evolution +/// rule). +#[cfg(test)] +mod member_names { + use serde::de::{self, DeserializeOwned, Deserializer, Visitor}; + use serde::{Serialize, forward_to_deserialize_any}; + + use super::super::{ + DataFileEntry, ExtenddbManifest, ItemOrder, KeyRange, SnapshotDescriptor, SourceTableSchema, + }; + use super::generate_golden_manifest::{golden, unordered}; + use crate::types::{ + AttributeDefinition, AttributeValue, BillingMode, DistanceFunction, GsiInput, + KeySchemaElement, KeyType, LsiInput, OnDemandThroughput, Projection, ProjectionType, + ProvisionedThroughput, ScalarAttributeType, SearchSchemaElement, SearchSchemaElementType, + SseDescription, SseType, StreamSpecification, StreamViewType, TimeToLiveDescription, + TimeToLiveStatus, VectorAttribute, VectorIndexSpecification, + }; + + /// A deserializer that records what a derived `Deserialize` passes to + /// `deserialize_struct` (the member names: every member the type reads, + /// whether or not it is written when absent) or to `deserialize_enum` + /// (the variant names it accepts), and reads nothing. + #[derive(Default)] + struct FieldsProbe { + fields: Option<&'static [&'static str]>, + variants: Option<&'static [&'static str]>, + } + + impl<'de> Deserializer<'de> for &mut FieldsProbe { + type Error = de::value::Error; + fn deserialize_any>(self, _: V) -> Result { + Err(de::Error::custom("not a struct or an enum")) + } + fn deserialize_struct>( + self, + _name: &'static str, + fields: &'static [&'static str], + _visitor: V, + ) -> Result { + self.fields = Some(fields); + Err(de::Error::custom("probe")) + } + fn deserialize_enum>( + self, + _name: &'static str, + variants: &'static [&'static str], + _visitor: V, + ) -> Result { + self.variants = Some(variants); + Err(de::Error::custom("probe")) + } + forward_to_deserialize_any! { + bool i8 i16 i32 i64 i128 u8 u16 u32 u64 u128 f32 f64 char str string + bytes byte_buf option unit unit_struct newtype_struct seq tuple + tuple_struct map identifier ignored_any + } + } + + fn fields_read() -> Vec<&'static str> { + let mut probe = FieldsProbe::default(); + let _ = T::deserialize(&mut probe); + probe + .fields + .expect("a derived struct deserializer") + .to_vec() + } + + /// A derived enum reads exactly the values `expected`, and each of them + /// parses and writes back as itself. + fn check_enum(name: &str, expected: &[&str]) { + let mut probe = FieldsProbe::default(); + let _ = T::deserialize(&mut probe); + let variants = probe.variants.expect("a derived enum deserializer"); + assert_eq!( + variants, expected, + "{name}: the values read changed; that is a format version bump" + ); + for wire in expected { + let value: T = serde_json::from_value(serde_json::json!(wire)) + .unwrap_or_else(|e| panic!("{name}: {wire:?} does not parse: {e}")); + assert_eq!( + serde_json::to_value(&value).unwrap(), + serde_json::json!(wire), + "{name}: {wire:?} is written differently" + ); + } + } + + /// An enum with a hand-written `Deserialize`, which the probe cannot + /// see: every row's value writes as its wire string and the string + /// parses as the value, the strings are distinct, and a spelling in + /// another case is refused. The rows come from an exhaustive `match` + /// with no wildcard at the call site, so a new variant does not compile + /// until it has a row. + fn check_wire_table(name: &str, rows: &[(T, &str)]) + where + T: Serialize + DeserializeOwned + PartialEq + std::fmt::Debug, + { + let mut seen = std::collections::BTreeSet::new(); + for (value, wire) in rows { + assert!(seen.insert(*wire), "{name}: {wire:?} is listed twice"); + assert_eq!( + serde_json::to_value(value).unwrap(), + serde_json::json!(wire), + "{name}: {value:?} is written differently; that is a format version bump" + ); + let parsed: T = serde_json::from_value(serde_json::json!(wire)) + .unwrap_or_else(|e| panic!("{name}: {wire:?} does not parse: {e}")); + assert_eq!(&parsed, value, "{name}: {wire:?} parses as another value"); + let other_case = wire.to_lowercase(); + if other_case != *wire { + assert!( + serde_json::from_value::(serde_json::json!(other_case)).is_err(), + "{name}: {other_case:?} is accepted" + ); + } + } + } + + /// `AttributeValue`, whose wire form is one object with the type + /// descriptor as its one member (`{"S":"a"}`), not a string: every + /// row's value writes as an object whose one member is its descriptor + /// and reads back as itself, the descriptors are distinct, a descriptor + /// in lower case is refused, and the descriptors the reader accepts, + /// among every string of one to four ASCII upper-case letters (the + /// alphabet and the lengths of `DynamoDB`'s descriptors), are exactly + /// the rows'. The rows come from an exhaustive `match` with no wildcard + /// at the call site, so a new variant does not compile until it has a + /// row, and a descriptor the reader starts to accept fails the last + /// check. + fn check_descriptor_table(name: &str, rows: &[(AttributeValue, &str)]) { + let mut descriptors = std::collections::BTreeSet::new(); + for (value, descriptor) in rows { + assert!( + descriptors.insert(*descriptor), + "{name}: {descriptor:?} is listed twice" + ); + let written = serde_json::to_value(value).unwrap(); + let members: Vec<&String> = written.as_object().unwrap().keys().collect(); + assert_eq!( + members, + [descriptor], + "{name}: {value:?} is written under another descriptor; that is a format version bump" + ); + let parsed: AttributeValue = serde_json::from_value(written.clone()) + .unwrap_or_else(|e| panic!("{name}: {written} does not parse: {e}")); + assert_eq!(&parsed, value, "{name}: {written} parses as another value"); + let other_case = serde_json::json!({ descriptor.to_lowercase(): written[*descriptor] }); + assert!( + serde_json::from_value::(other_case.clone()).is_err(), + "{name}: {other_case} is accepted" + ); + } + // A descriptor the reader knows fails on its payload or parses; + // one it does not know fails on the descriptor, whatever the + // payload. + let unknown = |candidate: &str| { + serde_json::from_value::(serde_json::json!({ candidate: null })) + .err() + .is_some_and(|e| { + e.to_string() + .starts_with("unknown AttributeValue type descriptor") + }) + }; + let mut accepted = std::collections::BTreeSet::new(); + let mut candidate = String::new(); + for length in 1..=4 { + let mut digits = vec![0u8; length]; + loop { + candidate.clear(); + candidate.extend(digits.iter().map(|d| char::from(b'A' + d))); + if !unknown(&candidate) { + accepted.insert(candidate.clone()); + } + let mut i = 0; + while i < length && digits[i] == 25 { + digits[i] = 0; + i += 1; + } + if i == length { + break; + } + digits[i] += 1; + } + } + let accepted: Vec<&str> = accepted.iter().map(String::as_str).collect(); + let rows: Vec<&str> = descriptors.into_iter().collect(); + assert_eq!( + accepted, rows, + "{name}: the descriptors read changed; that is a format version bump" + ); + } + + /// The enums the manifest carries, each with its wire values. `ItemOrder` + /// and the derived wire enums are read by the probe; `KeyType`, + /// `ScalarAttributeType` and `DistanceFunction` have hand-written + /// `Deserialize` impls and are pinned by tables, and so is + /// `AttributeValue`, the value type of every `KeyRange` endpoint and of + /// every data file line, with its ten type descriptors. Every enum-typed + /// member of the types `nested_types_have_the_pinned_members` lists is + /// here (`SseDescription.Status` and `table_class` are strings). + #[test] + fn enum_values_are_pinned() { + check_enum::("ItemOrder", &["key_ordered", "unordered"]); + check_enum::("BillingMode", &["PROVISIONED", "PAY_PER_REQUEST"]); + check_enum::("ProjectionType", &["ALL", "KEYS_ONLY", "INCLUDE"]); + check_enum::( + "StreamViewType", + &["KEYS_ONLY", "NEW_IMAGE", "OLD_IMAGE", "NEW_AND_OLD_IMAGES"], + ); + check_enum::("SseType", &["AES256", "KMS"]); + check_enum::( + "TimeToLiveStatus", + &["ENABLED", "ENABLING", "DISABLED", "DISABLING"], + ); + check_enum::( + "SearchSchemaElementType", + &["HASH", "INLINE_FILTER"], + ); + + let key_type = |v: KeyType| match v { + KeyType::Hash => "HASH", + KeyType::Range => "RANGE", + }; + check_wire_table( + "KeyType", + &[KeyType::Hash, KeyType::Range].map(|v| (v, key_type(v))), + ); + let scalar = |v: ScalarAttributeType| match v { + ScalarAttributeType::S => "S", + ScalarAttributeType::N => "N", + ScalarAttributeType::B => "B", + }; + check_wire_table( + "ScalarAttributeType", + &[ + ScalarAttributeType::S, + ScalarAttributeType::N, + ScalarAttributeType::B, + ] + .map(|v| (v, scalar(v))), + ); + let distance = |v: DistanceFunction| match v { + DistanceFunction::Cosine => "COSINE", + DistanceFunction::Euclidean => "EUCLIDEAN", + DistanceFunction::DotProduct => "DOT_PRODUCT", + }; + check_wire_table( + "DistanceFunction", + &[ + DistanceFunction::Cosine, + DistanceFunction::Euclidean, + DistanceFunction::DotProduct, + ] + .map(|v| (v, distance(v))), + ); + let descriptor = |v: &AttributeValue| match v { + AttributeValue::S(_) => "S", + AttributeValue::N(_) => "N", + AttributeValue::B(_) => "B", + AttributeValue::SS(_) => "SS", + AttributeValue::NS(_) => "NS", + AttributeValue::BS(_) => "BS", + AttributeValue::Bool(_) => "BOOL", + AttributeValue::Null => "NULL", + AttributeValue::L(_) => "L", + AttributeValue::M(_) => "M", + }; + let rows: Vec<(AttributeValue, &str)> = [ + AttributeValue::S("a".to_owned()), + AttributeValue::N("1".to_owned()), + AttributeValue::B(vec![0xde, 0xad]), + AttributeValue::SS(["a".to_owned()].into_iter().collect()), + AttributeValue::NS(["1".to_owned()].into_iter().collect()), + AttributeValue::BS([vec![0xde]].into_iter().collect()), + AttributeValue::Bool(true), + AttributeValue::Null, + AttributeValue::L(vec![AttributeValue::N("1".to_owned())]), + AttributeValue::M( + [("k".to_owned(), AttributeValue::S("v".to_owned()))] + .into_iter() + .collect(), + ), + ] + .into_iter() + .map(|v| { + let d = descriptor(&v); + (v, d) + }) + .collect(); + check_descriptor_table("AttributeValue", &rows); + // A value outside each table is refused: the hand-written readers + // have no fallback. + assert!(serde_json::from_value::(serde_json::json!("SORT")).is_err()); + assert!(serde_json::from_value::(serde_json::json!("SS")).is_err()); + assert!( + serde_json::from_value::(serde_json::json!("MANHATTAN")).is_err() + ); + } + + /// The top-level member names of `value`'s JSON, in the order written, + /// each with whether its value is `null`. + fn members_written(value: &T) -> Vec<(String, bool)> { + struct Keys; + impl<'de> Visitor<'de> for Keys { + type Value = Vec<(String, bool)>; + fn expecting(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str("an object") + } + fn visit_map>(self, mut map: A) -> Result { + let mut out = Vec::new(); + while let Some(key) = map.next_key::()? { + let value: serde_json::Value = map.next_value()?; + out.push((key, value.is_null())); + } + Ok(out) + } + } + let text = serde_json::to_string(value).unwrap(); + serde_json::Deserializer::from_str(&text) + .deserialize_map(Keys) + .unwrap() + } + + /// `value` has every member set: it writes exactly `expected`, in that + /// order, none of them `null`; and the type reads exactly `expected`. + fn check(name: &str, value: &T, expected: &[&str]) { + let written = members_written(value); + let names: Vec<&str> = written.iter().map(|(n, _)| n.as_str()).collect(); + assert_eq!( + names, expected, + "{name}: the members written changed; that is a format version bump" + ); + for (member, null) in &written { + assert!(!null, "{name}.{member} is not set in the value checked"); + } + assert_eq!( + fields_read::(), + expected, + "{name}: the members read changed; that is a format version bump" + ); + } + + #[test] + #[allow(clippy::too_many_lines)] // one pinned list per type + fn nested_types_have_the_pinned_members() { + let full = golden(); + let other = unordered(); + let t = &full.source_table; + let file = &full.data_files[0]; + let vector = &other.source_table.vector_indexes.as_ref().unwrap()[0]; + check::( + "ExtenddbManifest", + &full, + &[ + "format_version", + "backup_arn", + "backup_name", + "backup_creation_date_time", + "backup_size_bytes", + "source_backend", + "source_extenddb_version", + "snapshot", + "item_order", + "source_table", + "data_files", + ], + ); + check::("SnapshotDescriptor", &full.snapshot, &["kind", "marker"]); + check::( + "SourceTableSchema", + t, + &[ + "table_name", + "table_id", + "table_arn", + "table_creation_date_time", + "key_schema", + "attribute_definitions", + "billing_mode", + "provisioned_throughput", + "on_demand_throughput", + "table_class", + "sse_description", + "global_secondary_indexes", + "local_secondary_indexes", + "vector_indexes", + "stream_specification", + "time_to_live", + "item_count", + "table_size_bytes", + ], + ); + check::( + "DataFileEntry", + file, + &[ + "key", + "item_count", + "sha256", + "md5", + "bytes", + "uncompressed_bytes", + "key_range", + ], + ); + check::( + "KeyRange", + file.key_range.as_ref().unwrap(), + &["first", "last"], + ); + check::( + "KeySchemaElement", + &t.key_schema[0], + &["AttributeName", "KeyType"], + ); + check::( + "AttributeDefinition", + &t.attribute_definitions[0], + &["AttributeName", "AttributeType"], + ); + check::( + "ProvisionedThroughput", + t.provisioned_throughput.as_ref().unwrap(), + &["ReadCapacityUnits", "WriteCapacityUnits"], + ); + check::( + "OnDemandThroughput", + t.on_demand_throughput.as_ref().unwrap(), + &["MaxReadRequestUnits", "MaxWriteRequestUnits"], + ); + check::( + "SseDescription", + t.sse_description.as_ref().unwrap(), + &["Status", "SSEType", "KMSMasterKeyArn"], + ); + let gsi = &t.global_secondary_indexes[0]; + check::( + "GsiInput", + gsi, + &[ + "IndexName", + "KeySchema", + "Projection", + "ProvisionedThroughput", + ], + ); + check::( + "Projection", + &gsi.projection, + &["ProjectionType", "NonKeyAttributes"], + ); + check::( + "LsiInput", + &other.source_table.local_secondary_indexes[0], + &["IndexName", "KeySchema", "Projection"], + ); + check::( + "VectorIndexSpecification", + vector, + &[ + "IndexName", + "Dimensions", + "DistanceFunction", + "VectorAttribute", + "SearchSchema", + "Projection", + ], + ); + check::( + "VectorAttribute", + &vector.vector_attribute, + &["AttributeName"], + ); + check::( + "SearchSchemaElement", + &vector.search_schema.as_ref().unwrap()[0], + &["AttributeName", "SearchSchemaElementType"], + ); + check::( + "StreamSpecification", + t.stream_specification.as_ref().unwrap(), + &["StreamEnabled", "StreamViewType"], + ); + check::( + "TimeToLiveDescription", + t.time_to_live.as_ref().unwrap(), + &["TimeToLiveStatus", "AttributeName"], + ); + } +} diff --git a/crates/core/src/backup_format/key_order.rs b/crates/core/src/backup_format/key_order.rs new file mode 100644 index 000000000..b0060e30c --- /dev/null +++ b/crates/core/src/backup_format/key_order.rs @@ -0,0 +1,1219 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! The one definition of base-table key order for the backup format. +//! +//! A `key_ordered` backup promises its items ascend in this order within each +//! data file and from one file to the next: +//! +//! - Hash key: the text encoding [`hash_key_text`] produces (`S` as is, `N` +//! as the string held, `B` as standard base64 with `=` padding), compared +//! as UTF-8 bytes. A number hash key therefore sorts as text (`"10"` +//! before `"9"`), and a binary one by its base64 text, not its bytes +//! (`[0xff]` is `/w==` and sorts before `[0x00]`, `AA==`); this is how the +//! storage layer keys rows and the format inherits it rather than +//! defining a second order. The base64 text of a `B` hash key is the very +//! string that appears in the JSON of a `key_range` endpoint or a data +//! file line (`{"B":"/w=="}`), so a reader compares what it sees. +//! +//! [`KeyOrder::compare`] takes an `N` value as written; it does not +//! normalize. The wire `AttributeValue` deserializer rewrites every valid +//! number it parses to its normalized form (`5.0` to `5`, `1.5E3` to +//! `1500`), which covers request items and data file lines alike, so a +//! stored valid number holds the normalized string. It keeps an invalid +//! number as written, and whether that reaches a backend depends on the +//! operation. `PutItem` refuses it: its validation runs the item number +//! check (`validate_item_numbers`) before a backend stores the item. +//! `BatchWriteItem` and `TransactWriteItems` do not, and store an invalid +//! `N` key value as written (`1E126`, `abc` and `1E-131` are stored and +//! returned by `Scan` as written; #618). `UpdateItem` on an absent item, +//! which creates the item from its `Key`, and `ImportTable` do not run +//! that check on the key either; `UpdateItem` cannot change the key of an +//! existing item, and a restore validates every item. So a table can hold +//! an `N` key that [`KeyOrder::validate_key_item`] refuses. +//! +//! A key-ordered writer that meets such a stored key refuses the backup +//! with an error naming the key. The format cannot carry the key as a +//! `key_range` endpoint ([`KeyOrder::validate_key_item`] refuses it, so +//! [`super::BackupDocuments::build`] cannot write the manifest), and the +//! restore's item validation would refuse the item anyway. The writer +//! passes every item to a [`super::KeyRangeTracker`], which checks the +//! item's key as [`KeyOrder::endpoint_key`] does and refuses such a key +//! wherever it falls in a file, not only at a file's first or last item; +//! [`super::KeyRangeCheck`] is that tracker plus the recorded range, so a +//! reader refuses a key-ordered file holding such a key as well. +//! +//! A `key_range` endpoint is read differently: the manifest parser reads +//! its `N` strings as written, before the `AttributeValue` deserializer +//! would rewrite them, and refuses one not in normalized form +//! ([`super::KeyRange`]), so the stored bytes and the parsed range agree; +//! [`KeyOrder::validate_key_item`] refuses the same for an endpoint built +//! in code. A caller that builds items in code and compares them must +//! normalize its `N` values first. +//! - Range key: `S` as UTF-8 bytes, `B` as raw bytes (a shorter prefix +//! first), `N` by exact decimal value. +//! +//! What another implementation must reproduce to compute this order (the +//! Rust code is `crate::validation::number::validate_and_normalize_number` +//! and `KeyOrder::compare`; the test `doc_examples_normalize_as_stated` +//! pins the examples below): +//! +//! - `N` normalization. Accepted input: an optional `-` or `+`, digits with +//! at most one `.` and at least one digit, an optional `e` or `E` followed +//! by an optionally signed decimal exponent that fits a signed 32-bit +//! integer (`-2147483648` to `2147483647`); no whitespace, no other +//! characters. An exponent outside that range is refused whatever the +//! value would be: `0E2147483648` and `0E-2147483649` are refused, while +//! `0E2147483647` and `0.0E-2147483648` are `0`. The value must have at most 38 significant digits after +//! leading and trailing zeros are dropped and a magnitude within +//! 10^-130 to just under 10^126 (`1E-130` and `9.9E125` are accepted, +//! `1E-131` and `1E126` refused). Output: plain decimal without an +//! exponent, a `-` only for a negative value, no leading zeros except the +//! single `0` before a `.`, no trailing fractional zeros and no trailing +//! `.`, zero as `0` (`-0`, `-0.0` and `0E5` are all `0`). So `00042` is +//! `42`, `1.50` is `1.5`, `1.5E3` is `1500`, `+1` is `1`, `.5` is `0.5`, +//! `1E+125` is `1` followed by 125 zeros (126 characters), and `1E-130` is +//! `0.` followed by 129 zeros and `1` (132 characters). A hash-key `N` +//! compares as this string, bytewise, so the whole order of `N` hash keys +//! depends on these rules. +//! - `B`: standard base64 (`A`-`Z`, `a`-`z`, `0`-`9`, `+`, `/`) with `=` +//! padding in canonical form, not the URL-safe alphabet and not unpadded. +//! - `S`: compared as the UTF-8 bytes of the string, not as code points, +//! UTF-16 code units (`JavaScript`'s default string comparison) or a +//! collation; `é` sorts after `z`. +//! - `N` range keys: exact decimal comparison (`BigDecimal` here). +//! Doubles are insufficient: two keys that first differ in the 38th +//! significant digit are distinct keys in this order and equal as `f64`. +//! +//! The SQL backends can produce this order from their key columns, but only +//! with the right collation on the query, whatever the column declares: +//! +//! - SQLite: `ORDER BY pk, sk_s` (or `sk_b`) with the default `BINARY` +//! collation is memcmp; `ORDER BY pk, sk_n` is numeric because `sk_n` holds +//! `encode_orderable_number`'s order-preserving text encoding. The +//! `storage-sqlite` test `number_key_order_matches_backup_format` binds +//! that encoding to [`KeyOrder::compare`]. +//! - Postgres: `ORDER BY pk COLLATE "C", sk_s COLLATE "C"`; `sk_n` is +//! `NUMERIC` and `sk_b` is `bytea`, both already in the format's order. +//! Tables created before the key columns were declared `COLLATE "C"` keep +//! the database collation (`en_US.utf8` in the default setups, where +//! `a < B` and punctuation is weak), so a writer that omits the collation +//! on `pk` or `sk_s` emits a file that is not in this order. +//! +//! Backends that cannot produce this order write `unordered`. The manifest +//! parser binds [`KeyOrder::compare`] for its `key_range` checks, and +//! [`super::KeyRangeTracker`] (the writer's) and [`super::KeyRangeCheck`] +//! (the reader's, run by [`super::ExtenddbManifest::reader`] over each +//! file's items) compare in the same order for ascent (the tracker across +//! every file it spans, the check within its file) and the endpoints, with +//! the hash key's text bytewise and the range key parsed +//! once per item; the storage layer's +//! `pk_to_text` binds [`hash_key_text`], so for the hash key the writer, the +//! reader, and the row key agree by construction. (Other copies of the +//! `S`/`N`/`B`-to-text mapping remain, none on the backup path: hash keys +//! for stream shard assignment in `storage-postgres/src/data/tx_helpers.rs`, +//! `storage-sqlite/src/data/tx_helpers.rs`, +//! `storage-cassandra/src/stream_util.rs` and +//! `storage-mongodb/src/data_engine.rs`; sort keys in +//! `storage-mongodb/src/data/mod.rs` (`sk_to_text`, a verbatim copy of +//! [`hash_key_text`]), `storage-mongodb/src/data_engine.rs` +//! and `storage-cassandra/src/data/transactions.rs`; binary values in +//! `storage-mongodb/src/condition.rs`. Binding them is a follow-up.) +//! Range-key agreement rests on each backend's column order matching the +//! rules above: the `storage-sqlite` test named above binds `sk_n`, and the +//! storage layer's snapshot conformance check orders with +//! [`KeyOrder::compare`]. + +use std::borrow::Cow; +use std::cmp::Ordering; +use std::str::FromStr; + +use base64::Engine; +use base64::engine::general_purpose::STANDARD as BASE64; +use bigdecimal::BigDecimal; + +use super::FormatError; +use super::error::truncated; +use crate::types::{ + AttributeDefinition, AttributeValue, Item, KeySchemaElement, KeyType, ScalarAttributeType, +}; +use crate::validation::number::validate_and_normalize_number; + +/// Text encoding of a hash key value: the row key the SQL backends store. +/// `None` for a value that cannot be a hash key. +#[must_use] +pub fn hash_key_text(value: &AttributeValue) -> Option> { + match value { + AttributeValue::S(s) => Some(Cow::Borrowed(s)), + AttributeValue::N(n) => Some(Cow::Borrowed(n)), + AttributeValue::B(b) => Some(Cow::Owned(BASE64.encode(b))), + _ => None, + } +} + +/// A table's key shape, resolved once from its key schema and attribute +/// definitions, able to compare and validate items' keys. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct KeyOrder { + hash: (String, ScalarAttributeType), + range: Option<(String, ScalarAttributeType)>, +} + +impl KeyOrder { + /// Resolve the key shape. Only hash-only and single-range-key tables have + /// a format-defined order; a multi-part range key returns + /// [`FormatError::UnorderableKeySchema`], and a writer for such a table + /// writes `unordered`. + /// + /// # Errors + /// + /// [`FormatError::UnorderableKeySchema`] when the schema has no hash key, + /// more than one hash or range key, a range key listed before the hash + /// key (`DynamoDB` requires `HASH` first), a range key with the hash + /// key's name, a key attribute without a definition, or an attribute + /// defined more than once. + pub fn resolve( + key_schema: &[KeySchemaElement], + attribute_definitions: &[AttributeDefinition], + ) -> Result { + let type_of = |name: &str| { + let mut matching = attribute_definitions + .iter() + .filter(|d| d.attribute_name == name) + .map(|d| d.attribute_type); + let ty = matching.next().ok_or_else(|| { + FormatError::UnorderableKeySchema(format!( + "key attribute {:?} has no attribute definition", + truncated(name) + )) + })?; + if matching.next().is_some() { + return Err(FormatError::UnorderableKeySchema(format!( + "key attribute {:?} has more than one attribute definition", + truncated(name) + ))); + } + Ok(ty) + }; + let mut hash = None; + let mut range = None; + for element in key_schema { + if element.key_type == KeyType::Range && hash.is_none() { + return Err(FormatError::UnorderableKeySchema( + "the range key is listed before the hash key".to_owned(), + )); + } + let slot = match element.key_type { + KeyType::Hash => &mut hash, + KeyType::Range => &mut range, + }; + if slot.is_some() { + return Err(FormatError::UnorderableKeySchema( + "multi-part keys have no format-defined order".to_owned(), + )); + } + *slot = Some(( + element.attribute_name.clone(), + type_of(&element.attribute_name)?, + )); + } + let hash: (String, ScalarAttributeType) = hash.ok_or_else(|| { + FormatError::UnorderableKeySchema("key schema has no hash key".to_owned()) + })?; + if range.as_ref().is_some_and(|(name, _)| *name == hash.0) { + return Err(FormatError::UnorderableKeySchema(format!( + "the range key and the hash key are both {:?}", + truncated(&hash.0) + ))); + } + Ok(Self { hash, range }) + } + + /// Names of the key attributes, hash first. + pub fn attribute_names(&self) -> impl Iterator { + std::iter::once(self.hash.0.as_str()).chain(self.range.iter().map(|(n, _)| n.as_str())) + } + + /// Check that `key` holds exactly the key attributes with the declared + /// types and nothing else: the shape [`super::KeyRange`] carries. An `N` + /// value must be a valid `DynamoDB` number in its normalized form, and + /// an `S` or `B` value must not be empty (`DynamoDB` refuses an empty key + /// value). A parsed manifest's `N` endpoints were already checked for + /// normalized form as written ([`super::KeyRange`]), so for parsed input + /// this refuses invalid numbers (`1E200`, `1_000`, 39 significant + /// digits) and the normalized-form check guards items built in code. Key + /// lengths (2048 bytes for a hash key, 1024 for a range key) are not + /// checked here; the restore's item validation checks them. + /// + /// # Errors + /// + /// [`FormatError::InvalidKey`] naming what is wrong; for an invalid + /// number, the number validation's reason (`Number overflow. ...`). + pub fn validate_key_item(&self, key: &Item) -> Result<(), FormatError> { + let expected: usize = 1 + usize::from(self.range.is_some()); + if key.len() != expected { + return Err(FormatError::InvalidKey(format!( + "key has {} attributes, the key schema has {expected}", + key.len() + ))); + } + for (name, ty) in std::iter::once(&self.hash).chain(self.range.iter()) { + let value = key.get(name).ok_or_else(|| { + FormatError::InvalidKey(format!("key attribute {:?} missing", truncated(name))) + })?; + check_key_value(name, *ty, value)?; + } + Ok(()) + } + + /// The key attributes of `item` checked as [`Self::validate_key_item`] + /// checks a key, without building one: the values borrowed from `item`, + /// an `N` range key parsed once for comparison. Other attributes of + /// `item` are not looked at. The errors are [`Self::endpoint_key`]'s. + pub(super) fn checked_key<'a>(&self, item: &'a Item) -> Result, FormatError> { + let (hash, range) = self.key_values(item)?; + let range = match (&self.range, range) { + (Some((name, _)), Some(value)) => Some(match value { + AttributeValue::S(s) => RangeKey::S(s), + AttributeValue::B(b) => RangeKey::B(b), + AttributeValue::N(n) => RangeKey::N(n, Cow::Owned(parse_number(name, n)?)), + _ => unreachable!("check_key_value accepts S, N and B only"), + }), + _ => None, + }; + Ok(CheckedKey { + hash, + hash_text: hash_key_text(hash).expect("check_key_value accepts S, N and B only"), + range, + }) + } + + /// The hash and range key values of `item`, each checked with + /// [`check_key_value`]; an error names the key (bounded). + fn key_values<'a>( + &self, + item: &'a Item, + ) -> Result<(&'a AttributeValue, Option<&'a AttributeValue>), FormatError> { + let value_of = |name: &str| { + item.get(name).ok_or_else(|| { + FormatError::InvalidKey(format!("item has no key attribute {:?}", truncated(name))) + }) + }; + let hash = value_of(&self.hash.0)?; + let range = match &self.range { + Some((name, _)) => Some(value_of(name)?), + None => None, + }; + let checked = check_key_value(&self.hash.0, self.hash.1, hash).and_then(|()| { + match (&self.range, range) { + (Some((name, ty)), Some(value)) => check_key_value(name, *ty, value), + _ => Ok(()), + } + }); + checked.map_err(|e| match e { + FormatError::InvalidKey(reason) => { + let shown = serde_json::to_string(&self.key_item(hash, range)).unwrap_or_default(); + FormatError::InvalidKey(format!( + "item key {} cannot be a key_range endpoint: {reason}", + truncated(&shown) + )) + } + other => other, + })?; + Ok((hash, range)) + } + + /// A key-only item of the given values. + fn key_item(&self, hash: &AttributeValue, range: Option<&AttributeValue>) -> Item { + let mut key = Item::new(); + key.insert(self.hash.0.clone(), hash.clone()); + if let (Some((name, _)), Some(value)) = (&self.range, range) { + key.insert(name.clone(), value.clone()); + } + key + } + + /// The key of `item` as a [`super::KeyRange`] endpoint and as + /// [`super::KeyRangeCheck`] compares it: the key attributes only (other + /// attributes dropped), checked with [`Self::validate_key_item`]. A + /// key-ordered writer does not call it per item: + /// [`super::KeyRangeTracker`] makes the same check on each item without + /// building the key, and builds the first and last as the file's range; + /// an error refuses the backup (module docs: a stored `N` key such as + /// `abc` cannot be an endpoint, and the restore would refuse the item). + /// + /// # Errors + /// + /// [`FormatError::InvalidKey`] when a key attribute is missing or the + /// key fails [`Self::validate_key_item`]; the message names the key + /// (bounded) and the reason. + pub fn endpoint_key(&self, item: &Item) -> Result { + let (hash, range) = self.key_values(item)?; + Ok(self.key_item(hash, range)) + } + + /// Compare two items by their keys. Both must pass + /// [`Self::validate_key_item`] on their key attributes (extra attributes + /// are ignored here, so whole items can be compared). + /// + /// # Errors + /// + /// [`FormatError::InvalidKey`] when a key attribute is missing, has the + /// wrong type, or (for `N`) does not parse as a number. The last case is + /// defensive: a value that passed [`Self::validate_key_item`] always + /// parses. + pub fn compare(&self, a: &Item, b: &Item) -> Result { + let ha = hash_text(a, &self.hash)?; + let hb = hash_text(b, &self.hash)?; + match ha.as_bytes().cmp(hb.as_bytes()) { + Ordering::Equal => {} + other => return Ok(other), + } + let Some((name, ty)) = &self.range else { + return Ok(Ordering::Equal); + }; + let missing = + || FormatError::InvalidKey(format!("range key {:?} missing", truncated(name))); + let ra = a.get(name).ok_or_else(missing)?; + let rb = b.get(name).ok_or_else(missing)?; + match (ty, ra, rb) { + (ScalarAttributeType::S, AttributeValue::S(x), AttributeValue::S(y)) => { + Ok(x.as_bytes().cmp(y.as_bytes())) + } + (ScalarAttributeType::B, AttributeValue::B(x), AttributeValue::B(y)) => Ok(x.cmp(y)), + (ScalarAttributeType::N, AttributeValue::N(x), AttributeValue::N(y)) => { + Ok(parse_number(name, x)?.cmp(&parse_number(name, y)?)) + } + _ => Err(FormatError::InvalidKey(format!( + "range key {:?} does not match its declared type {ty:?}", + truncated(name) + ))), + } + } +} + +fn hash_text<'a>( + item: &'a Item, + (name, ty): &(String, ScalarAttributeType), +) -> Result, FormatError> { + let shown = || truncated(name); + let value = item + .get(name) + .ok_or_else(|| FormatError::InvalidKey(format!("hash key {:?} missing", shown())))?; + let actual = match value { + AttributeValue::S(_) => ScalarAttributeType::S, + AttributeValue::N(_) => ScalarAttributeType::N, + AttributeValue::B(_) => ScalarAttributeType::B, + _ => { + return Err(FormatError::InvalidKey(format!( + "hash key {:?} is not a scalar S, N or B", + shown() + ))); + } + }; + if actual != *ty { + return Err(FormatError::InvalidKey(format!( + "hash key {:?} is {actual:?}, the schema declares {ty:?}", + shown() + ))); + } + Ok(hash_key_text(value).expect("scalar checked above")) +} + +/// An `N` range key value as a decimal, for comparison. +fn parse_number(name: &str, n: &str) -> Result { + BigDecimal::from_str(n).map_err(|e| { + FormatError::InvalidKey(format!( + "range key {:?} value {:?}: {}", + truncated(name), + truncated(n), + truncated(&e.to_string()) + )) + }) +} + +/// One key attribute's value checked as [`KeyOrder::validate_key_item`] +/// checks it: the declared scalar type, not empty, and for `N` a valid +/// number in normalized form. The error is the reason alone. +fn check_key_value( + name: &str, + ty: ScalarAttributeType, + value: &AttributeValue, +) -> Result<(), FormatError> { + let shown = || truncated(name); + let empty = match value { + AttributeValue::S(s) => s.is_empty(), + AttributeValue::B(b) => b.is_empty(), + _ => false, + }; + if empty { + return Err(FormatError::InvalidKey(format!( + "key attribute {:?} is empty", + shown() + ))); + } + let actual = match value { + AttributeValue::S(_) => ScalarAttributeType::S, + AttributeValue::N(n) => { + let normalized = validate_and_normalize_number(n).map_err(|e| { + // The reason for a malformed number ends in the input, which + // is shown once, bounded, before it; the other reasons + // (overflow, underflow, too many digits) are fixed texts. + let reason = e.to_string(); + let reason = match reason.strip_suffix(n.as_str()) { + Some(rest) => rest.strip_suffix(": ").unwrap_or(rest).to_owned(), + None => reason, + }; + FormatError::InvalidKey(format!( + "key attribute {:?} value {:?} is not a valid number: {reason}", + shown(), + truncated(n), + )) + })?; + if normalized != *n { + return Err(FormatError::InvalidKey(format!( + "key attribute {:?} value {:?} is not in normalized form ({:?})", + shown(), + truncated(n), + truncated(&normalized) + ))); + } + ScalarAttributeType::N + } + AttributeValue::B(_) => ScalarAttributeType::B, + _ => { + return Err(FormatError::InvalidKey(format!( + "key attribute {:?} is not a scalar S, N or B", + shown() + ))); + } + }; + if actual != ty { + return Err(FormatError::InvalidKey(format!( + "key attribute {:?} is {actual:?}, the schema declares {ty:?}", + shown() + ))); + } + Ok(()) +} + +/// An item's key, checked by [`KeyOrder::checked_key`]: the values borrowed +/// from the item and an `N` range key parsed once, so the key-range checks +/// compare keys without building a key [`Item`]. [`HeldKey::as_checked`] +/// gives the same view of a held key. +pub(super) struct CheckedKey<'a> { + hash: &'a AttributeValue, + hash_text: Cow<'a, str>, + range: Option>, +} + +enum RangeKey<'a> { + S(&'a str), + B(&'a [u8]), + N(&'a str, Cow<'a, BigDecimal>), +} + +impl CheckedKey<'_> { + /// [`KeyOrder::compare`]'s order: the hash key's text bytewise, and the + /// range key only when the texts are equal (`S` and `B` bytewise, `N` by + /// value), so a key in another partition is decided by the text alone. + pub(super) fn cmp(&self, other: &CheckedKey<'_>) -> Ordering { + let range = || match (&self.range, &other.range) { + (Some(RangeKey::S(x)), Some(RangeKey::S(y))) => x.as_bytes().cmp(y.as_bytes()), + (Some(RangeKey::B(x)), Some(RangeKey::B(y))) => x.cmp(y), + (Some(RangeKey::N(_, x)), Some(RangeKey::N(_, y))) => x.cmp(y), + // Keys checked under one `KeyOrder` have one shape. + (x, y) => x.is_some().cmp(&y.is_some()), + }; + self.hash_text + .as_bytes() + .cmp(other.hash_text.as_bytes()) + .then_with(range) + } +} + +/// A key the key-range checks keep from one item to the next: a +/// [`CheckedKey`] owned, with the hash key's text and an `N` range key's +/// decimal kept, so comparing with it parses nothing. [`Self::assign`] +/// reuses its storage. +#[derive(Debug, Clone)] +pub(super) struct HeldKey { + hash: AttributeValue, + /// The hash key's text when it is not the value's own string (`B`). + hash_b64: Option, + range: Option, +} + +#[derive(Debug, Clone)] +enum HeldRange { + S(String), + B(Vec), + N(String, BigDecimal), +} + +impl HeldRange { + fn new(range: RangeKey<'_>) -> Self { + match range { + RangeKey::S(s) => Self::S(s.to_owned()), + RangeKey::B(b) => Self::B(b.to_vec()), + RangeKey::N(n, d) => Self::N(n.to_owned(), d.into_owned()), + } + } +} + +fn set_string(dst: &mut String, src: &str) { + dst.clear(); + dst.push_str(src); +} + +impl HeldKey { + pub(super) fn new(key: CheckedKey<'_>) -> Self { + Self { + hash_b64: matches!(key.hash, AttributeValue::B(_)).then(|| key.hash_text.into_owned()), + hash: key.hash.clone(), + range: key.range.map(HeldRange::new), + } + } + + /// Hold `key` instead, in this key's storage. Keys checked under one + /// [`KeyOrder`] have one shape, so every allocation is reused. + pub(super) fn assign(&mut self, key: CheckedKey<'_>) { + match (&mut self.hash, key.hash) { + (AttributeValue::S(dst), AttributeValue::S(src)) + | (AttributeValue::N(dst), AttributeValue::N(src)) => set_string(dst, src), + (AttributeValue::B(dst), AttributeValue::B(src)) => dst.clone_from(src), + (dst, src) => *dst = src.clone(), + } + match (&mut self.hash_b64, matches!(key.hash, AttributeValue::B(_))) { + (Some(text), true) => set_string(text, &key.hash_text), + (text, binary) => *text = binary.then(|| key.hash_text.into_owned()), + } + match (&mut self.range, key.range) { + (Some(HeldRange::S(dst)), Some(RangeKey::S(src))) => set_string(dst, src), + (Some(HeldRange::B(dst)), Some(RangeKey::B(src))) => { + dst.clear(); + dst.extend_from_slice(src); + } + (Some(HeldRange::N(dst, decimal)), Some(RangeKey::N(src, parsed))) => { + set_string(dst, src); + *decimal = parsed.into_owned(); + } + (dst, src) => *dst = src.map(HeldRange::new), + } + } + + pub(super) fn as_checked(&self) -> CheckedKey<'_> { + let hash_text = match (&self.hash_b64, &self.hash) { + (Some(text), _) => text.as_str(), + (None, AttributeValue::S(s) | AttributeValue::N(s)) => s.as_str(), + (None, _) => "", + }; + CheckedKey { + hash: &self.hash, + hash_text: Cow::Borrowed(hash_text), + range: self.range.as_ref().map(|r| match r { + HeldRange::S(s) => RangeKey::S(s), + HeldRange::B(b) => RangeKey::B(b), + HeldRange::N(n, d) => RangeKey::N(n, Cow::Borrowed(d)), + }), + } + } + + /// The key as a key-only [`Item`], the shape of a [`super::KeyRange`] + /// endpoint. + pub(super) fn to_item(&self, order: &KeyOrder) -> Item { + let mut key = Item::new(); + key.insert(order.hash.0.clone(), self.hash.clone()); + if let (Some((name, _)), Some(range)) = (&order.range, &self.range) { + let value = match range { + HeldRange::S(s) => AttributeValue::S(s.clone()), + HeldRange::B(b) => AttributeValue::B(b.clone()), + HeldRange::N(n, _) => AttributeValue::N(n.clone()), + }; + key.insert(name.clone(), value); + } + key + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn schema(range: Option) -> KeyOrder { + let mut ks = vec![KeySchemaElement { + attribute_name: "pk".to_owned(), + key_type: KeyType::Hash, + }]; + let mut defs = vec![AttributeDefinition { + attribute_name: "pk".to_owned(), + attribute_type: ScalarAttributeType::S, + }]; + if let Some(ty) = range { + ks.push(KeySchemaElement { + attribute_name: "sk".to_owned(), + key_type: KeyType::Range, + }); + defs.push(AttributeDefinition { + attribute_name: "sk".to_owned(), + attribute_type: ty, + }); + } + KeyOrder::resolve(&ks, &defs).unwrap() + } + + fn item(pk: &str, sk: Option) -> Item { + let mut m = Item::new(); + m.insert("pk".to_owned(), AttributeValue::S(pk.to_owned())); + if let Some(sk) = sk { + m.insert("sk".to_owned(), sk); + } + m + } + + #[test] + fn hash_key_text_matches_storage_encoding() { + assert_eq!(hash_key_text(&AttributeValue::S("a".into())).unwrap(), "a"); + assert_eq!( + hash_key_text(&AttributeValue::N("10".into())).unwrap(), + "10" + ); + assert_eq!( + hash_key_text(&AttributeValue::B(vec![0xde, 0xad])).unwrap(), + "3q0=" + ); + // Standard alphabet with padding: `+` and `/` distinguish it from + // URL-safe base64, and the storage row-key test pins the same bytes. + assert_eq!( + hash_key_text(&AttributeValue::B(vec![0xfb, 0xff, 0xbf])).unwrap(), + "+/+/" + ); + assert_eq!(hash_key_text(&AttributeValue::B(vec![1])).unwrap(), "AQ=="); + assert!(hash_key_text(&AttributeValue::Bool(true)).is_none()); + } + + /// The hash key's declared type is enforced on both sides of a compare, + /// not only the range key's. + #[test] + fn compare_rejects_hash_key_type_mismatch() { + let order = schema(None); + let good = item("p", None); + let mut bad = Item::new(); + bad.insert("pk".to_owned(), AttributeValue::N("1".into())); + assert!(order.compare(&good, &bad).is_err()); + assert!(order.compare(&bad, &good).is_err()); + } + + /// A `B` hash key sorts by its base64 text: `[0xff]` (`/w==`) before + /// `[0x00]` (`AA==`), the opposite of the raw bytes. A backend writing a + /// key-ordered file has to reproduce this order. + #[test] + fn binary_hash_keys_sort_by_base64_text() { + let ks = vec![KeySchemaElement { + attribute_name: "pk".to_owned(), + key_type: KeyType::Hash, + }]; + let defs = vec![AttributeDefinition { + attribute_name: "pk".to_owned(), + attribute_type: ScalarAttributeType::B, + }]; + let order = KeyOrder::resolve(&ks, &defs).unwrap(); + let b = |bytes: &[u8]| { + let mut m = Item::new(); + m.insert("pk".to_owned(), AttributeValue::B(bytes.to_vec())); + m + }; + assert_eq!( + hash_key_text(&AttributeValue::B(vec![0x00])).unwrap(), + "AA==" + ); + assert_eq!( + hash_key_text(&AttributeValue::B(vec![0xff])).unwrap(), + "/w==" + ); + assert_eq!( + order.compare(&b(&[0xff]), &b(&[0x00])).unwrap(), + Ordering::Less + ); + assert_eq!( + order.compare(&b(&[0x00]), &b(&[0xff])).unwrap(), + Ordering::Greater + ); + // `+` (0x2b) sorts before `/` (0x2f) and digits before letters. + assert_eq!( + order.compare(&b(&[0xfb]), &b(&[0xff])).unwrap(), + Ordering::Less + ); + assert_eq!( + order.compare(&b(&[0xd3]), &b(&[0x00])).unwrap(), + Ordering::Less + ); + // With a range key the hash key still decides first. + let mut ks = ks; + ks.push(KeySchemaElement { + attribute_name: "sk".to_owned(), + key_type: KeyType::Range, + }); + let mut defs = defs; + defs.push(AttributeDefinition { + attribute_name: "sk".to_owned(), + attribute_type: ScalarAttributeType::B, + }); + let order = KeyOrder::resolve(&ks, &defs).unwrap(); + let bb = |h: u8, r: u8| { + let mut m = b(&[h]); + m.insert("sk".to_owned(), AttributeValue::B(vec![r])); + m + }; + assert_eq!( + order.compare(&bb(0xff, 0xff), &bb(0x00, 0x00)).unwrap(), + Ordering::Less + ); + } + + /// Messages that name an attribute or quote a value from a manifest are + /// bounded. + #[test] + fn untrusted_names_and_values_are_bounded_in_errors() { + for n in [81, 300, 1000, 1_000_000] { + let name = "a".repeat(n); + let ks = vec![KeySchemaElement { + attribute_name: name.clone(), + key_type: KeyType::Hash, + }]; + let err = KeyOrder::resolve(&ks, &[]).unwrap_err(); + assert!( + err.to_string().len() < 200, + "{n}: {}", + err.to_string().len() + ); + let defs = vec![AttributeDefinition { + attribute_name: name.clone(), + attribute_type: ScalarAttributeType::N, + }]; + let order = KeyOrder::resolve(&ks, &defs).unwrap(); + let mut key = Item::new(); + key.insert(name.clone(), AttributeValue::N("9".repeat(n))); + for err in [ + order.validate_key_item(&key).unwrap_err(), + order + .validate_key_item(&Item::from([("x".to_owned(), AttributeValue::Null)])) + .unwrap_err(), + order.compare(&Item::new(), &key).unwrap_err(), + ] { + // The bound holds the two bounded fields and a fixed reason + // (the longest, 64 bytes, is the number validation's "more + // than 38 significant digits"). + assert!( + err.to_string().len() < 300, + "{n}: {}", + err.to_string().len() + ); + } + let mut wrong = Item::new(); + wrong.insert(name.clone(), AttributeValue::S("x".repeat(n))); + let err = order.validate_key_item(&wrong).unwrap_err(); + assert!( + err.to_string().len() < 200, + "{n}: {}", + err.to_string().len() + ); + // A range key with an unparseable value, through compare. + let order = schema(Some(ScalarAttributeType::N)); + let bad = item("p", Some(AttributeValue::N("x".repeat(n)))); + let err = order.compare(&bad, &bad).unwrap_err(); + assert!( + err.to_string().len() < 250, + "{n}: {}", + err.to_string().len() + ); + } + } + + #[test] + fn number_hash_keys_sort_as_text() { + let ks = vec![KeySchemaElement { + attribute_name: "pk".to_owned(), + key_type: KeyType::Hash, + }]; + let defs = vec![AttributeDefinition { + attribute_name: "pk".to_owned(), + attribute_type: ScalarAttributeType::N, + }]; + let order = KeyOrder::resolve(&ks, &defs).unwrap(); + let n = |s: &str| { + let mut m = Item::new(); + m.insert("pk".to_owned(), AttributeValue::N(s.to_owned())); + m + }; + assert_eq!(order.compare(&n("10"), &n("9")).unwrap(), Ordering::Less); + } + + #[test] + fn number_range_keys_sort_numerically() { + let order = schema(Some(ScalarAttributeType::N)); + let a = item("p", Some(AttributeValue::N("9".into()))); + let b = item("p", Some(AttributeValue::N("10".into()))); + let c = item("p", Some(AttributeValue::N("-1.5".into()))); + assert_eq!(order.compare(&a, &b).unwrap(), Ordering::Less); + assert_eq!(order.compare(&c, &a).unwrap(), Ordering::Less); + assert_eq!(order.compare(&a, &a).unwrap(), Ordering::Equal); + } + + #[test] + fn string_and_binary_range_keys_sort_bytewise() { + let order = schema(Some(ScalarAttributeType::S)); + let a = item("p", Some(AttributeValue::S("B".into()))); + let b = item("p", Some(AttributeValue::S("a".into()))); + assert_eq!(order.compare(&a, &b).unwrap(), Ordering::Less); + // Bytewise, not length-first: "ab" < "b". + let a = item("p", Some(AttributeValue::S("ab".into()))); + let b = item("p", Some(AttributeValue::S("b".into()))); + assert_eq!(order.compare(&a, &b).unwrap(), Ordering::Less); + let order = schema(Some(ScalarAttributeType::B)); + let a = item("p", Some(AttributeValue::B(vec![1, 2]))); + let b = item("p", Some(AttributeValue::B(vec![1, 2, 0]))); + assert_eq!(order.compare(&a, &b).unwrap(), Ordering::Less); + // Raw bytes, not their base64 text ("AA==" sorts after "/w=="), and + // not length-first. + let a = item("p", Some(AttributeValue::B(vec![0x00]))); + let b = item("p", Some(AttributeValue::B(vec![0xff]))); + assert_eq!(order.compare(&a, &b).unwrap(), Ordering::Less); + let a = item("p", Some(AttributeValue::B(vec![0x01, 0x00]))); + let b = item("p", Some(AttributeValue::B(vec![0x02]))); + assert_eq!(order.compare(&a, &b).unwrap(), Ordering::Less); + } + + #[test] + fn hash_key_decides_before_range_key() { + let order = schema(Some(ScalarAttributeType::N)); + let a = item("a", Some(AttributeValue::N("99".into()))); + let b = item("b", Some(AttributeValue::N("1".into()))); + assert_eq!(order.compare(&a, &b).unwrap(), Ordering::Less); + } + + #[test] + fn validate_key_item_rejects_wrong_shape() { + let order = schema(Some(ScalarAttributeType::N)); + order + .validate_key_item(&item("p", Some(AttributeValue::N("1".into())))) + .unwrap(); + assert!(order.validate_key_item(&item("p", None)).is_err()); + assert!( + order + .validate_key_item(&item("p", Some(AttributeValue::S("1".into())))) + .is_err() + ); + let mut extra = item("p", Some(AttributeValue::N("1".into()))); + extra.insert("v".to_owned(), AttributeValue::Null); + assert!(order.validate_key_item(&extra).is_err()); + } + + /// `N` endpoints must be valid `DynamoDB` numbers in normalized form; + /// `BigDecimal::from_str` alone would accept all of these. + #[test] + fn validate_key_item_rejects_non_dynamodb_numbers() { + let order = schema(Some(ScalarAttributeType::N)); + for bad in [ + "1E200", + "1_000", + "123456789012345678901234567890123456789", + "1.0", + "+1", + ".5", + "1.", + "-0", + "abc", + ] { + let err = order + .validate_key_item(&item("p", Some(AttributeValue::N(bad.into())))) + .err() + .unwrap_or_else(|| panic!("{bad:?} must be rejected")); + assert!(matches!(err, FormatError::InvalidKey(_)), "{bad:?}: {err}"); + } + for good in ["0", "1", "-1.5", "1E+125", "1E-130", "10"] { + let normalized = validate_and_normalize_number(good).unwrap(); + order + .validate_key_item(&item("p", Some(AttributeValue::N(normalized.clone())))) + .unwrap_or_else(|e| panic!("{normalized:?} (from {good:?}) rejected: {e}")); + } + } + + /// `endpoint_key` keeps the key attributes only and refuses, naming the + /// key, what `validate_key_item` refuses: the stored `N` keys + /// `BatchWriteItem` and `TransactWriteItems` accept (#618) among them. + #[test] + fn endpoint_key_takes_the_key_and_refuses_what_cannot_be_an_endpoint() { + let order = schema(Some(ScalarAttributeType::N)); + let mut whole = item("p", Some(AttributeValue::N("1.5".into()))); + whole.insert("v".to_owned(), AttributeValue::S("x".into())); + assert_eq!( + order.endpoint_key(&whole).unwrap(), + item("p", Some(AttributeValue::N("1.5".into()))) + ); + for bad in ["abc", "1E126", "1E-131", "5.0"] { + let err = order + .endpoint_key(&item("p", Some(AttributeValue::N(bad.into())))) + .unwrap_err(); + assert!( + matches!(&err, FormatError::InvalidKey(m) + if m.contains(&format!(r#""sk":{{"N":"{bad}"}}"#)) + && m.contains("cannot be a key_range endpoint")), + "{bad}: {err}" + ); + } + let err = order.endpoint_key(&item("p", None)).unwrap_err(); + assert!( + matches!(&err, FormatError::InvalidKey(m) if m.contains("no key attribute \"sk\"")), + "{err}" + ); + // The message stays bounded for a long key. + let long = item(&"p".repeat(5000), Some(AttributeValue::N("abc".into()))); + let err = order.endpoint_key(&long).unwrap_err(); + assert!(err.to_string().len() < 400, "{}", err.to_string().len()); + } + + #[test] + fn duplicate_attribute_definitions_are_unorderable() { + let ks = vec![KeySchemaElement { + attribute_name: "pk".to_owned(), + key_type: KeyType::Hash, + }]; + let defs = vec![ + AttributeDefinition { + attribute_name: "pk".to_owned(), + attribute_type: ScalarAttributeType::S, + }, + AttributeDefinition { + attribute_name: "pk".to_owned(), + attribute_type: ScalarAttributeType::N, + }, + ]; + let err = KeyOrder::resolve(&ks, &defs).unwrap_err(); + assert!(matches!(err, FormatError::UnorderableKeySchema(_)), "{err}"); + assert!(err.to_string().contains("more than one"), "{err}"); + } + + #[test] + fn compare_rejects_type_mismatch_and_bad_numbers() { + let order = schema(Some(ScalarAttributeType::N)); + let a = item("p", Some(AttributeValue::N("1".into()))); + let bad = item("p", Some(AttributeValue::S("1".into()))); + assert!(order.compare(&a, &bad).is_err()); + let nan = item("p", Some(AttributeValue::N("abc".into()))); + assert!(order.compare(&a, &nan).is_err()); + } + + #[test] + fn multi_part_keys_are_unorderable() { + let ks = vec![ + KeySchemaElement { + attribute_name: "a".to_owned(), + key_type: KeyType::Hash, + }, + KeySchemaElement { + attribute_name: "b".to_owned(), + key_type: KeyType::Hash, + }, + ]; + let defs = vec![ + AttributeDefinition { + attribute_name: "a".to_owned(), + attribute_type: ScalarAttributeType::S, + }, + AttributeDefinition { + attribute_name: "b".to_owned(), + attribute_type: ScalarAttributeType::S, + }, + ]; + assert!(matches!( + KeyOrder::resolve(&ks, &defs), + Err(FormatError::UnorderableKeySchema(_)) + )); + } + + #[test] + fn range_key_named_like_the_hash_key_is_refused() { + let ks = [ + KeySchemaElement { + attribute_name: "pk".to_owned(), + key_type: KeyType::Hash, + }, + KeySchemaElement { + attribute_name: "pk".to_owned(), + key_type: KeyType::Range, + }, + ]; + let defs = [AttributeDefinition { + attribute_name: "pk".to_owned(), + attribute_type: ScalarAttributeType::S, + }]; + let err = KeyOrder::resolve(&ks, &defs).unwrap_err(); + assert!( + matches!(&err, FormatError::UnorderableKeySchema(m) if m.contains("both")), + "{err}" + ); + } + + #[test] + fn range_key_before_the_hash_key_is_refused() { + let ks = [ + KeySchemaElement { + attribute_name: "sk".to_owned(), + key_type: KeyType::Range, + }, + KeySchemaElement { + attribute_name: "pk".to_owned(), + key_type: KeyType::Hash, + }, + ]; + let defs = [ + AttributeDefinition { + attribute_name: "pk".to_owned(), + attribute_type: ScalarAttributeType::S, + }, + AttributeDefinition { + attribute_name: "sk".to_owned(), + attribute_type: ScalarAttributeType::S, + }, + ]; + let err = KeyOrder::resolve(&ks, &defs).unwrap_err(); + assert!( + matches!(&err, FormatError::UnorderableKeySchema(m) if m.contains("before")), + "{err}" + ); + // The same elements HASH first resolve. + KeyOrder::resolve(&[ks[1].clone(), ks[0].clone()], &defs).unwrap(); + } + + /// Two `N` range keys that an `f64` cannot tell apart: they first differ + /// in the 38th significant digit, far beyond the 17 an `f64` holds, so a + /// comparator that went through floating point would call them equal. + #[test] + fn number_range_keys_differing_beyond_seventeen_digits_are_ordered() { + let order = schema(Some(ScalarAttributeType::N)); + let low = "1234567890123456789012345678901234567"; + let high = "1234567890123456789012345678901234568"; + let lo = item("p", Some(AttributeValue::N(low.to_owned()))); + let hi = item("p", Some(AttributeValue::N(high.to_owned()))); + #[allow(clippy::float_cmp)] + { + assert_eq!(low.parse::().unwrap(), high.parse::().unwrap()); + } + assert_eq!(order.compare(&lo, &hi).unwrap(), Ordering::Less); + assert_eq!(order.compare(&hi, &lo).unwrap(), Ordering::Greater); + // And small magnitudes alike. + let lo = item( + "p", + Some(AttributeValue::N("1.0000000000000000000001E-100".into())), + ); + let hi = item( + "p", + Some(AttributeValue::N("1.0000000000000000000002E-100".into())), + ); + assert_eq!(order.compare(&lo, &hi).unwrap(), Ordering::Less); + } + + /// The module doc's normalization and comparison examples, pinned so + /// the stated rules and the code cannot drift apart: each input's + /// normalized form, the two refused magnitudes, the exponent's 32-bit + /// range, the long forms' + /// lengths, the base64 alphabet, UTF-8 byte order of `S`, and the + /// 38th-digit `N` range-key pair. + #[test] + fn doc_examples_normalize_as_stated() { + for (input, normalized) in [ + ("00042", "42"), + ("1.50", "1.5"), + ("1.5E3", "1500"), + ("+1", "1"), + (".5", "0.5"), + ("-0", "0"), + ("-0.0", "0"), + ("0E5", "0"), + ("5.0", "5"), + ("1e2", "100"), + ("-1.5", "-1.5"), + ("9.9E125", &format!("99{}", "0".repeat(124))), + ] { + assert_eq!( + validate_and_normalize_number(input).unwrap(), + normalized, + "{input}" + ); + } + let big = validate_and_normalize_number("1E+125").unwrap(); + assert_eq!(big, format!("1{}", "0".repeat(125))); + assert_eq!(big.len(), 126); + let small = validate_and_normalize_number("1E-130").unwrap(); + assert_eq!(small, format!("0.{}1", "0".repeat(129))); + assert_eq!(small.len(), 132); + assert!(validate_and_normalize_number("1E-131").is_err()); + assert!(validate_and_normalize_number("1E126").is_err()); + assert!(validate_and_normalize_number(&"9".repeat(38)).is_ok()); + assert!(validate_and_normalize_number(&"9".repeat(39)).is_err()); + for refused in [" 5", "5 ", "1_000", "0x10", ".", "1e", "NaN", ""] { + assert!( + validate_and_normalize_number(refused).is_err(), + "{refused:?}" + ); + } + // The exponent is a signed 32-bit integer: one past either end is + // refused even where the value would be zero, and the ends are read. + for refused in ["0E2147483648", "0E-2147483649", "1E2147483648"] { + assert!( + validate_and_normalize_number(refused).is_err(), + "{refused:?}" + ); + } + for zero in [ + "0E2147483647", + "0E-2147483648", + "0E+2147483647", + "0.0E-2147483648", + ] { + assert_eq!(validate_and_normalize_number(zero).unwrap(), "0", "{zero}"); + } + // A non-zero value with an exponent at either end is refused by its + // magnitude, without overflow in the exponent arithmetic. + assert!(validate_and_normalize_number("10E2147483647").is_err()); + assert!(validate_and_normalize_number("0.1E-2147483648").is_err()); + // Normalized output is a fixed point of normalization. + for s in ["42", "1.5", "0", &big, &small] { + assert_eq!(validate_and_normalize_number(s).unwrap(), s); + } + // `B`: the standard alphabet with padding, so `+` and `/` appear and + // the text is what a JSON endpoint carries. + assert_eq!( + hash_key_text(&AttributeValue::B(vec![0xfb, 0xff, 0xbf])).unwrap(), + "+/+/" + ); + assert_eq!( + hash_key_text(&AttributeValue::B(vec![0xff])).unwrap(), + "/w==" + ); + // `S`: UTF-8 bytes; `é` (0xc3 0xa9) sorts after `z` (0x7a). + let order = schema(Some(ScalarAttributeType::S)); + let e = item("p", Some(AttributeValue::S("é".into()))); + let z = item("p", Some(AttributeValue::S("z".into()))); + assert_eq!(order.compare(&z, &e).unwrap(), Ordering::Less); + let order = schema(None); + assert_eq!( + order.compare(&item("z", None), &item("é", None)).unwrap(), + Ordering::Less + ); + // `N` range keys: exact decimal; 38 significant digits are kept. + let order = schema(Some(ScalarAttributeType::N)); + let low = "1234567890123456789012345678901234567"; + let high = "1234567890123456789012345678901234568"; + assert_eq!( + order + .compare( + &item("p", Some(AttributeValue::N(low.into()))), + &item("p", Some(AttributeValue::N(high.into()))) + ) + .unwrap(), + Ordering::Less + ); + } +} diff --git a/crates/core/src/backup_format/layout.rs b/crates/core/src/backup_format/layout.rs new file mode 100644 index 000000000..7dd912bae --- /dev/null +++ b/crates/core/src/backup_format/layout.rs @@ -0,0 +1,767 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! Store key layout for a backup prefix, and the backup ARN. +//! +//! A backup lives under `///` in the store +//! and holds the fixed object names defined here plus numbered data files +//! under `data/`. +//! +//! This module is the one implementation of the backup id and the backup ARN +//! shape `arn::dynamodb:::table/
/backup/`. +//! The engine and every storage backend reach it through +//! `extenddb_storage::util` (`check_backup_arn`, `parse_backup_arn`, +//! `is_backup_id`, `format_backup_id`, ...), which re-exports the functions +//! defined here; the field split itself is [`crate::arn::fields`], the same +//! split authorization uses, so the engine and authorization cannot read a +//! different resource from one string. One other reader remains: +//! `extenddb_storage::util::is_backup_resource_arn` splits the string with +//! its own `splitn(6, ':')` to tell a backup ARN from a table ARN. [`BackupArn`] is the owned form the off-host backup format +//! uses: it adds the store-prefix rules (a table name that is only dots is +//! refused, since `.` and `..` are path components) on top of +//! [`check_backup_arn`], and turns the ARN into a store prefix. +//! +//! Every component is validated on construction, whether it comes from an ARN +//! string or from the engine's own fields, so a prefix can never contain a +//! separator, a traversal component, or a control character: the store's key +//! validation is a second line, not the first. + +use super::FormatError; +use super::error::truncated; + +/// The `ExtendDB` manifest, written last; its presence marks a complete backup. +pub const EXTENDDB_MANIFEST_FILE: &str = "extenddb-manifest.json"; + +/// The sha256 of the manifest's bytes, written before the manifest. See +/// [`super::verify_manifest_checksum`] and, for the write order, +/// [`super::BackupDocuments`]. +pub const MANIFEST_CHECKSUM_FILE: &str = "extenddb-manifest.sha256"; + +/// The `DynamoDB` export summary manifest. +pub const SUMMARY_MANIFEST_FILE: &str = "manifest-summary.json"; + +/// The `DynamoDB` export files manifest. +pub const FILES_MANIFEST_FILE: &str = "manifest-files.json"; + +/// The prefix that holds only data files, so `ImportTable` against it never +/// sees a manifest. +pub const DATA_PREFIX: &str = "data/"; + +/// Longest table name `DynamoDB` accepts, kept here as a path-safety bound: +/// a longer component would exceed common filesystem name limits. The +/// engine's configurable minimum length is not enforced here; a shorter name +/// the engine accepted is a valid ARN component. The engine's +/// `max_table_name_length` can be raised above 255; a table with a longer +/// name has a backup ARN `check_backup_arn` accepts and this type refuses, +/// so the off-host backup path cannot write it (the design has `CreateBackup` +/// refuse it up front). +const MAX_TABLE_NAME_LEN: usize = 255; + +/// The key of the `index`-th data file within a backup prefix: +/// `data/000001.json.gz` for index 1. +#[must_use] +pub fn data_file_key(index: u32) -> String { + format!("{DATA_PREFIX}{index:06}.json.gz") +} + +// --------------------------------------------------------------------------- +// Backup id +// --------------------------------------------------------------------------- + +/// Digits of the epoch-millisecond prefix of a backup id. The service +/// zero-pads to 14 (`01489602797149-73d8d5bc`); 13 digits cover today's +/// instants, so the padding is visible as a leading zero. +pub const BACKUP_ID_MILLIS_DIGITS: usize = 14; + +/// Format a backup id from its parts: 14-digit zero-padded epoch +/// milliseconds, `-`, 8 lowercase hex digits. The random suffix is part of +/// the identifier, not decoration: a timestamp alone makes a backup id +/// derivable by anyone who knows roughly when the backup ran, and the suffix +/// keeps two backups created in the same millisecond apart. +/// `extenddb_storage::util::new_backup_id` draws the suffix. +#[must_use] +pub fn format_backup_id(epoch_millis: u64, suffix: u32) -> String { + format!("{epoch_millis:0BACKUP_ID_MILLIS_DIGITS$}-{suffix:08x}") +} + +fn all_digits(s: &str, n: usize) -> bool { + s.len() == n && s.bytes().all(|b| b.is_ascii_digit()) +} + +/// Whether `s` is `n` lowercase hex digits. +pub(super) fn lower_hex(s: &str, n: usize) -> bool { + s.len() == n + && s.bytes() + .all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b)) +} + +/// Whether `id` is a backup id this codebase ever issued: 14 or 13 digits, +/// `-`, 8 lowercase hex digits (the 14-digit form is the current one; the +/// 13-digit form is what every backend issued before the shared id), or 13 +/// bare digits (epoch milliseconds with no suffix, the form PostgreSQL issued +/// in v0.1.0 to v0.1.2 and MongoDB before its suffix was added). No build has +/// put a UUID in a backup ARN, so none is accepted. +#[must_use] +pub fn is_backup_id(id: &str) -> bool { + match id.split_once('-') { + Some((millis, suffix)) => { + (all_digits(millis, BACKUP_ID_MILLIS_DIGITS) + || all_digits(millis, BACKUP_ID_MILLIS_DIGITS - 1)) + && lower_hex(suffix, 8) + } + None => all_digits(id, BACKUP_ID_MILLIS_DIGITS - 1), + } +} + +/// The epoch milliseconds a backup id (or a whole backup ARN) was minted at, +/// or `None` if it is not in the shape [`format_backup_id`] produces. +#[must_use] +pub fn backup_id_epoch_millis(id_or_arn: &str) -> Option { + let id = id_or_arn.rsplit('/').next()?; + let (millis, suffix) = id.split_once('-')?; + if !all_digits(millis, BACKUP_ID_MILLIS_DIGITS) || !lower_hex(suffix, 8) { + return None; + } + millis.parse().ok() +} + +/// The instant a backup id from [`format_backup_id`] was minted, as epoch +/// seconds with millisecond precision. This is the backup's +/// `BackupCreationDateTime`: the off-host backup job mints the id from the +/// instant its snapshot pinned (design 5.1), and the manifest parser requires +/// `backup_creation_date_time` to equal this value, so the ARN, the manifest +/// and the API agree on one instant. +#[must_use] +#[allow(clippy::cast_precision_loss)] // epoch ms fit f64's 53-bit mantissa +pub fn backup_creation_date_time(id_or_arn: &str) -> Option { + backup_id_epoch_millis(id_or_arn).map(|ms| ms as f64 / 1000.0) +} + +// --------------------------------------------------------------------------- +// Backup ARN +// --------------------------------------------------------------------------- + +/// The parts of a well-formed backup ARN, borrowed from the input. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct BackupArnParts<'a> { + /// The ARN with surrounding ASCII whitespace removed, as the other fields + /// were read from it. + pub arn: &'a str, + pub partition: &'a str, + pub region: &'a str, + pub account_id: &'a str, + pub table_name: &'a str, + pub backup_id: &'a str, +} + +/// Why a string is not a backup ARN. The two classes carry the two texts the +/// service answers with (measured on `DynamoDB`). Which text goes with which +/// input was captured for one input only (a bare 13-digit id is "not +/// valid"), so the split is the parser's: a string that is not an ARN of the +/// right shape, is too long, or carries a character no ARN part allows is +/// `Invalid`; an ARN of the right shape whose backup id is not one is `Id`. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum BackupArnError { + /// Not `arn::dynamodb:::table//backup/` + /// with a non-empty region, account, and name, over + /// [`MAX_BACKUP_ARN_LEN`] bytes, with a control or non-ASCII character, or + /// with a region outside `[a-z0-9-]`, a table name outside + /// `[a-zA-Z0-9_.-]`, or an account outside `[0-9]`. The service's text: + /// `Invalid Request: BackupArn is not valid`. + Invalid, + /// The backup id is neither the current shape nor one an earlier build + /// issued (see [`is_backup_id`]). The service's text: `Invalid Backup ARN`. + Id, +} + +impl std::fmt::Display for BackupArnError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(match self { + Self::Invalid => "not a backup ARN", + Self::Id => "not a backup id", + }) + } +} + +/// The longest `BackupArn` the service model accepts (`BackupArn` shape, +/// botocore 1.33.13: max 1024). +pub const MAX_BACKUP_ARN_LEN: usize = 1024; + +/// Whether every byte is in `allowed` or an ASCII alphanumeric. +fn ascii_word(s: &str, allowed: &[u8]) -> bool { + !s.is_empty() + && s.bytes() + .all(|b| b.is_ascii_alphanumeric() || allowed.contains(&b)) +} + +/// The region shape a backup ARN may carry: non-empty `[a-z0-9-]`. The +/// manifest checks a table ARN's region with the same predicate. +pub(super) fn is_region(s: &str) -> bool { + ascii_word(s, b"-") && !s.bytes().any(|b| b.is_ascii_uppercase()) +} + +/// Strip surrounding ASCII whitespace from a `BackupArn`. Measured on +/// `DynamoDB`: an ARN with a trailing newline or CRLF resolves, and one with a +/// leading space or tab is refused with `Invalid Backup ARN`. This parser +/// trims both ends and every ASCII whitespace character, as main's parser +/// did; the leading-whitespace case is kept for compatibility with callers +/// of that parser, not because the service accepts it. Every layer that +/// reads a backup ARN from a request, the engine through +/// [`check_backup_arn`] and authorization before it derives the resource, +/// trims with this one function so they act on the same string. +#[must_use] +pub fn trim_backup_arn(arn: &str) -> &str { + arn.trim_matches(|c: char| c.is_ascii_whitespace()) +} + +/// Format a backup ARN from its parts. The one place the string is built; +/// `extenddb_storage::util::backup_arn` and [`BackupArn`] call it. +#[must_use] +pub fn format_backup_arn( + partition: &str, + region: &str, + account_id: &str, + table_name: &str, + backup_id: &str, +) -> String { + format!("arn:{partition}:dynamodb:{region}:{account_id}:table/{table_name}/backup/{backup_id}") +} + +/// Parse a backup ARN, or `None` if [`check_backup_arn`] refuses it. +#[must_use] +pub fn parse_backup_arn(arn: &str) -> Option> { + check_backup_arn(arn).ok() +} + +/// Parse a backup ARN after trimming surrounding ASCII whitespace with +/// [`trim_backup_arn`] (wider than the service, which resolves a trailing +/// newline or CRLF and refuses leading whitespace; see there), with the +/// length limit below applied to what remains, refusing one that is not exactly six +/// colon-separated fields, not `arn::dynamodb:...`, has an empty +/// or malformed region, account, or table name, a resource other than +/// `table//backup/`, a control or non-ASCII character, more than +/// [`MAX_BACKUP_ARN_LEN`] bytes, or an id that is not a backup id +/// ([`is_backup_id`]). The region and partition are checked for shape only: +/// an ARN in another region of the caller's account resolves on the service, +/// so the engine substitutes its own region rather than refuse it, and the +/// engine decides which partitions it serves. +/// +/// The field split is [`crate::arn::fields`], shared with authorization. +/// +/// # Errors +/// +/// [`BackupArnError`], one class per service text. +pub fn check_backup_arn(arn: &str) -> Result, BackupArnError> { + let arn = trim_backup_arn(arn); + if arn.len() > MAX_BACKUP_ARN_LEN || !arn.bytes().all(|b| b.is_ascii_graphic()) { + return Err(BackupArnError::Invalid); + } + let f = crate::arn::fields(arn).ok_or(BackupArnError::Invalid)?; + if !ascii_word(f.partition, b"-") + || f.service != "dynamodb" + || !is_region(f.region) + || f.account.is_empty() + || !f.account.bytes().all(|b| b.is_ascii_digit()) + // `fields` keeps any further `:` inside the resource; a backup + // resource has none. + || f.resource.contains(':') + { + return Err(BackupArnError::Invalid); + } + let parts: Vec<&str> = f.resource.split('/').collect(); + let ["table", table_name, "backup", backup_id] = parts.as_slice() else { + return Err(BackupArnError::Invalid); + }; + // The table-name alphabet is CreateTable's, from the function CreateTable + // calls (`validate_table_name_chars`), so a name CreateTable accepts is a + // name this parser accepts. The engine's `validate_name_member` and + // `validate_index_name` spell the same alphabet again. + if table_name.is_empty() + || crate::validation::validate_table_name_chars(table_name).is_err() + || backup_id.is_empty() + { + return Err(BackupArnError::Invalid); + } + if !is_backup_id(backup_id) { + return Err(BackupArnError::Id); + } + Ok(BackupArnParts { + arn, + partition: f.partition, + region: f.region, + account_id: f.account, + table_name, + backup_id, + }) +} + +/// A validated backup ARN. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct BackupArn { + partition: String, + region: String, + account_id: String, + table_name: String, + backup_id: String, +} + +impl BackupArn { + /// Build an ARN from its components, validating each with the rules of + /// [`check_backup_arn`] (the components are formatted and parsed back, so + /// one that carries a `:` or `/` cannot shift the fields) plus the + /// store-prefix rules: the table name is at most 255 bytes and is not + /// made only of dots (`.` and `..` are path components; a longer run of + /// dots is a legal `DynamoDB` name but is refused to keep the rule + /// simple). The backup id must be one this codebase issues + /// ([`is_backup_id`]), which no dot-only string is. + /// + /// # Errors + /// + /// Returns [`FormatError::InvalidArn`] naming the component that failed. + pub fn new( + partition: &str, + region: &str, + account_id: &str, + table_name: &str, + backup_id: &str, + ) -> Result { + let formatted = format_backup_arn(partition, region, account_id, table_name, backup_id); + let parts = check_backup_arn(&formatted).map_err(|e| { + FormatError::InvalidArn(format!( + "{e}: partition {:?}, region {:?}, account id {:?}, table name {:?}, \ + backup id {:?}", + truncated(partition), + truncated(region), + truncated(account_id), + truncated(table_name), + truncated(backup_id) + )) + })?; + // A separator inside a component would have parsed back as different + // fields; a leading or trailing space would have been trimmed away. + if ( + parts.partition, + parts.region, + parts.account_id, + parts.table_name, + parts.backup_id, + ) != (partition, region, account_id, table_name, backup_id) + { + return Err(FormatError::InvalidArn(format!( + "a component contains a separator or surrounding whitespace: {:?}", + truncated(&formatted) + ))); + } + Self::from_parts(&parts) + } + + /// Parse an ARN of the shape + /// `arn::dynamodb:::table/
/backup/` + /// with [`check_backup_arn`] and the store-prefix rules of [`Self::new`]. + /// + /// # Errors + /// + /// Returns [`FormatError::InvalidArn`] when the string does not have that + /// shape or a component fails validation. + pub fn parse(arn: &str) -> Result { + // The input may come from an untrusted manifest; report a bounded + // slice of it, not the whole string. + let shown = || truncated(trim_backup_arn(arn)); + let parts = check_backup_arn(arn) + .map_err(|e| FormatError::InvalidArn(format!("{:?}: {e}", shown())))?; + Self::from_parts(&parts).map_err(|e| match e { + FormatError::InvalidArn(reason) => { + FormatError::InvalidArn(format!("{:?}: {reason}", shown())) + } + other => other, + }) + } + + /// The store-prefix rules over already-checked parts: the two rules + /// `check_backup_arn` does not apply, and the only two inputs it accepts + /// that this type refuses. + fn from_parts(parts: &BackupArnParts<'_>) -> Result { + if parts.table_name.len() > MAX_TABLE_NAME_LEN { + return Err(FormatError::InvalidArn(format!( + "table name of {} bytes is longer than {MAX_TABLE_NAME_LEN}", + parts.table_name.len() + ))); + } + if parts.table_name.bytes().all(|b| b == b'.') { + return Err(FormatError::InvalidArn(format!( + "table name {:?} is only dots", + truncated(parts.table_name) + ))); + } + Ok(Self { + partition: parts.partition.to_owned(), + region: parts.region.to_owned(), + account_id: parts.account_id.to_owned(), + table_name: parts.table_name.to_owned(), + backup_id: parts.backup_id.to_owned(), + }) + } + + /// The ARN string. + #[must_use] + pub fn to_arn_string(&self) -> String { + format_backup_arn( + &self.partition, + &self.region, + &self.account_id, + &self.table_name, + &self.backup_id, + ) + } + + /// The store key prefix for this backup: + /// `///`. + #[must_use] + pub fn store_prefix(&self) -> String { + format!( + "{}/{}/{}/", + self.account_id, self.table_name, self.backup_id + ) + } + + /// Partition field (`aws`, `aws-cn`, `aws-us-gov`). + #[must_use] + pub fn partition(&self) -> &str { + &self.partition + } + + /// Region field. + #[must_use] + pub fn region(&self) -> &str { + &self.region + } + + /// Account id field. + #[must_use] + pub fn account_id(&self) -> &str { + &self.account_id + } + + /// Table name from the resource path. + #[must_use] + pub fn table_name(&self) -> &str { + &self.table_name + } + + /// Backup id from the resource path. + #[must_use] + pub fn backup_id(&self) -> &str { + &self.backup_id + } +} + +impl std::fmt::Display for BackupArn { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(&self.to_arn_string()) + } +} + +impl std::str::FromStr for BackupArn { + type Err = FormatError; + + fn from_str(s: &str) -> Result { + Self::parse(s) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + const ARN: &str = + "arn:aws:dynamodb:us-east-1:123456789012:table/Music/backup/01489602797149-73d8d5bc"; + + #[test] + fn store_prefix_shape() { + let arn = BackupArn::parse(ARN).unwrap(); + assert_eq!( + arn.store_prefix(), + "123456789012/Music/01489602797149-73d8d5bc/" + ); + } + + #[test] + fn data_file_key_is_zero_padded() { + assert_eq!(data_file_key(1), "data/000001.json.gz"); + assert_eq!(data_file_key(42), "data/000042.json.gz"); + assert_eq!(data_file_key(1_000_000), "data/1000000.json.gz"); + } + + #[test] + fn parse_valid_arn() { + let arn = BackupArn::parse(ARN).unwrap(); + assert_eq!(arn.partition(), "aws"); + assert_eq!(arn.region(), "us-east-1"); + assert_eq!(arn.account_id(), "123456789012"); + assert_eq!(arn.table_name(), "Music"); + assert_eq!(arn.backup_id(), "01489602797149-73d8d5bc"); + assert_eq!(arn.to_arn_string(), ARN); + assert_eq!(arn.to_string(), ARN); + assert_eq!(ARN.parse::().unwrap(), arn); + } + + #[test] + fn other_partitions_parse() { + for arn in [ + "arn:aws-cn:dynamodb:cn-north-1:123456789012:table/Music/backup/01489602797149-73d8d5bc", + "arn:aws-us-gov:dynamodb:us-gov-west-1:123456789012:table/Music/backup/01489602797149-73d8d5bc", + ] { + let parsed = BackupArn::parse(arn).unwrap(); + assert_eq!(parsed.to_arn_string(), arn); + } + } + + #[test] + fn new_and_parse_agree() { + let built = BackupArn::new( + "aws", + "eu-west-1", + "000000000001", + "My.Table-2", + "01489602797149-73d8d5bc", + ) + .unwrap(); + let parsed = BackupArn::parse(&built.to_arn_string()).unwrap(); + assert_eq!(parsed, built); + assert_eq!( + built.store_prefix(), + "000000000001/My.Table-2/01489602797149-73d8d5bc/" + ); + // Every id shape `is_backup_id` accepts is a valid prefix component. + for id in ["1489602797149-73d8d5bc", "1489602797149"] { + BackupArn::new("aws", "eu-west-1", "000000000001", "t", id).unwrap(); + } + } + + #[test] + fn parse_rejects_malformed_arns() { + const ID: &str = "01489602797149-73d8d5bc"; + for arn in [ + "", + "arn:aws:dynamodb", + "arn:aws:dynamodb:us-east-1", + "arn:aws:dynamodb:us-east-1:123456789012", + "arn:aws:dynamodb:us-east-1:123456789012:table/Music", + &format!("arn:aws:dynamodb:us-east-1::table/Music/backup/{ID}"), + &format!("arn:aws:dynamodb::123456789012:table/Music/backup/{ID}"), + &format!("arn:aws:dynamodb:us-east-1:123456789012:table//backup/{ID}"), + "arn:aws:dynamodb:us-east-1:123456789012:table/Music/backup/", + &format!("arn:aws:dynamodb:us-east-1:123456789012:index/Music/backup/{ID}"), + &format!("arn:aws:dynamodb:us-east-1:123456789012:table/Music/backup/{ID}/b"), + &format!("arn:aws:s3:us-east-1:123456789012:table/Music/backup/{ID}"), + &format!("arn::dynamodb:us-east-1:123456789012:table/Music/backup/{ID}"), + // Extra colon fields: the account must be field 4, nothing else. + &format!("arn:aws:dynamodb:us-east-1:evil:123456789012:table/Music/backup/{ID}"), + &format!("arn:aws:dynamodb:us-east-1:123456789012:table/Music/backup/{ID}:x"), + // Traversal and separators inside components. + &format!("arn:aws:dynamodb:us-east-1:123456789012:table/../backup/{ID}"), + "arn:aws:dynamodb:us-east-1:123456789012:table/Music/backup/..", + "arn:aws:dynamodb:us-east-1:123456789012:table/Music/backup/a\\b", + &format!("arn:aws:dynamodb:us-east-1:12345678 9012:table/Music/backup/{ID}"), + &format!("arn:aws:dynamodb:us-east-1:123456789012:table/Mu sic/backup/{ID}"), + // Account is digits; region is lowercase. + &format!("arn:aws:dynamodb:us-east-1:12345678901x:table/Music/backup/{ID}"), + &format!("arn:aws:dynamodb:US-EAST-1:123456789012:table/Music/backup/{ID}"), + // Ids no build issued. + "arn:aws:dynamodb:us-east-1:123456789012:table/Music/backup/1", + "arn:aws:dynamodb:us-east-1:123456789012:table/Music/backup/a4b1d6c2-0000-4000-8000-000000000000", + // Dot-only names are traversal components. + &format!("arn:aws:dynamodb:us-east-1:123456789012:table/./backup/{ID}"), + &format!("arn:aws:dynamodb:us-east-1:123456789012:table/..../backup/{ID}"), + ] { + assert!( + BackupArn::parse(arn).is_err(), + "expected rejection for {arn:?}" + ); + } + let long = "a".repeat(256); + assert!( + BackupArn::parse(&format!( + "arn:aws:dynamodb:us-east-1:123456789012:table/{long}/backup/{ID}" + )) + .is_err() + ); + // Surrounding whitespace is trimmed at both ends (the service trims + // only a trailing newline; `trim_backup_arn`); the parsed ARN is the + // trimmed one. + let padded = format!(" {ARN}\n"); + assert_eq!(BackupArn::parse(&padded).unwrap().to_arn_string(), ARN); + } + + /// `check_backup_arn` and `BackupArn::parse` agree on every input except + /// the store-prefix rules `BackupArn` adds. + #[test] + fn check_and_typed_parse_agree() { + for arn in [ + ARN, + "arn:aws:dynamodb:us-east-1:123456789012:table/Music/backup/1489602797149-73d8d5bc", + "arn:aws:dynamodb:us-east-1:123456789012:table/Music/backup/1489602797149", + "arn:aws-cn:dynamodb:cn-north-1:123456789012:table/a.b/backup/01489602797149-73d8d5bc", + "arn:aws:dynamodb:us-east-1:123456789012:table/Music/backup/1", + "arn:aws:dynamodb:us-east-1:123456789012:table/Mu sic/backup/01489602797149-73d8d5bc", + "not-an-arn", + ] { + let checked = check_backup_arn(arn); + let typed = BackupArn::parse(arn); + assert_eq!(checked.is_ok(), typed.is_ok(), "{arn}"); + if let (Ok(c), Ok(t)) = (checked, typed) { + assert_eq!(c.partition, t.partition()); + assert_eq!(c.region, t.region()); + assert_eq!(c.account_id, t.account_id()); + assert_eq!(c.table_name, t.table_name()); + assert_eq!(c.backup_id, t.backup_id()); + } + } + // The two divergences: a dot-only table name and a name over 255 + // bytes are legal ARNs but not store prefixes. + for arn in [ + "arn:aws:dynamodb:us-east-1:123456789012:table/.../backup/01489602797149-73d8d5bc" + .to_owned(), + format!( + "arn:aws:dynamodb:us-east-1:123456789012:table/{}/backup/01489602797149-73d8d5bc", + "a".repeat(256) + ), + ] { + assert!(check_backup_arn(&arn).is_ok(), "{arn}"); + assert!(BackupArn::parse(&arn).is_err(), "{arn}"); + } + // A 255-byte name is both. + let max = format!( + "arn:aws:dynamodb:us-east-1:123456789012:table/{}/backup/01489602797149-73d8d5bc", + "a".repeat(255) + ); + assert!(BackupArn::parse(&max).is_ok()); + } + + /// The id rule is a grid over millis length and suffix length, not a + /// list of instances; and the two error classes map to two service + /// texts, so they are pinned here where the parser lives. + #[test] + fn id_shape_grid_and_error_classes() { + for millis_len in 0..=16usize { + for suffix_len in 0..=10usize { + let id = format!("{}-{}", "1".repeat(millis_len), "a".repeat(suffix_len)); + let expected = (millis_len == 13 || millis_len == 14) && suffix_len == 8; + assert_eq!(is_backup_id(&id), expected, "{id}"); + } + let bare = "1".repeat(millis_len); + assert_eq!(is_backup_id(&bare), millis_len == 13, "{bare}"); + } + let arn_with = + |id: &str| format!("arn:aws:dynamodb:us-east-1:123456789012:table/Music/backup/{id}"); + assert_eq!( + check_backup_arn(&arn_with("1")).unwrap_err(), + BackupArnError::Id + ); + assert_eq!( + check_backup_arn(&arn_with("a4b1d6c2-0000-4000-8000-000000000000")).unwrap_err(), + BackupArnError::Id + ); + for invalid in [ + "arn:aws:dynamodb:us-east-1:123456789012:table/Music/backup/01489602797149-73d8d5bc:x", + "arn:aws:dynamodb:us-east-1:123456789012:table/Music/backup/01489602797149-73d8d5bc/x", + "arn:aws:dynamodb:US-EAST-1:123456789012:table/Music/backup/01489602797149-73d8d5bc", + "arn:aws:dynamodb:us-east-1:12345678901x:table/Music/backup/01489602797149-73d8d5bc", + "arn:aws:s3:us-east-1:123456789012:table/Music/backup/01489602797149-73d8d5bc", + ] { + assert_eq!( + check_backup_arn(invalid).unwrap_err(), + BackupArnError::Invalid, + "{invalid}" + ); + } + // Over the service maximum, with the trimmed length reported rather + // than the input. + let long = arn_with(&format!("{}-73d8d5bc", "1".repeat(1100))); + let err = BackupArn::parse(&long).unwrap_err().to_string(); + assert!(err.contains("bytes>"), "{err}"); + assert!(err.len() < 200, "{}", err.len()); + } + + #[test] + fn format_is_the_one_string() { + let arn = format_backup_arn( + "aws", + "us-east-1", + "123456789012", + "Music", + "01489602797149-73d8d5bc", + ); + assert_eq!(arn, ARN); + assert_eq!(BackupArn::parse(ARN).unwrap().to_arn_string(), arn); + assert_eq!(trim_backup_arn(" \t\n"), ""); + assert_eq!(trim_backup_arn(&format!("\t{ARN}\n")), ARN); + assert_eq!( + trim_backup_arn("\u{a0}x"), + "\u{a0}x", + "non-ASCII whitespace is not trimmed" + ); + } + + #[test] + fn ids_are_fourteen_zero_padded_digits_and_eight_hex() { + assert_eq!( + format_backup_id(1_489_602_797_149, 0x73d8_d5bc), + "01489602797149-73d8d5bc" + ); + assert_eq!(format_backup_id(5, 1), "00000000000005-00000001"); + assert!(is_backup_id(&format_backup_id(5, 1))); + assert_eq!(backup_id_epoch_millis(ARN), Some(1_489_602_797_149)); + assert!((backup_creation_date_time(ARN).unwrap() - 1_489_602_797.149).abs() < 1e-6); + for bad in [ + "1489602797149-73d8d5bc", + "01489602797149-73D8D5BC", + "01489602797149", + "0148960279714x-73d8d5bc", + "01489602797149-73d8d5b", + "a4b1d6c2-0000-4000-8000-000000000000", + ] { + assert_eq!(backup_id_epoch_millis(bad), None, "{bad}"); + } + } + + #[test] + fn new_rejects_bad_components() { + const ID: &str = "01489602797149-73d8d5bc"; + assert!(BackupArn::new("aws", "us-east-1", "1234", "Music", "a/b").is_err()); + assert!(BackupArn::new("aws", "us-east-1", "12/34", "Music", ID).is_err()); + assert!(BackupArn::new("aws", "us-east-1", "1234", "Music", "...").is_err()); + assert!(BackupArn::new("aws", "us-east-1", "1234", "...", ID).is_err()); + // Short names are the engine's call (its minimum length is + // configurable); the ARN accepts them. + BackupArn::new("aws", "us-east-1", "1234", "ab", ID).unwrap(); + // The error names the failing input when it is short enough to show + // whole (`truncated`: 80 bytes). + let err = BackupArn::parse(&format!( + "arn:aws:dynamodb:us-east-1:1:table/Mu sic/backup/{ID}" + )) + .unwrap_err(); + assert!(err.to_string().contains("Mu sic"), "{err}"); + assert!(BackupArn::new("aws", "us:east", "1234", "Music", ID).is_err()); + assert!(BackupArn::new("", "us-east-1", "1234", "Music", ID).is_err()); + // A component with surrounding whitespace would parse back trimmed. + // Inner components are refused by the character rules; the round-trip + // comparison is what catches whitespace at the two ends of the string. + assert!(BackupArn::new("aws", "us-east-1", "1234", "Music ", ID).is_err()); + assert!(BackupArn::new("aws", "us-east-1", " 1234", "Music", ID).is_err()); + assert!(BackupArn::new(" aws", "us-east-1", "1234", "Music", ID).is_err()); + assert!(BackupArn::new("aws", "us-east-1", "1234", "Music", &format!("{ID}\n")).is_err()); + assert!(BackupArn::new("aws", "us-east-1", "1234", "Music", &format!("{ID} ")).is_err()); + // Dots inside names are fine; DynamoDB allows them. + BackupArn::new("aws", "us-east-1", "1234", "a.b.c", ID).unwrap(); + } + + #[test] + fn fixed_object_names() { + assert_eq!(EXTENDDB_MANIFEST_FILE, "extenddb-manifest.json"); + assert_eq!(MANIFEST_CHECKSUM_FILE, "extenddb-manifest.sha256"); + assert_eq!(SUMMARY_MANIFEST_FILE, "manifest-summary.json"); + assert_eq!(FILES_MANIFEST_FILE, "manifest-files.json"); + assert_eq!(DATA_PREFIX, "data/"); + } +} diff --git a/crates/core/src/backup_format/manifest.rs b/crates/core/src/backup_format/manifest.rs new file mode 100644 index 000000000..28658df85 --- /dev/null +++ b/crates/core/src/backup_format/manifest.rs @@ -0,0 +1,3800 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! The `extenddb-manifest.json` types: the `ExtendDB` extension to the +//! `DynamoDB` export layout, and the commit record of a backup. +//! +//! The manifest is written last; its presence marks a complete backup. It +//! carries everything a restore needs that the `DynamoDB` export manifests do +//! not: the full source table definition, the snapshot descriptor, the item +//! order guarantee, and a sha256 per data file. +//! +//! Integrity of the manifest itself is a sidecar object, +//! `extenddb-manifest.sha256`, in `sha256sum` format: the lowercase hex +//! sha256 of the exact bytes of `extenddb-manifest.json`, two spaces, the +//! file name, newline. The writer computes it from the bytes it is about to +//! store and writes the sidecar before the manifest; a reader hashes the +//! bytes it fetched and compares, and an operator can run +//! `sha256sum -c extenddb-manifest.sha256`. The reader accepts that one line, +//! with or without its final newline, and refuses any other shape, a bare +//! digest included, which `sha256sum -c` refuses too +//! ([`verify_manifest_checksum`]). Hashing stored bytes avoids +//! defining a canonical JSON form: an earlier revision hashed a re-serialized +//! struct with the checksum field blanked, which tied the digest to one JSON +//! library's float formatting and field order and would have failed for any +//! field an older reader did not model. +//! +//! Optional members of [`SourceTableSchema`] are always serialized, as `null` +//! when absent, for human readers and non-Rust tools; index lists are plain +//! arrays, empty when the table has none. The index lists, `vector_indexes`, +//! `stream_specification` and `time_to_live` are required members: a +//! document without one is refused (no v1 writer omits them, and a missing +//! `vector_indexes` would read as "the source had no vector support" and +//! drop vector indexes silently). This build's reader accepts a missing +//! scalar optional member (`provisioned_throughput`, `on_demand_throughput`, +//! `table_class`, `sse_description`) as `null` (`#[serde(default)]`), so +//! "the writer did not know this field" is signalled by `format_version`, +//! not by absence. `vector_indexes` is the one member whose `null` carries +//! meaning, see its docs. `stream_specification` and `time_to_live` record +//! what `DescribeTable` and `DescribeTimeToLive` reported for the source; +//! `DescribeBackup` reports them as `SourceTableFeatureDetails` and a +//! restore does not reproduce them (the restored table has no stream and no +//! TTL, as on the service). Nested members reuse the `DynamoDB` wire types, as on the wire. Most +//! of their optional members are omitted when absent; `OnDemandThroughput` +//! is not, and writes `null` for an absent `MaxReadRequestUnits` or +//! `MaxWriteRequestUnits` (`{"MaxReadRequestUnits":100, +//! "MaxWriteRequestUnits":null}`), which is its wire serialization and part +//! of format version 1. The checked-in golden manifests under +//! `testdata/extenddb-manifest/` pin the exact bytes, and two tests pin each +//! type's members and each enum's values, so a change to one of those +//! shared types fails a test here (the testdata README says which test +//! catches which change). +//! +//! Casing: top-level and `source_table` member names are `snake_case`; +//! `item_order` values are `snake_case`; members of the nested wire types +//! and their enum values (`PROVISIONED`, `HASH`) keep the `DynamoDB` API +//! spelling. Frozen with format version 1. +//! +//! Grammar facts the field docs do not state, which another implementation +//! must reproduce: the document is UTF-8 JSON without a byte order mark. +//! `format_version` is a JSON integer in `u32` range; `backup_size_bytes`, +//! `source_table.item_count` and `source_table.table_size_bytes` are `i64` +//! (not negative; a data-file total of 2^63 or more is refused as a size +//! mismatch); each data file's `item_count`, `bytes` and +//! `uncompressed_bytes` are `u64`; a vector index's `Dimensions` is `u32`; a +//! JSON `-1` or `1.5` in any of them is a parse error. The two timestamps +//! are JSON numbers read as `f64`, so a JSON integer (`1757500000`) is +//! accepted where a writer omits the fraction. The backup id in `backup_arn` +//! is the 14-digit form `<14 digits of epoch milliseconds>-<8 lowercase +//! hex>` ([`super::format_backup_id`]); the 13-digit and bare ids earlier +//! releases issued, which [`super::is_backup_id`] accepts for API requests, +//! are refused here because they carry no instant to check +//! `backup_creation_date_time` against. Within a `key_range` endpoint an +//! `S` or `B` value is non-empty, a `B` value is standard base64 with `=` +//! padding in canonical form (`AQ==`, not `AQ` or `AR==`), and an `N` value +//! is a `DynamoDB` number in the normalized form [`super::key_order`] +//! states; a data file's `key_range` is `null` or absent on an unordered +//! manifest and an object on a key-ordered one. Member order within an +//! object does not matter to the reader; the writer emits the order the +//! struct declares, which the golden fixtures pin. + +use std::cmp::Ordering; + +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; + +use super::FormatError; +use super::data_file::{DataFileDecoder, hex_encode}; +use super::error::truncated; +use super::key_order::{CheckedKey, HeldKey, KeyOrder}; +use super::layout::{ + BackupArn, DATA_PREFIX, EXTENDDB_MANIFEST_FILE, MANIFEST_CHECKSUM_FILE, + backup_creation_date_time, backup_id_epoch_millis, is_region, lower_hex, +}; +use super::strict_item::item_without_duplicates_raw; +use crate::types::{ + AttributeDefinition, BillingMode, GsiInput, Item, KeySchemaElement, KeyType, LsiInput, + OnDemandThroughput, ProvisionedThroughput, SseDescription, StreamSpecification, + TimeToLiveDescription, VectorIndexSpecification, +}; + +/// The backup format version this build reads and writes. +pub const CURRENT_FORMAT_VERSION: u32 = 1; + +/// How the source backend took its consistent snapshot. +/// +/// `kind` names the mechanism (for example `postgres_repeatable_read`) and +/// `marker` records the backend-specific position, kept for diagnostics and +/// for a later change-log archive to anchor against. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct SnapshotDescriptor { + /// The snapshot mechanism, named by the backend that took it; the + /// backends in this tree write one of the `KIND_*` constants on this + /// type. It is a free string on the wire: the reader does not validate + /// it, so a later backend can name a new mechanism without a format + /// change, and a restore must not branch on it (every kind is a + /// consistent snapshot; the data files mean the same whatever the + /// mechanism). It is recorded for diagnostics. + pub kind: String, + /// Backend-specific snapshot position. + pub marker: String, +} + +impl SnapshotDescriptor { + /// PostgreSQL: a `REPEATABLE READ READ ONLY` transaction; the marker is + /// `pg_current_snapshot()` text. + pub const KIND_POSTGRES_REPEATABLE_READ: &'static str = "postgres_repeatable_read"; + /// SQLite: a WAL read transaction; the marker is empty (a WAL snapshot has + /// no addressable position). + pub const KIND_SQLITE_WAL_READ_TRANSACTION: &'static str = "sqlite_wal_read_transaction"; + /// MongoDB: a session with snapshot read concern; the marker is the + /// cluster time `:`. + pub const KIND_MONGODB_SNAPSHOT_READ_CONCERN: &'static str = "mongodb_snapshot_read_concern"; +} + +/// Order of items within and across a backup's data files. +/// +/// For `KeyOrdered`, the manifest parser proves only that the files' ranges +/// ascend without overlap. [`ExtenddbManifest::reader`] also runs a +/// [`KeyRangeCheck`] over each file's items as it decodes them: that is what +/// ties a file's items to its range and so rules out a duplicate key across +/// files. The writer runs a [`KeyRangeTracker`] over the items it encodes, +/// so it does not publish a file the check refuses. +/// +/// A restore may exploit `KeyOrdered` (SQLite inserts into a fresh table 3 to +/// 4 times faster when rows arrive in primary-key order; measured with the +/// bundled SQLite on a 1M-item fixture, ratio only) but must produce the same +/// table either way. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ItemOrder { + /// Items ascend in the order [`super::key_order::KeyOrder`] defines (hash + /// key by its text encoding, bytewise; range key `S`/`B` bytewise, `N` + /// numeric), strictly within each file and from one file to the next; + /// every [`DataFileEntry::key_range`] is set and the ranges do not + /// overlap. Only hash-only and single-range-key tables can be ordered. + KeyOrdered, + /// No ordering guarantee. + Unordered, +} + +/// The source table definition captured at backup time. +/// +/// Field types reuse the existing wire types, so the nested JSON matches what +/// the `DynamoDB` API itself carries (`PascalCase` members inside each element). +/// Timestamps are `f64` epoch seconds, the same representation +/// [`crate::types::BackupDetails`] uses. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct SourceTableSchema { + /// Name of the source table. + pub table_name: String, + /// Unique id of the source table. + pub table_id: String, + /// ARN of the source table. + pub table_arn: String, + /// When the source table was created, as epoch seconds. + pub table_creation_date_time: f64, + /// Base table key schema. + pub key_schema: Vec, + /// Attribute definitions for all key attributes. + pub attribute_definitions: Vec, + /// Billing mode at backup time. + pub billing_mode: BillingMode, + /// Provisioned throughput; `null` when the table is on demand. + #[serde(default)] + pub provisioned_throughput: Option, + /// On-demand throughput limits; `null` when none are configured. + #[serde(default)] + pub on_demand_throughput: Option, + /// Table class; `null` when the source reported none. + #[serde(default)] + pub table_class: Option, + /// Server-side encryption as `DescribeTable` reports it; `null` when the + /// source reported none. A restore maps it to an `SSESpecification`. + #[serde(default)] + pub sse_description: Option, + /// Global secondary indexes defined on the source table; empty when none. + /// Required (module docs). + pub global_secondary_indexes: Vec, + /// Local secondary indexes defined on the source table; empty when none. + /// Required (module docs). + pub local_secondary_indexes: Vec, + /// Vector indexes, as creation specifications. Three states: `null`, the + /// source backend has no vector support (a target with vector support + /// restores the table without vector indexes); `[]`, the source supports + /// vector search and the table had no vector indexes; a non-empty list, + /// which a target without vector support refuses to restore rather than + /// dropping them. Required: a missing member is refused, not read as + /// `null` (module docs). + #[serde(deserialize_with = "Option::deserialize")] + pub vector_indexes: Option>, + /// The source table's stream specification as `DescribeTable` reported + /// it at backup time; `null` when the table had none. `DescribeBackup` + /// reports it as `SourceTableFeatureDetails.StreamDescription`; a + /// restore does not create a stream. Required: a missing member is + /// refused (module docs). + #[serde(deserialize_with = "Option::deserialize")] + pub stream_specification: Option, + /// The source table's TTL description as `DescribeTimeToLive` reported + /// it at backup time; `null` when the source reported none. + /// `DescribeBackup` reports it as + /// `SourceTableFeatureDetails.TimeToLiveDescription`; a restore does not + /// enable TTL. Required: a missing member is refused (module docs). + #[serde(deserialize_with = "Option::deserialize")] + pub time_to_live: Option, + /// Item count at backup time; not negative. + pub item_count: i64, + /// Table size in bytes at backup time; not negative. + pub table_size_bytes: i64, +} + +/// First and last base-table key in a data file, as `DynamoDB` JSON items +/// holding exactly the key attributes (enforced by +/// [`ExtenddbManifest::validate_data_files`]). Required when the backup is +/// [`ItemOrder::KeyOrdered`] and refused when it is unordered. The parser +/// refuses overlapping ranges before any file is downloaded; that alone does +/// not rule out a duplicate primary key across files, since nothing in the +/// manifest ties a file's items to its range (two files with ranges `A..F` +/// and `G..M` could each hold `A..M`). [`ExtenddbManifest::reader`] runs a +/// [`KeyRangeCheck`] over each file's items: strict ascent within every +/// file, first and last item equal to the range's endpoints, and +/// non-overlapping ranges together mean no key appears twice in the backup. +/// The writer records the range with a [`KeyRangeTracker`]. +/// +/// An `N` endpoint must be written in normalized form (`5`, not `5.0` or +/// `5E0`): the parser reads the string as written, before the +/// `AttributeValue` deserializer normalizes it, and refuses any other +/// spelling, so the bytes the sidecar vouches for have one reading. +/// +/// An endpoint that names one attribute twice is refused at parse time: a +/// map type would keep the last value silently, and a reader in another +/// language might keep the first and compute a different range from the +/// bytes the sidecar vouches for. The data file reader refuses the same +/// shape in an item line with the same deserializer. An endpoint value is +/// one JSON object with exactly one member, its type descriptor: a value +/// that repeats its descriptor (`{"S":"0","S":"A"}`) or names two is refused +/// at parse time for the same reason, as a data file line of that shape is. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct KeyRange { + /// Key of the first item in the file. + #[serde(deserialize_with = "key_item_as_written")] + pub first: Item, + /// Key of the last item in the file. + #[serde(deserialize_with = "key_item_as_written")] + pub last: Item, +} + +/// The writer's side of a key-ordered data file: it takes every item the +/// writer encodes, in file order, and gives the file's [`KeyRange`] at the +/// end. +/// +/// [`Self::push`] refuses an item whose key [`KeyOrder::endpoint_key`] +/// refuses (an `N` key `abc`, for one) and an item that is not strictly +/// after the one before it, so a snapshot stream that is out of order, or +/// repeats a key, fails the backup instead of producing a file every +/// restore refuses. [`KeyRangeCheck`] is this tracker plus the comparison +/// with the recorded range, so the writer and the reader apply one ascent +/// rule. The tracker keeps two keys, the first and the latest; the latest +/// is overwritten in place and the item is never copied. +/// +/// One tracker serves a whole backup: [`Self::next_file`] closes the file +/// so far, returning its range, and keeps the latest key, so the next +/// file's first item is checked strictly after the previous file's last. +/// A writer that calls it between files is refused at that item, with +/// nothing of the new file encoded, where a writer with one tracker per +/// file is refused only at [`super::BackupDocuments::build`] +/// ([`ExtenddbManifest::validate_data_files`] compares the ranges), after +/// every file is stored. [`Self::finish`] closes the last file. +#[derive(Debug, Clone)] +pub struct KeyRangeTracker { + order: KeyOrder, + /// The current file's first key; `None` until it has an item. + first: Option, + /// The latest key taken, of this file or, until this file has an item, + /// of the previous one. + latest: Option, + /// Items taken in the current file. + items: u64, +} + +impl KeyRangeTracker { + /// A tracker for the files of one backup under the table's key order. + #[must_use] + pub fn new(order: &KeyOrder) -> Self { + Self { + order: order.clone(), + first: None, + latest: None, + items: 0, + } + } + + /// Take the next item of the file. + /// + /// # Errors + /// + /// [`FormatError::InvalidKey`] when the item's key fails + /// [`KeyOrder::endpoint_key`] (a key attribute missing or of the wrong + /// type, an empty `S` or `B`, an `N` that is not a valid number in + /// normalized form), and [`FormatError::KeyOrderViolation`] when the item + /// is not strictly after the one before it; for the first item of a file + /// after [`Self::next_file`], the one before it is the previous file's + /// last item and the message names that boundary and its key. Nothing + /// is recorded for a refused item. + pub fn push(&mut self, item: &Item) -> Result<(), FormatError> { + self.push_key(item).map(|_| ()) + } + + /// [`Self::push`], returning the key now held as the latest. + fn push_key(&mut self, item: &Item) -> Result<&HeldKey, FormatError> { + let key = self.order.checked_key(item)?; + self.push_checked(key) + } + + /// [`Self::push_key`] for a key already checked under this order. + fn push_checked(&mut self, key: CheckedKey<'_>) -> Result<&HeldKey, FormatError> { + let n = self.items + 1; + let latest = match self.latest.take() { + None => HeldKey::new(key), + Some(mut latest) => { + if latest.as_checked().cmp(&key) != Ordering::Less { + let message = if self.items == 0 { + let shown = + serde_json::to_string(&latest.to_item(&self.order)).unwrap_or_default(); + format!( + "item 1 is not strictly after the previous file's last item, key {}", + truncated(&shown) + ) + } else { + format!("item {n} is not strictly after item {}", n - 1) + }; + self.latest = Some(latest); + return Err(FormatError::KeyOrderViolation(message)); + } + latest.assign(key); + latest + } + }; + if self.items == 0 { + self.first = Some(latest.clone()); + } + self.items = n; + Ok(self.latest.insert(latest)) + } + + /// Number of items taken in the current file. + #[must_use] + pub fn item_count(&self) -> u64 { + self.items + } + + /// Close the current file and start the next: the closed file's range, + /// its first and latest keys as key-only items, or `None` when it had no + /// items (nothing changes then). The latest key is kept, so the next + /// file's first item must be strictly after it; [`Self::item_count`] + /// starts again at 0. + #[must_use] + pub fn next_file(&mut self) -> Option { + let latest = self.latest.as_ref()?; + let first = self.first.take()?; + self.items = 0; + Some(KeyRange { + first: first.to_item(&self.order), + last: latest.to_item(&self.order), + }) + } + + /// The last file's range, its first and latest keys as key-only items, + /// or `None` when no item was taken since the last [`Self::next_file`] + /// (the format has no empty data file). + #[must_use] + pub fn finish(self) -> Option { + Some(KeyRange { + first: self.first?.to_item(&self.order), + last: self.latest?.to_item(&self.order), + }) + } +} + +/// Checks one data file's items, in file order, against the file's +/// [`KeyRange`] in a key-ordered backup. [`ExtenddbManifest::reader`] runs +/// it for every item it decodes and [`Self::finish`] at the end of the file; +/// it is public for callers that decode otherwise. +/// +/// It is a [`KeyRangeTracker`] (so it refuses an item whose key +/// [`KeyOrder::endpoint_key`] refuses, wherever it falls in the file, and +/// an item that is not strictly after the one before it) plus the recorded +/// range: it refuses a first item whose key is not `range.first`, an item +/// after `range.last` (as soon as it is seen), and at the end a last item +/// whose key is not `range.last` or a file with no items. Together with the +/// manifest's non-overlapping ranges +/// ([`ExtenddbManifest::validate_data_files`]) this means the backup's keys +/// ascend strictly across all files, so no key appears twice; the manifest +/// alone cannot prove that. Keys compare in [`KeyOrder::compare`]'s order, +/// so "equal" means equal on the key attributes; other attributes are not +/// looked at. The recorded endpoints are checked and parsed once, in +/// [`Self::new`]; a range [`KeyOrder::validate_key_item`] refuses makes +/// every call return that error. +#[derive(Debug, Clone)] +pub struct KeyRangeCheck { + tracker: KeyRangeTracker, + /// The recorded `first` and `last`, or the reason they are not keys. + range: Result<(HeldKey, HeldKey), String>, +} + +impl KeyRangeCheck { + /// A check for one file with recorded range `range`, under the table's + /// key order. + #[must_use] + pub fn new(order: &KeyOrder, range: &KeyRange) -> Self { + let endpoint = |which: &str, key: &Item| { + order + .validate_key_item(key) + .and_then(|()| order.checked_key(key).map(HeldKey::new)) + .map_err(|e| { + let reason = match e { + FormatError::InvalidKey(reason) => reason, + other => other.to_string(), + }; + let shown = serde_json::to_string(key).unwrap_or_default(); + format!( + "key_range {which} {} is not a key of this order: {reason}", + truncated(&shown) + ) + }) + }; + let range = endpoint("first", &range.first) + .and_then(|first| Ok((first, endpoint("last", &range.last)?))); + Self { + tracker: KeyRangeTracker::new(order), + range, + } + } + + fn range( + range: &Result<(HeldKey, HeldKey), String>, + ) -> Result<&(HeldKey, HeldKey), FormatError> { + range + .as_ref() + .map_err(|reason| FormatError::InvalidKey(reason.clone())) + } + + /// Check the next item of the file. + /// + /// # Errors + /// + /// [`FormatError::KeyOrderViolation`] when the item breaks the file's + /// order or range (type docs), [`FormatError::InvalidKey`] when its key + /// fails [`KeyOrder::endpoint_key`]: a key attribute missing or of the + /// wrong type, an empty `S` or `B`, or an `N` value that is not a valid + /// number in normalized form (a key-ordered file cannot hold a key its + /// writer could not record as an endpoint). As for + /// [`KeyRangeTracker::push`], nothing is recorded for a refused item: + /// the range comparisons run before the tracker takes the key. + pub fn check(&mut self, item: &Item) -> Result<(), FormatError> { + let (first, last) = Self::range(&self.range)?; + let key = self.tracker.order.checked_key(item)?; + let n = self.tracker.items + 1; + if n == 1 && key.cmp(&first.as_checked()) != Ordering::Equal { + return Err(FormatError::KeyOrderViolation( + "the first item's key is not the file's key_range first".to_owned(), + )); + } + if key.cmp(&last.as_checked()) == Ordering::Greater { + return Err(FormatError::KeyOrderViolation(format!( + "item {n} is after the file's key_range last" + ))); + } + self.tracker.push_checked(key).map(|_| ()) + } + + /// Check the end of the file: it had items and the last one is + /// `range.last`. + /// + /// # Errors + /// + /// [`FormatError::KeyOrderViolation`] when the file had no items or its + /// last item is not the range's `last`, and the [`Self::new`] error for + /// a range that is not two keys. + pub fn finish(&self) -> Result<(), FormatError> { + let (_, last) = Self::range(&self.range)?; + let Some(latest) = &self.tracker.latest else { + return Err(FormatError::KeyOrderViolation( + "the file has a key_range and no items".to_owned(), + )); + }; + if latest.as_checked().cmp(&last.as_checked()) != Ordering::Equal { + return Err(FormatError::KeyOrderViolation(format!( + "the last item (item {}) is not the file's key_range last", + self.tracker.items + ))); + } + Ok(()) + } +} + +/// One data file read for a restore, from [`ExtenddbManifest::reader`]: a +/// [`DataFileDecoder`] whose items, in a key-ordered backup, have passed the +/// file's [`KeyRangeCheck`], and whose end of stream runs the check's +/// [`KeyRangeCheck::finish`] after the decoder's own end-of-stream checks. +/// +/// As with the decoder, an item is unverified until the stream ends (or the +/// source was verified first, [`ExtenddbManifest::reader_prehashed`]); any +/// error ends the stream, every later [`Self::next_item`] returns +/// [`FormatError::Decode`] naming the earlier failure, and the [`Iterator`] +/// impl yields the first error once and then `None`. +pub struct DataFileReader { + decoder: DataFileDecoder, + check: Option, + /// The end of stream was reached and checked. + ended: bool, + /// An error was returned; the iterator yields nothing more. + errored: bool, + /// Set by the first error of the range check, replayed by every later + /// call (the decoder replays its own). + check_failed: Option<&'static str>, +} + +impl DataFileReader { + fn new(decoder: DataFileDecoder, check: Option) -> Self { + Self { + decoder, + check, + ended: false, + errored: false, + check_failed: None, + } + } + + /// The next item, or `Ok(None)` once the decoder's end-of-stream checks + /// and the key range check's [`KeyRangeCheck::finish`] have passed. + /// + /// # Errors + /// + /// The errors of [`DataFileDecoder::next_item`] and of + /// [`KeyRangeCheck::check`] and [`KeyRangeCheck::finish`]. + pub fn next_item(&mut self) -> Result, FormatError> { + if let Some(reason) = self.check_failed { + return Err(FormatError::Decode(format!( + "data file reader failed earlier: {reason}" + ))); + } + if self.ended { + return Ok(None); + } + let checked = match self.decoder.next_item() { + Err(e) => { + self.errored = true; + return Err(e); + } + Ok(Some(item)) => match &mut self.check { + Some(check) => check.check(&item).map(|()| Some(item)), + None => Ok(Some(item)), + }, + Ok(None) => { + self.ended = true; + match &self.check { + Some(check) => check.finish().map(|()| None), + None => Ok(None), + } + } + }; + if let Err(e) = &checked { + self.errored = true; + self.check_failed = Some(match e { + FormatError::KeyOrderViolation(_) => "key order violation", + _ => "invalid key", + }); + } + checked + } +} + +impl Iterator for DataFileReader { + type Item = Result; + + /// Fused, as [`DataFileDecoder`]'s: the first error once, then `None`. + fn next(&mut self) -> Option { + if self.errored { + return None; + } + self.next_item().transpose() + } +} + +impl std::iter::FusedIterator for DataFileReader {} + +/// `deserialize_with` target for a [`KeyRange`] endpoint: an item read as +/// [`item_without_duplicates`] reads it (a repeated top-level name refused), +/// each value read from the text as its one type descriptor and payload (an +/// empty value, or one with a second descriptor, `{"S":"0","S":"A"}` +/// included, refused before a value is built), with each `N` value checked +/// as written. A valid number that is not in normalized form is refused; the +/// `AttributeValue` deserializer would rewrite it, so the parsed range and +/// the stored bytes would disagree. An invalid number is left for +/// [`KeyOrder::validate_key_item`] to refuse. +fn key_item_as_written<'de, D>(deserializer: D) -> Result +where + D: serde::Deserializer<'de>, +{ + use crate::validation::number::validate_and_normalize_number; + use serde::de::Error as _; + let item = item_without_duplicates_raw(deserializer)?; + let mut out = Item::new(); + for (name, (descriptor, payload)) in item { + if descriptor == "N" + && let Some(n) = payload.as_str() + && let Ok(normalized) = validate_and_normalize_number(n) + && normalized != n + { + return Err(D::Error::custom(format!( + "key_range attribute {:?} N value {:?} is not in normalized form ({:?})", + truncated(&name), + truncated(n), + truncated(&normalized) + ))); + } + let raw = serde_json::Value::Object(std::iter::once((descriptor, payload)).collect()); + let value = serde_json::from_value(raw).map_err(D::Error::custom)?; + out.insert(name, value); + } + Ok(out) +} + +/// One data file recorded in the manifest. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct DataFileEntry { + /// Store key of the file, relative to the backup prefix + /// (for example `data/000001.json.gz`). + pub key: String, + /// Number of items in the file. + pub item_count: u64, + /// Lowercase hex sha256 of the file's compressed bytes. + pub sha256: String, + /// Base64 MD5 of the file's compressed bytes; the same value the + /// `manifest-files.json` line for this file carries as `md5Checksum`. + pub md5: String, + /// Size of the file's compressed bytes. A decoder enforces it as it + /// reads, so a file longer than recorded is refused once the reader has + /// consumed one byte more than this. + pub bytes: u64, + /// Size of the file's decompressed stream: every line with its newline, + /// except that the reader contract (`data_file` module docs) lets the + /// last line's be absent. A decoder enforces it incrementally, so a corrupt or hostile file is rejected + /// before it is fully expanded. + pub uncompressed_bytes: u64, + /// Key range covered by the file. Required for a key-ordered backup and + /// must be `null` for an unordered one; both enforced by + /// [`ExtenddbManifest::validate_data_files`]. + #[serde(default)] + pub key_range: Option, +} + +impl DataFileEntry { + /// The entry for a data file the encoder produced: its digests, sizes + /// and count copied from `checksums`, so the manifest cannot record a + /// value the file does not have, and `key_range` as the writer observed + /// it ([`KeyRangeTracker::finish`]; `None` on an unordered backup). + /// [`super::BackupDocuments::build`] derives the `manifest-files.json` + /// line from the result, so the two manifests carry one `md5` and one + /// `item_count` by construction. + #[must_use] + pub fn from_checksums( + key: String, + checksums: &super::data_file::DataFileChecksums, + key_range: Option, + ) -> Self { + Self { + key, + item_count: checksums.item_count, + sha256: checksums.sha256_hex.clone(), + md5: checksums.md5_base64.clone(), + bytes: checksums.compressed_bytes, + uncompressed_bytes: checksums.uncompressed_bytes, + key_range, + } + } + + /// Validate that `key` names a data file within the backup's `data/` + /// prefix and nowhere else. + /// + /// The key must be [`DATA_PREFIX`] followed by exactly one path segment + /// that matches the data-file naming pattern (`.json.gz`). This + /// rejects `..`, absolute paths, and any extra directory level, so a + /// malicious manifest cannot point the restore path at a store key outside + /// the backup. Validation lives in the format layer, the natural home for + /// the layout rule, independent of any store-level sandboxing. + /// + /// # Errors + /// + /// Returns [`FormatError::InvalidDataFileKey`] when the key is not a + /// single data-file entry under the prefix. + pub fn validate_key(&self) -> Result<(), FormatError> { + let reject = |reason: &'static str| FormatError::InvalidDataFileKey { + key: truncated(&self.key), + reason, + }; + + let name = self + .key + .strip_prefix(DATA_PREFIX) + .ok_or_else(|| reject("must start with the data/ prefix"))?; + + if name.is_empty() { + return Err(reject("names no file under data/")); + } + // Exactly one segment: no further path separators, no traversal, no + // absolute component. + if name.contains('/') { + return Err(reject("must hold exactly one path segment under data/")); + } + if name.contains('\\') { + return Err(reject("must not contain a backslash")); + } + if name == "." || name == ".." || name.contains("..") { + return Err(reject("must not contain a path-traversal component")); + } + // The data-file naming pattern: one or more digits then .json.gz. + let digits = name + .strip_suffix(".json.gz") + .ok_or_else(|| reject("must end with .json.gz"))?; + if digits.is_empty() || !digits.bytes().all(|b| b.is_ascii_digit()) { + return Err(reject("file stem must be all digits")); + } + Ok(()) + } + + /// The expectations a decoder enforces for this file: every digest, + /// size and count the entry records (`sha256`, `md5`, `bytes`, + /// `item_count`, `uncompressed_bytes`). + #[must_use] + pub fn expectations(&self) -> super::data_file::DataFileExpectations { + super::data_file::DataFileExpectations { + sha256_hex: Some(self.sha256.clone()), + md5_base64: Some(self.md5.clone()), + compressed_bytes: Some(self.bytes), + item_count: Some(self.item_count), + uncompressed_bytes: Some(self.uncompressed_bytes), + } + } + + /// Streaming decoder over this entry's data file read from `compressed`, + /// with the entry's digests and counts as expectations. Validates the key + /// first. The digests are checked when the stream ends, so a caller that + /// must know them earlier hashes a local copy with + /// [`crate::backup_format::CompressedHasher`] first and decodes it with + /// [`Self::decoder_prehashed`]. A restore reads through + /// [`ExtenddbManifest::reader`], which wraps this decoder and adds the + /// key range check. + /// + /// # Errors + /// + /// Returns [`FormatError::InvalidDataFileKey`] when the key is not a + /// single data-file entry under the prefix. + pub fn decoder( + &self, + compressed: R, + ) -> Result, FormatError> { + self.validate_key()?; + Ok(super::data_file::DataFileDecoder::new( + compressed, + self.expectations(), + )) + } + + /// [`Self::decoder`] for a source whose `sha256`, `md5` and size the + /// caller has just verified against this entry (a + /// [`crate::backup_format::CompressedHasher`] pass and + /// [`crate::backup_format::DataFileExpectations::verify_digest`]): the + /// bytes are not hashed a second time, and the compressed size, the + /// uncompressed size, the item count and every line are still checked. + /// Safe when `compressed` yields exactly the bytes that were verified: + /// a local spool file this process wrote and hashed, or a file read in + /// place from a store root that only the service's user can open; not a + /// second download, which is a second set of bytes. A source that + /// differs is not refused by its digest; what remains is the compressed + /// size check, which refuses a file replaced by one of another length + /// and does not detect a replacement of the same length. The second hash + /// would cost about 0.43 µs per 318-byte item (sha256 and md5 at about + /// 150 MB/s of compressed bytes on a Xeon Platinum 8124M, which has no + /// SHA extensions), about 9% of decoding. + /// + /// # Errors + /// + /// Returns [`FormatError::InvalidDataFileKey`] when the key is not a + /// single data-file entry under the prefix. + pub fn decoder_prehashed( + &self, + compressed: R, + ) -> Result, FormatError> { + self.validate_key()?; + Ok(super::data_file::DataFileDecoder::prehashed( + compressed, + self.expectations(), + )) + } + + /// Decode this entry's in-memory data file into a `Vec`, validating the + /// key first and verifying the digests before the first item. For tests + /// and files known to be small; see [`Self::decoder`] for the streaming + /// form. + /// + /// # Errors + /// + /// Returns [`FormatError::InvalidDataFileKey`] when the key is not a + /// single data-file entry under the prefix, and any error + /// [`crate::backup_format::decode_data_file`] can produce. + pub fn decode(&self, compressed: &[u8]) -> Result, FormatError> { + self.validate_key()?; + super::data_file::decode_data_file(compressed, self.expectations()) + } +} + +/// The `extenddb-manifest.json` document. +/// +/// Serde leaves unknown fields tolerated (no `deny_unknown_fields`) at every +/// level, so a reader does not fail on a member it does not know; the hard +/// gate is [`Self::check_supported`], which refuses a `format_version` this +/// build does not understand. +/// +/// Evolution rule: any change to the bytes a writer produces for the golden +/// fixtures' inputs (`testdata/extenddb-manifest/`) is a format version +/// bump, a new member included, whether or not an older reader could have +/// ignored it. A new member that is omitted when absent does not change +/// those bytes, nor does a new or renamed enum value no fixture uses; the +/// member list of every type and the value list of every enum in the +/// document (`AttributeValue`'s type descriptors included) are pinned by +/// tests (`nested_types_have_the_pinned_members`, +/// `enum_values_are_pinned`), so it fails there instead. A member is added by bumping [`CURRENT_FORMAT_VERSION`], +/// giving the member `#[serde(default)]` so the new reader still reads v1, +/// and writing `v{N}*.json` fixtures beside the v1 ones; every version's +/// fixtures are permanent parse tests. Readers accept +/// `1..=CURRENT_FORMAT_VERSION`, so a bump excludes older readers from +/// newer backups whatever the backup contains, and there is no +/// minimum-reader-version escape; that is the chosen trade (one rule, one +/// fixture set per version) over a per-member negotiation. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct ExtenddbManifest { + /// Format version; see [`CURRENT_FORMAT_VERSION`]. + pub format_version: u32, + /// ARN of the backup, in the shape + /// `arn::dynamodb:::table/
/backup/`. + pub backup_arn: String, + /// Name given to the backup at creation: 3 to 255 characters of + /// `[a-zA-Z0-9_.-]`, the service's `BackupName` rule, enforced on load + /// because the value is echoed to API clients. + pub backup_name: String, + /// The instant the snapshot pinned, as epoch seconds with millisecond + /// precision, the representation [`crate::types::BackupDetails`] uses. + /// It equals the instant encoded in `backup_arn`'s backup id + /// ([`super::backup_creation_date_time`]); the parser refuses a manifest + /// where the two differ, so the ARN, the manifest and the API carry one + /// time. + pub backup_creation_date_time: f64, + /// Total size of the backup's data files, in compressed bytes. + pub backup_size_bytes: i64, + /// The storage backend that produced the backup. + pub source_backend: String, + /// The `ExtendDB` version that produced the backup. + pub source_extenddb_version: String, + /// How the consistent snapshot was taken. + pub snapshot: SnapshotDescriptor, + /// Order of items in the data files. + pub item_order: ItemOrder, + /// The source table definition. + pub source_table: SourceTableSchema, + /// The data files that make up the backup, in the order a restore applies + /// them. This order is authoritative; a store listing is never used to + /// order files (zero-padding in [`super::data_file_key`] runs out at a + /// million files). + pub data_files: Vec, +} + +/// The content of the `extenddb-manifest.sha256` sidecar for a stored +/// manifest: ` extenddb-manifest.json\n`, the +/// `sha256sum` output format. +#[must_use] +pub(super) fn manifest_checksum(manifest_bytes: &[u8]) -> String { + format!( + "{} {EXTENDDB_MANIFEST_FILE}\n", + hex_encode(&Sha256::digest(manifest_bytes)) + ) +} + +/// Compare fetched manifest bytes against the sidecar's content, both as +/// the store returned them. +/// +/// The sidecar is exactly what the writer stores +/// ([`super::BackupDocuments::sidecar`]), the `sha256sum` +/// output line `<64 lowercase hex digits> extenddb-manifest.json\n`, with or +/// without the final `\n`. Anything else is refused rather than read +/// leniently: bytes that are not UTF-8, a bare digest (which `sha256sum -c` +/// refuses too, so the sidecar would not check from a shell), another file +/// name (the sidecar of a different object), a second line, upper-case hex +/// (the manifest records digests in lower case, and one spelling keeps the +/// sidecar comparable as bytes), `\r\n`, or `sha256sum`'s binary-mode `*` +/// marker. +/// +/// # Errors +/// +/// Returns [`FormatError::InvalidManifest`] when the sidecar is not in that +/// shape (an empty one and a non-UTF-8 one included) and +/// [`FormatError::ChecksumMismatch`] when the digests differ. +pub fn verify_manifest_checksum(manifest_bytes: &[u8], sidecar: &[u8]) -> Result<(), FormatError> { + let Ok(sidecar) = std::str::from_utf8(sidecar) else { + return Err(FormatError::InvalidManifest(format!( + "{MANIFEST_CHECKSUM_FILE} is not UTF-8 ({} bytes)", + sidecar.len() + ))); + }; + let line = sidecar.strip_suffix('\n').unwrap_or(sidecar); + let suffix = format!(" {EXTENDDB_MANIFEST_FILE}"); + let digest = line.strip_suffix(suffix.as_str()).unwrap_or(""); + if !is_sha256_hex(digest) { + return Err(FormatError::InvalidManifest(format!( + "{MANIFEST_CHECKSUM_FILE} is not one line of 64 lowercase hex digits, \ + two spaces and {EXTENDDB_MANIFEST_FILE}: {:?}", + truncated(sidecar) + ))); + } + let expected = digest.to_owned(); + let computed = hex_encode(&Sha256::digest(manifest_bytes)); + if expected == computed { + Ok(()) + } else { + Err(FormatError::ChecksumMismatch { + subject: "manifest", + expected, + computed, + }) + } +} + +/// The members a reader needs before it can decide whether to parse the +/// rest: the version gate runs on this so a document from a later version +/// fails with [`FormatError::UnsupportedFormatVersion`] rather than with a +/// parse error about a member this build does not know. +#[derive(Deserialize)] +struct VersionOnly { + format_version: u32, +} + +impl ExtenddbManifest { + /// Serialize to the bytes stored as `extenddb-manifest.json`, after + /// [`Self::validate`], so a writer cannot produce bytes + /// [`Self::from_json_bytes`] refuses (an un-normalized `N` endpoint built + /// in code, a 13-digit backup id, a `backup_size_bytes` off by one). + /// [`super::BackupDocuments::build`] calls it and hashes exactly these + /// bytes for the sidecar. + /// + /// # Errors + /// + /// The errors of [`Self::validate`], and [`FormatError::Json`] when + /// serialization fails. + pub(super) fn to_json_bytes(&self) -> Result, FormatError> { + self.validate()?; + Ok(serde_json::to_vec(self)?) + } + + /// Every check a parsed or built manifest must pass, in the order + /// [`Self::from_json_bytes`] runs them: [`Self::check_supported`], + /// [`Self::validate_source_table`], [`Self::validate_identity`], + /// [`Self::validate_data_files`]. [`super::BackupDocuments::build`] runs + /// it too, so the reader and the writer enforce one rule set and a + /// backup job does not repeat the sequence. + /// + /// # Errors + /// + /// The first failing check's error. + pub fn validate(&self) -> Result<(), FormatError> { + self.check_supported()?; + self.validate_source_table()?; + self.validate_identity()?; + self.validate_data_files() + } + + /// Parse stored manifest bytes: the version gate first, then the full + /// document, then [`Self::validate`]. Callers verify the sidecar + /// checksum against the same bytes first; see + /// [`verify_manifest_checksum`]. + /// + /// The gate: the document must be JSON and `format_version` a JSON + /// integer in `u32` range (otherwise the error is [`FormatError::Json`], + /// whatever the version); then the version is checked before any other + /// member is interpreted, so a document from a later version is reported + /// as unsupported rather than as a parse error about a member this build + /// does not know. The document is parsed twice for that, once for the + /// version and once in full; the version parse is under 10% of the + /// whole. The cost grows with the data files: with short keys (a 15-byte + /// `S` hash key and a 6-character `N` range key, about 325 bytes of + /// document per file) parsing takes about 6 to 12 µs and retains about + /// 1.6 KB per data file: 0.6 s on an idle host and 1.0 to 1.2 s on a + /// loaded one, and 155 MiB, at 10^5 files (release build, one shared + /// 16-core Xeon Platinum 8124M). Longer keys cost more: the two + /// endpoints are most of each entry. + /// + /// # Errors + /// + /// Returns [`FormatError::UnsupportedFormatVersion`] when the version is + /// not one this build reads ([`check_format_version`]), + /// [`FormatError::Json`] when the bytes do not parse (its message bounds + /// serde's text, which quotes the input), and the errors of + /// [`Self::validate`]. + pub fn from_json_bytes(bytes: &[u8]) -> Result { + // The gate reads only `format_version`, so a document from a later + // version fails with that reason instead of a deserialization error + // about a member this build does not know. + let version: VersionOnly = serde_json::from_slice(bytes)?; + check_format_version(version.format_version)?; + let manifest: Self = serde_json::from_slice(bytes)?; + manifest.validate()?; + Ok(manifest) + } + + /// The parsed backup ARN. + /// + /// # Errors + /// + /// Returns [`FormatError::InvalidArn`] when `backup_arn` is malformed. + pub fn backup_arn(&self) -> Result { + BackupArn::parse(&self.backup_arn) + } + + /// Check that the manifest names one table: `backup_arn` parses, its + /// table name equals `source_table.table_name`, and + /// `source_table.table_arn` is a table ARN for the same account and + /// table. `backup_arn` is the authoritative identity; the source-table + /// fields are checked against it so a restore never has two answers to + /// "which table is this". The table ARN's region is not compared: it is + /// recorded when the table is created and the backup ARN carries the + /// region configured when the backup is taken. + /// + /// # Errors + /// + /// [`FormatError::InvalidArn`] when `backup_arn` is malformed, + /// [`FormatError::InvalidManifest`] when the source-table fields disagree + /// with it. + pub fn validate_identity(&self) -> Result<(), FormatError> { + let arn = self.backup_arn()?; + // A machine-written field has one spelling; whitespace the parser + // would tolerate on a request is a defect here. + if self.backup_arn != arn.to_arn_string() { + return Err(FormatError::InvalidManifest(format!( + "backup_arn {:?} is not in canonical form", + truncated(&self.backup_arn) + ))); + } + let Some(id_millis) = backup_id_epoch_millis(arn.backup_id()) else { + return Err(FormatError::InvalidManifest(format!( + "backup id {:?} does not carry the backup's creation instant", + arn.backup_id() + ))); + }; + // Exact equality, as an f64: the writer derives the field from the + // id with `backup_creation_date_time`, so any other value is a + // different time, however close. An ms/1000 value has a short + // decimal form, which `serde_json` writes and reads back to the same + // bits. That does not hold for every f64 with the default parser + // (no `float_roundtrip`): a microsecond timestamp divided by 1e9, as + // `table_creation_date_time` can be, may come back one ulp off, + // which nothing refuses. + #[allow(clippy::cast_precision_loss)] + let expected = id_millis as f64 / 1000.0; + debug_assert_eq!( + Some(expected.to_bits()), + backup_creation_date_time(arn.backup_id()).map(f64::to_bits) + ); + if self.backup_creation_date_time.to_bits() != expected.to_bits() { + return Err(FormatError::InvalidManifest(format!( + "backup_creation_date_time {:?} does not match the instant in backup_arn \ + ({id_millis} ms)", + self.backup_creation_date_time + ))); + } + if !is_valid_backup_name(&self.backup_name) { + return Err(FormatError::InvalidManifest(format!( + "backup_name {:?} is not 3 to 255 characters of [a-zA-Z0-9_.-]", + truncated(&self.backup_name) + ))); + } + if arn.table_name() != self.source_table.table_name { + return Err(FormatError::InvalidManifest(format!( + "backup_arn names table {:?} but source_table.table_name is {:?}", + truncated(arn.table_name()), + truncated(&self.source_table.table_name) + ))); + } + // The table ARN is read with `crate::arn::parse`, the parser + // authorization uses (`extenddb_storage::util::is_backup_resource_arn` + // still splits ARNs on its own). The region is checked for the shape + // `check_backup_arn` requires, not compared (recorded at table + // creation, while the backup ARN carries the region configured at + // backup time). + let table_arn = &self.source_table.table_arn; + let matches = crate::arn::parse(table_arn).is_some_and(|t| { + t.sub.is_none() + && t.service == "dynamodb" + && t.partition == arn.partition() + && is_region(&t.region) + && t.account == arn.account_id() + && t.table == arn.table_name() + }); + if !matches { + return Err(FormatError::InvalidManifest(format!( + "source_table.table_arn {:?} is not a table ARN in partition {} for account {} \ + and table {:?} named by backup_arn", + truncated(table_arn), + truncated(arn.partition()), + truncated(arn.account_id()), + truncated(arn.table_name()) + ))); + } + Ok(()) + } + + /// The key order of the source table, when it has one. + /// + /// # Errors + /// + /// Returns [`FormatError::UnorderableKeySchema`] for a multi-part key. + pub fn key_order(&self) -> Result { + KeyOrder::resolve( + &self.source_table.key_schema, + &self.source_table.attribute_definitions, + ) + } + + /// The restore's reader of one of this manifest's data files, read from + /// `compressed`: [`DataFileEntry::decoder`]'s checks, and when the + /// manifest is key-ordered every item passed through a + /// [`KeyRangeCheck`] of `entry.key_range` and the check's + /// [`KeyRangeCheck::finish`] run at the end of the stream, after the + /// decoder's own end-of-stream checks. The caller does not construct the + /// check, so it cannot omit it. `entry` is one of `self.data_files`. + /// + /// # Errors + /// + /// [`FormatError::InvalidDataFileKey`] for a bad key, + /// [`FormatError::InvalidDataFileEntry`] when a key-ordered manifest's + /// entry has no `key_range` or an unordered one's has one, and + /// [`FormatError::UnorderableKeySchema`] when a key-ordered manifest's + /// schema has no order. + pub fn reader( + &self, + entry: &DataFileEntry, + compressed: R, + ) -> Result, FormatError> { + let check = self.range_check(entry)?; + Ok(DataFileReader::new(entry.decoder(compressed)?, check)) + } + + /// [`Self::reader`] over [`DataFileEntry::decoder_prehashed`], for a + /// source whose digests the caller has just verified: a spooled download + /// this process hashed, or a file read in place from a store root only + /// the service's user can open. The digests are not compared again, so a + /// source that changed after the hash is refused only by its compressed + /// size (another length), not when the replacement has the same length + /// (see there). + /// + /// # Errors + /// + /// Those of [`Self::reader`]. + pub fn reader_prehashed( + &self, + entry: &DataFileEntry, + compressed: R, + ) -> Result, FormatError> { + let check = self.range_check(entry)?; + Ok(DataFileReader::new( + entry.decoder_prehashed(compressed)?, + check, + )) + } + + fn range_check(&self, entry: &DataFileEntry) -> Result, FormatError> { + match (self.item_order, &entry.key_range) { + (ItemOrder::KeyOrdered, Some(range)) => { + Ok(Some(KeyRangeCheck::new(&self.key_order()?, range))) + } + (ItemOrder::KeyOrdered, None) => Err(FormatError::InvalidDataFileEntry { + key: truncated(&entry.key), + reason: "key_ordered manifest entry has no key_range", + }), + (ItemOrder::Unordered, None) => Ok(None), + (ItemOrder::Unordered, Some(_)) => Err(FormatError::InvalidDataFileEntry { + key: truncated(&entry.key), + reason: "unordered manifest entry has a key_range", + }), + } + } + + /// Refuse a manifest whose format version this build does not read + /// ([`check_format_version`]). + /// + /// # Errors + /// + /// Returns [`FormatError::UnsupportedFormatVersion`] when `format_version` + /// is 0 or later than [`CURRENT_FORMAT_VERSION`]. + pub fn check_supported(&self) -> Result<(), FormatError> { + check_format_version(self.format_version) + } + + /// Check what the manifest alone can prove about the source table: + /// `item_count` and `table_size_bytes` are not negative, + /// `table_creation_date_time` is finite (JSON cannot carry NaN or an + /// infinity; `backup_creation_date_time`, the other `f64`, is checked + /// against the backup id by [`Self::validate_identity`]), and the key + /// schema has the shape every table `CreateTable` accepts, on every + /// manifest whatever its `item_order`: at least one `HASH` element, + /// every `HASH` element before every `RANGE` element, each key attribute + /// named once, and each key attribute with exactly one definition in + /// `attribute_definitions`. The number of `HASH` and `RANGE` elements is + /// not bounded here: `CreateTable` accepts a multi-part key when the + /// operator enables the preview (`allow_multipart_table_keys`), such a + /// table is backed up `unordered` ([`KeyOrder::resolve`] refuses it an + /// order), and the restore's `CreateTable` validation decides whether + /// the target accepts the shape. Key endpoint lengths (`DynamoDB`'s + /// 2048-byte hash key and 1024-byte range key limits) are left to the + /// restore's item validation, which checks every item against the + /// target's limits. + /// + /// # Errors + /// + /// [`FormatError::InvalidManifest`] naming the negative count, the + /// non-finite timestamp or the key schema rule. + pub fn validate_source_table(&self) -> Result<(), FormatError> { + for (name, value) in [ + ("item_count", self.source_table.item_count), + ("table_size_bytes", self.source_table.table_size_bytes), + ] { + if value < 0 { + return Err(FormatError::InvalidManifest(format!( + "source_table.{name} is negative ({value})" + ))); + } + } + // JSON has no NaN or infinity: serde_json writes them as `null`, + // which the reader refuses as a type error. + if !self.source_table.table_creation_date_time.is_finite() { + return Err(FormatError::InvalidManifest(format!( + "source_table.table_creation_date_time is not a finite number ({})", + self.source_table.table_creation_date_time + ))); + } + let schema = &self.source_table.key_schema; + let first_range = schema + .iter() + .position(|e| e.key_type == KeyType::Range) + .unwrap_or(schema.len()); + if first_range == 0 { + return Err(FormatError::InvalidManifest( + "source_table.key_schema has no HASH element before its RANGE elements".to_owned(), + )); + } + if schema[first_range..] + .iter() + .any(|e| e.key_type == KeyType::Hash) + { + return Err(FormatError::InvalidManifest( + "source_table.key_schema lists a HASH element after a RANGE element".to_owned(), + )); + } + let mut names = std::collections::BTreeSet::new(); + for element in schema { + let name = element.attribute_name.as_str(); + if !names.insert(name) { + return Err(FormatError::InvalidManifest(format!( + "source_table.key_schema names attribute {:?} more than once", + truncated(name) + ))); + } + let definitions = self + .source_table + .attribute_definitions + .iter() + .filter(|d| d.attribute_name == name) + .count(); + if definitions != 1 { + return Err(FormatError::InvalidManifest(format!( + "source_table key attribute {:?} has {definitions} attribute definitions, \ + not one", + truncated(name) + ))); + } + } + Ok(()) + } + + /// Validate the data-file list: every key passes the layout rule and + /// appears once; every `sha256` is 64 lowercase hex digits and every `md5` + /// the 24-character base64 of 16 bytes, so a mismatch is reported here and not as a + /// decode-time checksum failure; a file with items has non-zero sizes; + /// `backup_size_bytes` equals the sum of the files' sizes (an empty table + /// is a manifest with no data files and `backup_size_bytes` 0); + /// for a key-ordered manifest every file carries a `key_range` whose + /// `first` and `last` are key-only items of the schema's shape with + /// `first <= last`, and the ranges ascend without overlap from one file + /// to the next; for an unordered manifest no file carries a `key_range` + /// (there is no order for it to describe, and a reader must not be + /// handed ranges it cannot check). + /// + /// Callers run this when a manifest is parsed, and the decode path runs + /// the same per-entry [`DataFileEntry::validate_key`] again before it + /// resolves a key, so a key that slips past one check is caught at the + /// other. Within-file order, and the match between a file's items and + /// its `key_range`, are checked while the restore streams the file + /// ([`KeyRangeCheck`], which [`Self::reader`] runs for a key-ordered + /// backup); this check covers what the manifest alone can prove. + /// + /// # Errors + /// + /// [`FormatError::InvalidDataFileKey`] for a bad or duplicate key; + /// [`FormatError::InvalidDataFileEntry`] for a malformed digest, a zero + /// size or item count, a missing range on a key-ordered manifest, or a + /// range on an unordered one; [`FormatError::InvalidManifest`] for a size mismatch or a range + /// violation; [`FormatError::InvalidKey`] for a malformed range endpoint; + /// [`FormatError::UnorderableKeySchema`] for a key-ordered manifest whose + /// schema has no order. + pub fn validate_data_files(&self) -> Result<(), FormatError> { + let mut seen = std::collections::BTreeSet::new(); + let mut total: u64 = 0; + for entry in &self.data_files { + entry.validate_key()?; + if !seen.insert(entry.key.as_str()) { + return Err(FormatError::InvalidDataFileKey { + key: truncated(&entry.key), + reason: "listed more than once", + }); + } + if self.item_order == ItemOrder::Unordered && entry.key_range.is_some() { + return Err(FormatError::InvalidDataFileEntry { + key: truncated(&entry.key), + reason: "unordered manifest entry has a key_range", + }); + } + if !is_sha256_hex(&entry.sha256) { + return Err(FormatError::InvalidDataFileEntry { + key: truncated(&entry.key), + reason: "sha256 is not 64 lowercase hex digits", + }); + } + if !is_md5_base64(&entry.md5) { + return Err(FormatError::InvalidDataFileEntry { + key: truncated(&entry.key), + reason: "md5 is not the base64 of 16 bytes", + }); + } + // An empty table is a manifest with no files; the format has no + // empty data file (it would carry a key_range with no items and + // is not a gzip stream at 0 bytes). + if entry.item_count == 0 { + return Err(FormatError::InvalidDataFileEntry { + key: truncated(&entry.key), + reason: "a data file has no items", + }); + } + if entry.bytes == 0 || entry.uncompressed_bytes == 0 { + return Err(FormatError::InvalidDataFileEntry { + key: truncated(&entry.key), + reason: "a file with items has zero bytes", + }); + } + total = total.checked_add(entry.bytes).ok_or_else(|| { + FormatError::InvalidManifest("data file sizes overflow".to_owned()) + })?; + } + if i64::try_from(total).ok() != Some(self.backup_size_bytes) { + return Err(FormatError::InvalidManifest(format!( + "backup_size_bytes is {} but the data files total {total}", + self.backup_size_bytes + ))); + } + if self.item_order == ItemOrder::KeyOrdered { + let order = self.key_order()?; + let mut previous_last: Option<&Item> = None; + for entry in &self.data_files { + let range = + entry + .key_range + .as_ref() + .ok_or_else(|| FormatError::InvalidDataFileEntry { + key: truncated(&entry.key), + reason: "key_ordered manifest entry has no key_range", + })?; + order.validate_key_item(&range.first)?; + order.validate_key_item(&range.last)?; + if order.compare(&range.first, &range.last)? == std::cmp::Ordering::Greater { + return Err(FormatError::InvalidManifest(format!( + "{:?}: key_range first is after last", + truncated(&entry.key) + ))); + } + if let Some(prev) = previous_last + && order.compare(prev, &range.first)? != std::cmp::Ordering::Less + { + return Err(FormatError::InvalidManifest(format!( + "{:?}: key_range overlaps or does not follow the previous file", + truncated(&entry.key) + ))); + } + previous_last = Some(&range.last); + } + } + Ok(()) + } +} + +/// Whether this build reads manifests of format version `found`: every +/// version from 1 to [`CURRENT_FORMAT_VERSION`]. A build keeps reading the +/// versions before its own, so a backup taken by an earlier release stays +/// restorable; 0 was never written. +/// +/// # Errors +/// +/// [`FormatError::UnsupportedFormatVersion`] for 0 or a version later than +/// [`CURRENT_FORMAT_VERSION`]. +pub fn check_format_version(found: u32) -> Result<(), FormatError> { + if (1..=CURRENT_FORMAT_VERSION).contains(&found) { + Ok(()) + } else { + Err(FormatError::UnsupportedFormatVersion { + found, + supported: CURRENT_FORMAT_VERSION, + }) + } +} + +/// The service's `BackupName` rule: 3 to 255 characters of `[a-zA-Z0-9_.-]`, +/// the table-name alphabet of `validate_table_name_chars`. That is not the +/// only copy of the alphabet: the engine's `validate_name_member` +/// (`BackupName` on `CreateBackup`) and `validate_index_name` spell it again, +/// and the engine counts the bounds in characters where this counts bytes +/// (the same for this ASCII alphabet). +fn is_valid_backup_name(name: &str) -> bool { + (3..=255).contains(&name.len()) && crate::validation::validate_table_name_chars(name).is_ok() +} + +fn is_sha256_hex(s: &str) -> bool { + lower_hex(s, 64) +} + +/// The base64 of an MD5 digest: 24 characters that decode, with the standard +/// alphabet and canonical padding, to exactly 16 bytes (`manifest-files.json`'s +/// `md5Checksum` encoding). A string of the right length and alphabet is not +/// enough: 24 `=` characters are both. +fn is_md5_base64(s: &str) -> bool { + use base64::Engine; + s.len() == 24 + && base64::engine::general_purpose::STANDARD + .decode(s) + .is_ok_and(|bytes| bytes.len() == 16) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::types::{KeyType, Projection, ProjectionType, ScalarAttributeType}; + + #[allow(clippy::too_many_lines)] + fn sample_manifest() -> ExtenddbManifest { + ExtenddbManifest { + format_version: CURRENT_FORMAT_VERSION, + backup_arn: + "arn:aws:dynamodb:us-east-1:123456789012:table/Music/backup/01757500000123-73d8d5bc" + .to_owned(), + backup_name: "music-nightly".to_owned(), + backup_creation_date_time: 1_757_500_000.123, + backup_size_bytes: 61_728, + source_backend: "postgres".to_owned(), + source_extenddb_version: "0.1.11".to_owned(), + snapshot: SnapshotDescriptor { + kind: SnapshotDescriptor::KIND_POSTGRES_REPEATABLE_READ.to_owned(), + marker: "1234:1234:".to_owned(), + }, + source_table: SourceTableSchema { + table_name: "Music".to_owned(), + table_id: "12345a12-abcd-123a-ab12-1234abc12345".to_owned(), + table_arn: "arn:aws:dynamodb:us-east-1:123456789012:table/Music".to_owned(), + table_creation_date_time: 1_757_400_000.0, + key_schema: vec![ + KeySchemaElement { + attribute_name: "Artist".to_owned(), + key_type: KeyType::Hash, + }, + KeySchemaElement { + attribute_name: "SongTitle".to_owned(), + key_type: KeyType::Range, + }, + ], + attribute_definitions: vec![ + AttributeDefinition { + attribute_name: "Artist".to_owned(), + attribute_type: ScalarAttributeType::S, + }, + AttributeDefinition { + attribute_name: "SongTitle".to_owned(), + attribute_type: ScalarAttributeType::S, + }, + AttributeDefinition { + attribute_name: "Genre".to_owned(), + attribute_type: ScalarAttributeType::S, + }, + ], + billing_mode: BillingMode::Provisioned, + provisioned_throughput: Some(ProvisionedThroughput { + read_capacity_units: 5, + write_capacity_units: 5, + }), + on_demand_throughput: None, + table_class: Some("STANDARD".to_owned()), + sse_description: Some(SseDescription { + status: "ENABLED".to_owned(), + sse_type: Some(crate::types::SseType::KMS), + kms_master_key_arn: Some( + "arn:aws:kms:us-east-1:123456789012:key/abcd".to_owned(), + ), + }), + global_secondary_indexes: vec![GsiInput { + index_name: "GenreIndex".to_owned(), + key_schema: vec![KeySchemaElement { + attribute_name: "Genre".to_owned(), + key_type: KeyType::Hash, + }], + projection: Projection { + projection_type: ProjectionType::All, + non_key_attributes: None, + }, + provisioned_throughput: Some(ProvisionedThroughput { + read_capacity_units: 1, + write_capacity_units: 1, + }), + }], + local_secondary_indexes: vec![LsiInput { + index_name: "TitleIndex".to_owned(), + key_schema: vec![ + KeySchemaElement { + attribute_name: "Artist".to_owned(), + key_type: KeyType::Hash, + }, + KeySchemaElement { + attribute_name: "SongTitle".to_owned(), + key_type: KeyType::Range, + }, + ], + projection: Projection { + projection_type: ProjectionType::KeysOnly, + non_key_attributes: None, + }, + }], + vector_indexes: Some(vec![VectorIndexSpecification { + index_name: "embeddings".to_owned(), + dimensions: 1024, + distance_function: crate::types::DistanceFunction::Cosine, + vector_attribute: crate::types::VectorAttribute { + attribute_name: "vec".to_owned(), + }, + search_schema: None, + projection: None, + }]), + stream_specification: Some(StreamSpecification { + stream_enabled: true, + stream_view_type: Some(crate::types::StreamViewType::NewAndOldImages), + }), + time_to_live: Some(TimeToLiveDescription { + time_to_live_status: crate::types::TimeToLiveStatus::Enabled, + attribute_name: Some("expires_at".to_owned()), + }), + item_count: 100_000, + table_size_bytes: 123_456, + }, + item_order: ItemOrder::Unordered, + data_files: vec![DataFileEntry { + key: "data/000001.json.gz".to_owned(), + item_count: 50_000, + sha256: "0".repeat(64), + md5: "1B2M2Y8AsgTpgAmY7PhCfg==".to_owned(), + bytes: 61_728, + uncompressed_bytes: 300_000, + key_range: None, + }], + } + } + + /// The bytes of `manifest` without [`ExtenddbManifest::to_json_bytes`]'s + /// validation, for tests that hand the parser a manifest the writer + /// refuses. + fn raw_bytes(manifest: &ExtenddbManifest) -> Vec { + serde_json::to_vec(manifest).unwrap() + } + + /// The writer runs every check the reader runs: a manifest the parser + /// would refuse cannot be serialized. An `N` endpoint built in code in + /// a spelling other than its normalized one is the case a backup job + /// would otherwise ship. + #[test] + fn to_json_bytes_refuses_what_from_json_bytes_refuses() { + let mut manifest = sample_manifest(); + manifest.item_order = ItemOrder::KeyOrdered; + manifest.source_table.attribute_definitions[1].attribute_type = ScalarAttributeType::N; + let n = |v: &str| -> Item { + [ + ( + "Artist".to_owned(), + crate::types::AttributeValue::S("a".into()), + ), + ( + "SongTitle".to_owned(), + crate::types::AttributeValue::N(v.into()), + ), + ] + .into_iter() + .collect() + }; + manifest.data_files[0].key_range = Some(KeyRange { + first: n("1"), + last: n("2"), + }); + manifest + .to_json_bytes() + .expect("normalized endpoints serialize"); + manifest.data_files[0].key_range = Some(KeyRange { + first: n("1"), + last: n("2.0"), + }); + let err = manifest.to_json_bytes().unwrap_err(); + assert!( + matches!(&err, FormatError::InvalidKey(m) if m.contains("normalized form")), + "{err}" + ); + // The other validation steps are run too, in the reader's order. + let mut off = sample_manifest(); + off.backup_size_bytes += 1; + assert!(matches!( + off.to_json_bytes(), + Err(FormatError::InvalidManifest(_)) + )); + let mut legacy = sample_manifest(); + legacy.backup_arn = + "arn:aws:dynamodb:us-east-1:123456789012:table/Music/backup/1757500000123-73d8d5bc" + .to_owned(); + assert!(matches!( + legacy.to_json_bytes(), + Err(FormatError::InvalidManifest(_)) + )); + let mut negative = sample_manifest(); + negative.source_table.item_count = -1; + assert!(matches!( + negative.to_json_bytes(), + Err(FormatError::InvalidManifest(_)) + )); + let mut later = sample_manifest(); + later.format_version = CURRENT_FORMAT_VERSION + 1; + assert!(matches!( + later.to_json_bytes(), + Err(FormatError::UnsupportedFormatVersion { .. }) + )); + // A non-finite timestamp would be written as `null`, which the + // reader refuses as a type error; both `f64` members are refused. + for bad in [f64::NAN, f64::INFINITY, f64::NEG_INFINITY] { + let mut table_time = sample_manifest(); + table_time.source_table.table_creation_date_time = bad; + let err = table_time.to_json_bytes().unwrap_err(); + assert!( + matches!(&err, FormatError::InvalidManifest(m) + if m.contains("table_creation_date_time") && m.contains("finite")), + "{bad}: {err}" + ); + let mut backup_time = sample_manifest(); + backup_time.backup_creation_date_time = bad; + let err = backup_time.to_json_bytes().unwrap_err(); + assert!( + matches!(&err, FormatError::InvalidManifest(m) + if m.contains("backup_creation_date_time")), + "{bad}: {err}" + ); + } + // `validate` alone reports the same. + assert!(matches!( + later.validate(), + Err(FormatError::UnsupportedFormatVersion { .. }) + )); + sample_manifest().validate().unwrap(); + } + + /// The key schema's shape is checked on every manifest, an unordered one + /// included: a multi-part key (which `CreateTable` accepts under the + /// preview flag and which is backed up `unordered`) passes, while a + /// schema no table can have is refused. + #[test] + fn key_schema_shape_is_checked_on_every_manifest() { + let good = sample_manifest(); + assert_eq!(good.item_order, ItemOrder::Unordered); + good.validate_source_table().unwrap(); + + let element = |name: &str, key_type| KeySchemaElement { + attribute_name: name.to_owned(), + key_type, + }; + let definition = |name: &str| AttributeDefinition { + attribute_name: name.to_owned(), + attribute_type: ScalarAttributeType::S, + }; + // Two HASH and two RANGE parts: unorderable, but a valid table. + let mut multi = good.clone(); + multi.source_table.key_schema = vec![ + element("a", KeyType::Hash), + element("b", KeyType::Hash), + element("c", KeyType::Range), + element("d", KeyType::Range), + ]; + multi.source_table.attribute_definitions = + ["a", "b", "c", "d"].iter().map(|n| definition(n)).collect(); + multi.validate_source_table().unwrap(); + ExtenddbManifest::from_json_bytes(&multi.to_json_bytes().unwrap()).unwrap(); + assert!(matches!( + multi.key_order(), + Err(FormatError::UnorderableKeySchema(_)) + )); + + let cases: [(&str, Vec, Vec); 7] = [ + ("no HASH element", vec![], vec![]), + ( + "no HASH element", + vec![element("a", KeyType::Range)], + vec![definition("a")], + ), + ( + "HASH element after a RANGE", + vec![ + element("a", KeyType::Hash), + element("b", KeyType::Range), + element("c", KeyType::Hash), + ], + vec![definition("a"), definition("b"), definition("c")], + ), + ( + "more than once", + vec![element("a", KeyType::Hash), element("a", KeyType::Range)], + vec![definition("a")], + ), + ( + "0 attribute definitions", + vec![element("a", KeyType::Hash), element("b", KeyType::Range)], + vec![definition("a")], + ), + ( + "0 attribute definitions", + vec![element("a", KeyType::Hash)], + vec![], + ), + ( + "2 attribute definitions", + vec![element("a", KeyType::Hash)], + vec![ + definition("a"), + AttributeDefinition { + attribute_name: "a".to_owned(), + attribute_type: ScalarAttributeType::N, + }, + ], + ), + ]; + for (reason, key_schema, attribute_definitions) in cases { + let mut m = good.clone(); + m.source_table.key_schema = key_schema; + m.source_table.attribute_definitions = attribute_definitions; + let err = m.validate_source_table().unwrap_err(); + assert!( + matches!(&err, FormatError::InvalidManifest(text) if text.contains(reason)), + "{reason}: {err}" + ); + // Through the parser, on an unordered manifest, where the + // key-range rules would not have looked at the schema. + let err = ExtenddbManifest::from_json_bytes(&raw_bytes(&m)).unwrap_err(); + assert!( + matches!(&err, FormatError::InvalidManifest(text) if text.contains(reason)), + "{reason}: {err}" + ); + } + // A long attribute name is bounded in the message. + let mut long = good; + long.source_table.key_schema = vec![element(&"k".repeat(5000), KeyType::Hash)]; + let err = long.validate_source_table().unwrap_err(); + assert!(err.to_string().len() < 200, "{}", err.to_string().len()); + } + + /// `DataFileEntry::from_checksums` copies every digest, size and count + /// from the encoder's result, so the manifest's entry and the file agree + /// by construction. + #[test] + fn data_file_entry_from_checksums_matches_the_encoder() { + let items = vec![key("a", "a"), key("b", "b")]; + let (compressed, checksums) = + crate::backup_format::encode_data_file(items.clone()).unwrap(); + let entry = DataFileEntry::from_checksums( + crate::backup_format::data_file_key(1), + &checksums, + Some(KeyRange { + first: key("a", "a"), + last: key("b", "b"), + }), + ); + assert_eq!(entry.key, "data/000001.json.gz"); + assert_eq!(entry.item_count, checksums.item_count); + assert_eq!(entry.item_count, 2); + assert_eq!(entry.sha256, checksums.sha256_hex); + assert_eq!(entry.md5, checksums.md5_base64); + assert_eq!(entry.bytes, checksums.compressed_bytes); + assert_eq!(entry.bytes, compressed.len() as u64); + assert_eq!(entry.uncompressed_bytes, checksums.uncompressed_bytes); + assert!(entry.key_range.is_some()); + // The entry's expectations verify the file it was built from. + assert_eq!(entry.decode(&compressed).unwrap(), items); + // And the entry passes the manifest's rules in a key-ordered manifest. + let mut manifest = sample_manifest(); + manifest.item_order = ItemOrder::KeyOrdered; + manifest.backup_size_bytes = i64::try_from(entry.bytes).unwrap(); + manifest.data_files = vec![entry]; + manifest.to_json_bytes().unwrap(); + } + + #[test] + fn manifest_serde_round_trip() { + let manifest = sample_manifest(); + let bytes = manifest.to_json_bytes().unwrap(); + let parsed = ExtenddbManifest::from_json_bytes(&bytes).unwrap(); + assert_eq!(parsed, manifest); + } + + #[test] + fn snapshot_descriptor_serde_round_trip() { + let snapshot = SnapshotDescriptor { + kind: SnapshotDescriptor::KIND_SQLITE_WAL_READ_TRANSACTION.to_owned(), + marker: String::new(), + }; + let json = serde_json::to_string(&snapshot).unwrap(); + let parsed: SnapshotDescriptor = serde_json::from_str(&json).unwrap(); + assert_eq!(parsed, snapshot); + } + + #[test] + fn source_table_schema_serde_round_trip() { + let schema = sample_manifest().source_table; + let json = serde_json::to_string(&schema).unwrap(); + let parsed: SourceTableSchema = serde_json::from_str(&json).unwrap(); + assert_eq!(parsed, schema); + } + + #[test] + fn data_file_entry_serde_round_trip() { + let entry = DataFileEntry { + key: "data/000042.json.gz".to_owned(), + item_count: 7, + sha256: "ab".repeat(32), + md5: "cd".repeat(11), + bytes: 1234, + uncompressed_bytes: 5678, + key_range: Some(KeyRange { + first: [( + "pk".to_owned(), + crate::types::AttributeValue::S("a".to_owned()), + )] + .into_iter() + .collect(), + last: [( + "pk".to_owned(), + crate::types::AttributeValue::S("z".to_owned()), + )] + .into_iter() + .collect(), + }), + }; + let json = serde_json::to_string(&entry).unwrap(); + let parsed: DataFileEntry = serde_json::from_str(&json).unwrap(); + assert_eq!(parsed, entry); + } + + /// Every optional member is written as an explicit `null` and every list + /// as `[]`: the member is present, not omitted (indexing a + /// `serde_json::Value` gives `null` for a missing key too, so presence + /// is asserted with `contains_key`). + #[test] + fn absent_optional_members_serialize_as_null() { + let mut manifest = sample_manifest(); + let t = &mut manifest.source_table; + t.provisioned_throughput = None; + t.on_demand_throughput = None; + t.table_class = None; + t.sse_description = None; + t.vector_indexes = None; + t.stream_specification = None; + t.time_to_live = None; + t.global_secondary_indexes.clear(); + t.local_secondary_indexes.clear(); + manifest.data_files[0].key_range = None; + let value = serde_json::to_value(&manifest).unwrap(); + let table = value["source_table"].as_object().unwrap(); + for member in [ + "provisioned_throughput", + "on_demand_throughput", + "table_class", + "sse_description", + "vector_indexes", + "stream_specification", + "time_to_live", + ] { + assert!(table.contains_key(member), "{member} omitted"); + assert!(table[member].is_null(), "{member}: {}", table[member]); + } + for member in ["global_secondary_indexes", "local_secondary_indexes"] { + assert!(table.contains_key(member), "{member} omitted"); + assert_eq!(table[member], serde_json::json!([]), "{member}"); + } + let file = value["data_files"][0].as_object().unwrap(); + assert!(file.contains_key("key_range"), "key_range omitted"); + assert!(file["key_range"].is_null()); + // An empty table: the file list is present and empty. + manifest.data_files.clear(); + let value = serde_json::to_value(&manifest).unwrap(); + assert!(value.as_object().unwrap().contains_key("data_files")); + assert_eq!(value["data_files"], serde_json::json!([])); + } + + #[test] + fn vector_indexes_distinguish_unsupported_from_none() { + let mut manifest = sample_manifest(); + manifest.source_table.vector_indexes = Some(vec![]); + let bytes = manifest.to_json_bytes().unwrap(); + let parsed = ExtenddbManifest::from_json_bytes(&bytes).unwrap(); + assert_eq!(parsed.source_table.vector_indexes, Some(vec![])); + manifest.source_table.vector_indexes = None; + let bytes = manifest.to_json_bytes().unwrap(); + let parsed = ExtenddbManifest::from_json_bytes(&bytes).unwrap(); + assert_eq!(parsed.source_table.vector_indexes, None); + } + + fn key(artist: &str, title: &str) -> Item { + [ + ( + "Artist".to_owned(), + crate::types::AttributeValue::S(artist.to_owned()), + ), + ( + "SongTitle".to_owned(), + crate::types::AttributeValue::S(title.to_owned()), + ), + ] + .into_iter() + .collect() + } + + fn second_file(first: Item, last: Item) -> DataFileEntry { + DataFileEntry { + key: "data/000002.json.gz".to_owned(), + item_count: 1, + sha256: "1".repeat(64), + md5: "AAAAAAAAAAAAAAAAAAAAAA==".to_owned(), + bytes: 100, + uncompressed_bytes: 200, + key_range: Some(KeyRange { first, last }), + } + } + + #[test] + fn key_ordered_manifest_requires_valid_non_overlapping_key_ranges() { + let mut manifest = sample_manifest(); + manifest.item_order = ItemOrder::KeyOrdered; + let err = manifest.validate_data_files().unwrap_err(); + assert!( + matches!( + err, + FormatError::InvalidDataFileEntry { + reason: "key_ordered manifest entry has no key_range", + .. + } + ), + "{err}" + ); + + // Endpoints must be key-only items of the schema's shape. + manifest.data_files[0].key_range = Some(KeyRange { + first: Item::new(), + last: Item::new(), + }); + assert!(matches!( + manifest.validate_data_files(), + Err(FormatError::InvalidKey(_)) + )); + let mut with_extra = key("a", "a"); + with_extra.insert( + "Genre".to_owned(), + crate::types::AttributeValue::S("x".into()), + ); + manifest.data_files[0].key_range = Some(KeyRange { + first: with_extra, + last: key("a", "b"), + }); + assert!(matches!( + manifest.validate_data_files(), + Err(FormatError::InvalidKey(_)) + )); + + // last is checked as a key item too. + manifest.data_files[0].key_range = Some(KeyRange { + first: key("a", "a"), + last: Item::new(), + }); + assert!(matches!( + manifest.validate_data_files(), + Err(FormatError::InvalidKey(_)) + )); + + // first must not be after last. + manifest.data_files[0].key_range = Some(KeyRange { + first: key("b", "a"), + last: key("a", "a"), + }); + assert!(matches!( + manifest.validate_data_files(), + Err(FormatError::InvalidManifest(_)) + )); + + // Valid single file. + manifest.data_files[0].key_range = Some(KeyRange { + first: key("a", "a"), + last: key("m", "z"), + }); + manifest.validate_data_files().unwrap(); + assert_eq!( + serde_json::to_value(&manifest).unwrap()["item_order"], + serde_json::json!("key_ordered") + ); + + // A second file must start strictly after the first ends. + manifest + .data_files + .push(second_file(key("m", "z"), key("z", "z"))); + manifest.backup_size_bytes += 100; + assert!(matches!( + manifest.validate_data_files(), + Err(FormatError::InvalidManifest(_)) + )); + manifest.data_files[1].key_range = Some(KeyRange { + first: key("n", "a"), + last: key("z", "z"), + }); + manifest.validate_data_files().unwrap(); + // Files out of order are rejected too. + manifest.data_files.swap(0, 1); + manifest.data_files[0].key = "data/000001.json.gz".to_owned(); + manifest.data_files[1].key = "data/000002.json.gz".to_owned(); + assert!(matches!( + manifest.validate_data_files(), + Err(FormatError::InvalidManifest(_)) + )); + } + + fn ordered_files(ranges: &[(Item, Item)]) -> ExtenddbManifest { + let mut manifest = sample_manifest(); + manifest.item_order = ItemOrder::KeyOrdered; + manifest.data_files = ranges + .iter() + .enumerate() + .map(|(i, (first, last))| { + let mut f = second_file(first.clone(), last.clone()); + f.key = crate::backup_format::data_file_key(u32::try_from(i + 1).unwrap()); + f + }) + .collect(); + manifest.backup_size_bytes = 100 * i64::try_from(ranges.len()).unwrap(); + manifest + } + + /// Each file is compared with the one before it, not only with the + /// first: [A..F], [G..M], [H..Z] overlaps between the second and third + /// file only. + #[test] + fn overlap_between_later_files_is_refused() { + let good = ordered_files(&[ + (key("A", "a"), key("F", "z")), + (key("G", "a"), key("M", "z")), + (key("N", "a"), key("Z", "z")), + ]); + good.validate_data_files().unwrap(); + let overlap = ordered_files(&[ + (key("A", "a"), key("F", "z")), + (key("G", "a"), key("M", "z")), + (key("H", "a"), key("Z", "z")), + ]); + let err = overlap.validate_data_files().unwrap_err(); + assert!( + matches!(&err, FormatError::InvalidManifest(m) if m.contains("data/000003.json.gz")), + "{err}" + ); + // Touching endpoints overlap too: one key in two files. + let touching = ordered_files(&[ + (key("A", "a"), key("F", "z")), + (key("G", "a"), key("M", "z")), + (key("M", "z"), key("Z", "z")), + ]); + assert!(touching.validate_data_files().is_err()); + } + + /// `last` is checked as a key item on its own: a valid key plus a + /// non-key attribute compares fine (compare ignores extra attributes), + /// so only the shape check refuses it. + #[test] + fn last_endpoint_with_a_non_key_attribute_is_refused() { + let mut last = key("M", "z"); + last.insert( + "Genre".to_owned(), + crate::types::AttributeValue::S("x".into()), + ); + let manifest = ordered_files(&[(key("A", "a"), last)]); + let err = manifest.validate_data_files().unwrap_err(); + assert!( + matches!(&err, FormatError::InvalidKey(m) if m.contains("attributes")), + "{err}" + ); + // And an `N`-typed endpoint that is not normalized, in `last` only. + let mut manifest = sample_manifest(); + manifest.item_order = ItemOrder::KeyOrdered; + manifest.source_table.attribute_definitions[1].attribute_type = ScalarAttributeType::N; + let n = |v: &str| -> Item { + [ + ( + "Artist".to_owned(), + crate::types::AttributeValue::S("a".into()), + ), + ( + "SongTitle".to_owned(), + crate::types::AttributeValue::N(v.into()), + ), + ] + .into_iter() + .collect() + }; + manifest.data_files[0].key_range = Some(KeyRange { + first: n("1"), + last: n("2"), + }); + manifest.validate_data_files().unwrap(); + manifest.data_files[0].key_range = Some(KeyRange { + first: n("1"), + last: n("2.0"), + }); + assert!(matches!( + manifest.validate_data_files(), + Err(FormatError::InvalidKey(_)) + )); + } + + /// File sizes whose sum does not fit a u64 are refused as an overflow, + /// even when `backup_size_bytes` equals the wrapped sum. + #[test] + fn data_file_sizes_that_overflow_are_refused() { + let mut manifest = sample_manifest(); + let mut second = manifest.data_files[0].clone(); + second.key = "data/000002.json.gz".to_owned(); + manifest.data_files[0].bytes = u64::MAX; + second.bytes = 101; + manifest.data_files.push(second); + // u64::MAX + 101 wraps to 100. + manifest.backup_size_bytes = 100; + let err = manifest.validate_data_files().unwrap_err(); + assert!( + matches!(&err, FormatError::InvalidManifest(m) if m.contains("overflow")), + "{err}" + ); + // A sum that fits u64 but not i64 is a size mismatch. + manifest.data_files[0].bytes = u64::MAX - 101; + manifest.backup_size_bytes = i64::MAX; + let err = manifest.validate_data_files().unwrap_err(); + assert!( + matches!(&err, FormatError::InvalidManifest(m) if m.contains("total")), + "{err}" + ); + } + + #[test] + fn duplicate_data_file_keys_are_rejected() { + let mut manifest = sample_manifest(); + let dup = manifest.data_files[0].clone(); + manifest.data_files.push(dup); + manifest.backup_size_bytes *= 2; + let err = manifest.validate_data_files().unwrap_err(); + assert!(matches!( + err, + FormatError::InvalidDataFileKey { + reason: "listed more than once", + .. + } + )); + } + + #[test] + fn backup_size_must_equal_the_sum_of_data_files() { + let mut manifest = sample_manifest(); + manifest.backup_size_bytes += 1; + assert!(matches!( + manifest.validate_data_files(), + Err(FormatError::InvalidManifest(_)) + )); + } + + #[test] + fn version_gate_runs_before_the_full_parse() { + // A later version with a shape this build cannot parse must still be + // reported as an unsupported version, not as a JSON error. + let bytes = br#"{"format_version": 7, "something_new": {"x": 1}}"#; + let err = ExtenddbManifest::from_json_bytes(bytes).unwrap_err(); + assert!( + matches!(err, FormatError::UnsupportedFormatVersion { found: 7, .. }), + "{err}" + ); + } + + #[test] + fn malformed_backup_arn_is_rejected_on_load() { + let mut manifest = sample_manifest(); + manifest.backup_arn = + "arn:aws:dynamodb:us-east-1:evil:1234:table/Music/backup/1".to_owned(); + let err = ExtenddbManifest::from_json_bytes(&raw_bytes(&manifest)).unwrap_err(); + assert!(matches!(err, FormatError::InvalidArn(_)), "{err}"); + } + + /// `backup_arn` is the identity; the source-table name and ARN must agree + /// with it. + #[test] + fn source_table_must_match_backup_arn() { + let good = sample_manifest(); + good.validate_identity().unwrap(); + + let mut other_name = good.clone(); + other_name.source_table.table_name = "Other".to_owned(); + let err = ExtenddbManifest::from_json_bytes(&raw_bytes(&other_name)).unwrap_err(); + assert!(matches!(err, FormatError::InvalidManifest(_)), "{err}"); + assert!(err.to_string().contains("source_table.table_name"), "{err}"); + + for bad_arn in [ + "arn:aws:dynamodb:us-east-1:123456789012:table/Other", + "arn:aws:dynamodb:us-east-1:999999999999:table/Music", + "arn:aws:s3:us-east-1:123456789012:table/Music", + "arn:aws:dynamodb:us-east-1:123456789012:table/Music/index/x", + "Music", + ] { + let mut m = good.clone(); + m.source_table.table_arn = bad_arn.to_owned(); + let err = m.validate_identity().unwrap_err(); + assert!( + matches!(err, FormatError::InvalidManifest(_)), + "{bad_arn}: {err}" + ); + assert!(err.to_string().contains("table_arn"), "{err}"); + } + // The table ARN's region is recorded at table creation and is not + // compared. + let mut other_region = good; + other_region.source_table.table_arn = + "arn:aws:dynamodb:eu-west-1:123456789012:table/Music".to_owned(); + other_region.validate_identity().unwrap(); + } + + /// The identity rules over hostile input: non-canonical ARN spelling, + /// an id that disagrees with the creation time, a name the API would + /// echo, and a table ARN in another partition. + #[test] + fn identity_rejects_non_canonical_and_disagreeing_fields() { + let good = sample_manifest(); + good.validate_identity().unwrap(); + + let mut padded = good.clone(); + padded.backup_arn = format!(" {}\n", good.backup_arn); + let err = padded.validate_identity().unwrap_err(); + assert!(err.to_string().contains("canonical"), "{err}"); + + let mut off = good.clone(); + off.backup_creation_date_time += 0.001; + let err = off.validate_identity().unwrap_err(); + assert!( + err.to_string().contains("does not match the instant"), + "{err}" + ); + // Equality is exact: half a millisecond off is a different time. + let mut near = good.clone(); + near.backup_creation_date_time += 0.0004; + let err = near.validate_identity().unwrap_err(); + assert!( + err.to_string().contains("does not match the instant"), + "{err}" + ); + let mut negative = good.clone(); + negative.backup_creation_date_time = -5.0; + assert!(negative.validate_identity().is_err()); + + let mut legacy = good.clone(); + legacy.backup_arn = + "arn:aws:dynamodb:us-east-1:123456789012:table/Music/backup/1757500000123-73d8d5bc" + .to_owned(); + let err = legacy.validate_identity().unwrap_err(); + assert!(err.to_string().contains("creation instant"), "{err}"); + + for bad_name in ["ab", "a b", "a/b", &"x".repeat(256), "\u{0}\u{7}