diff --git a/.github/workflows/integration-cassandra.yml b/.github/workflows/integration-cassandra.yml index 8bad0fbb5..834ad0d5f 100644 --- a/.github/workflows/integration-cassandra.yml +++ b/.github/workflows/integration-cassandra.yml @@ -148,6 +148,10 @@ jobs: - name: Build release (cassandra backend) run: cargo build --release --no-default-features --features cassandra - name: Run Cassandra pytest suite + env: + # run-cassandra-tests initializes the server without [backup.store], + # and Cassandra takes the in-catalog create with one. + EXTENDDB_TEST_BACKUP_STORE: none run: devtools/run-cassandra-tests -- --pytest --comprehensive --parallel cassandra-rust: @@ -199,6 +203,10 @@ jobs: - name: Build release (cassandra backend) run: cargo build --release --no-default-features --features cassandra - name: Run Cassandra rust integration suite + env: + # run-cassandra-tests initializes the server without [backup.store]: + # the Rust backup tests restore the in-catalog create's backups. + EXTENDDB_TEST_BACKUP_STORE: none run: devtools/run-cassandra-tests -- --rust-integration cassandra-production-build: diff --git a/.github/workflows/integration-mongodb.yml b/.github/workflows/integration-mongodb.yml index 03223f967..1400431fc 100644 --- a/.github/workflows/integration-mongodb.yml +++ b/.github/workflows/integration-mongodb.yml @@ -49,6 +49,10 @@ jobs: - name: Build release (mongodb backend) run: cargo build --release --no-default-features --features mongodb-test-hooks - name: Run MongoDB pytest suite + env: + # run-mongodb-tests initializes the server without [backup.store]: + # CreateBackup is the in-catalog create. + EXTENDDB_TEST_BACKUP_STORE: none run: devtools/run-mongodb-tests -- --pytest --comprehensive --parallel # SnapshotEngine consistency check against a disposable replica set. The @@ -117,6 +121,10 @@ jobs: - name: Build release (mongodb backend) run: cargo build --release --no-default-features --features mongodb-test-hooks - name: Run MongoDB rust integration suite + env: + # run-mongodb-tests initializes the server without [backup.store]: + # the Rust backup tests restore the in-catalog create's backups. + EXTENDDB_TEST_BACKUP_STORE: none run: devtools/run-mongodb-tests -- --rust --rust-integration mongodb-production-build: diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index dd26dae46..8084153d3 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -67,19 +67,34 @@ jobs: password=$(echo "$output" | grep -oP 'Password: \K\S+') echo "admin_password=$password" >> "$GITHUB_OUTPUT" + # A filesystem backup store, so this job's CreateBackup is the + # store-backed job (PostgreSQL has no default root). The suite reads + # EXTENDDB_TEST_BACKUP_STORE=filesystem below to expect that path. + - name: Configure a filesystem backup store + run: | + mkdir -m 0700 "$RUNNER_TEMP/backups" + printf '\n[backup.store]\nkind = "filesystem"\n[backup.store.filesystem]\npath = "%s/backups"\n' \ + "$RUNNER_TEMP" >> extenddb.toml + - name: Start ExtendDB run: | # --write-pid-file so devtools/run-tests can restart the server with # 'extenddb stop' when it needs to apply a config change. + # The wait is for the backup store component to be healthy, not for + # the word "healthy" anywhere in the reply: a server whose store + # failed its check reports "unhealthy" (which that word matches) + # and the suite would then fail on a backup cell with no pointer to + # the cause. ./target/release/extenddb serve --config extenddb.toml --foreground --write-pid-file & for i in $(seq 1 30); do - if curl -sk https://127.0.0.1:18443/health | grep -q healthy; then - echo "Server ready" + if curl -sk https://127.0.0.1:18443/health | python3 -c 'import json, sys; sys.exit(0 if json.load(sys.stdin).get("components", {}).get("backup_store", {}).get("status") == "healthy" else 1)' 2>/dev/null; then + echo "Server ready with a healthy backup store" exit 0 fi sleep 1 done - echo "Server failed to start" + echo "Server failed to start, or its backup store did not become healthy" + curl -sk https://127.0.0.1:18443/health || true exit 1 - uses: actions/setup-python@v5 @@ -150,8 +165,17 @@ jobs: EXTENDDB_ADMIN_USER: admin EXTENDDB_ADMIN_PASSWORD: ${{ steps.init.outputs.admin_password }} EXTENDDB_TEST_PG_ADMIN_CONNECTION_STRING: postgresql://postgres:devpass@127.0.0.1:5432 + EXTENDDB_TEST_BACKUP_STORE: filesystem run: devtools/run-tests --extenddb --pytest --comprehensive --parallel --filter "not import_export" + # The backup store and job suites start servers of their own, so they + # are excluded from run-tests' line and run by name. + - name: Run the backup store and job suites + env: + EXTENDDB_TEST_PG_CONNECTION_STRING: postgresql://postgres:devpass@127.0.0.1:5432 + EXTENDDB_TEST_PG_ADMIN_CONNECTION_STRING: postgresql://postgres:devpass@127.0.0.1:5432 + run: python3 -m pytest tests/test_backup_store_config.py tests/test_backup_job.py -v -rP -p no:cacheprovider + # run-tests restarts the server daemonized to apply the import/export # config, and a daemonized server logs to syslog, not the job log. Without # this dump, the storage error behind a client-visible 500 leaves no @@ -220,15 +244,21 @@ jobs: run: | # --write-pid-file so devtools/run-tests can restart the server with # 'extenddb stop' when it needs to apply a config change. + # The wait is for the backup store component to be healthy, not for + # the word "healthy" anywhere in the reply: a server whose store + # failed its check reports "unhealthy" (which that word matches) + # and the suite would then fail on a backup cell with no pointer to + # the cause. ./target/release/extenddb serve --config extenddb.toml --foreground --write-pid-file & for i in $(seq 1 30); do - if curl -sk https://127.0.0.1:18443/health | grep -q healthy; then - echo "Server ready" + if curl -sk https://127.0.0.1:18443/health | python3 -c 'import json, sys; sys.exit(0 if json.load(sys.stdin).get("components", {}).get("backup_store", {}).get("status") == "healthy" else 1)' 2>/dev/null; then + echo "Server ready with a healthy backup store" exit 0 fi sleep 1 done - echo "Server failed to start" + echo "Server failed to start, or its backup store did not become healthy" + curl -sk https://127.0.0.1:18443/health || true exit 1 - uses: actions/setup-python@v5 @@ -243,14 +273,132 @@ jobs: EXTENDDB_TEST_ENDPOINT: https://127.0.0.1:18443 EXTENDDB_ADMIN_USER: admin EXTENDDB_ADMIN_PASSWORD: ${{ steps.init.outputs.admin_password }} + # A file-backed SQLite server without [backup.store] writes to the + # defaulted root beside the database file. + EXTENDDB_TEST_BACKUP_STORE: filesystem run: devtools/run-tests --extenddb --pytest --comprehensive --parallel --filter "not import_export" + - name: Run the backup store and job suites + run: python3 -m pytest tests/test_backup_store_config.py tests/test_backup_job.py -v -rP -p no:cacheprovider + # Same rationale as the postgres job: the daemonized server logs to # syslog, so this dump is the only server-side evidence on failure. - name: Dump server syslog on failure if: failure() run: sudo journalctl -t extenddb --no-pager -n 500 || true + # The S3 leg: the SQLite build against an S3 backup store on moto. The + # backup tests of the main suite (the tests named "backup") run against it + # with EXTENDDB_TEST_BACKUP_STORE=s3, and the fidelity suite's health check + # confirms the server's store is the S3 one; the store configuration + # suite's S3 cells need the same endpoint. The job suite + # (tests/test_backup_job.py) starts servers of its own with the SQLite + # build's defaulted filesystem root, as on the SQLite job: its deployment + # fixture has no S3 branch, so it covers the job and not this store. + run-integration-sqlite-s3: + runs-on: extenddb_ubuntu-2404_4-core + timeout-minutes: 40 + steps: + - uses: actions/checkout@v6 + + - uses: dtolnay/rust-toolchain@stable + + - uses: Swatinem/rust-cache@v2 + with: + cache-on-failure: true + + - name: Build release (SQLite backend) + run: cargo build --release -p extenddb --no-default-features --features sqlite + + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Install Python dependencies + run: pip install -r requirements.txt + + # moto is a pip package pinned in requirements-moto.txt (test.yml runs + # the same server for the store crate's conformance suite); its boto3 + # floor differs from requirements.txt's pin, so it gets an environment + # of its own. + - name: Start S3 mock + run: | + python3 -m venv "$RUNNER_TEMP/moto" + "$RUNNER_TEMP/moto/bin/pip" install -r requirements-moto.txt + nohup "$RUNNER_TEMP/moto/bin/moto_server" -p 9000 > moto.log 2>&1 & + for _ in $(seq 1 30); do + if curl -sf http://127.0.0.1:9000/ > /dev/null; then exit 0; fi + sleep 1 + done + echo "moto_server did not start"; cat moto.log; exit 1 + + - name: Initialize ExtendDB + id: init + env: + AWS_ACCESS_KEY_ID: testing + AWS_SECRET_ACCESS_KEY: testing + AWS_REGION: us-east-1 + run: | + output=$(./target/release/extenddb init --backend sqlite --config extenddb.toml 2>&1) + echo "$output" + password=$(echo "$output" | grep -oP 'Password: \K\S+') + echo "admin_password=$password" >> "$GITHUB_OUTPUT" + python3 -c 'import boto3; boto3.client("s3", endpoint_url="http://127.0.0.1:9000", region_name="us-east-1").create_bucket(Bucket="extenddb-ci-backups")' + printf '\n[backup.store]\nkind = "s3"\n[backup.store.s3]\nbucket = "extenddb-ci-backups"\nendpoint = "http://127.0.0.1:9000"\nforce_path_style = true\nregion = "us-east-1"\n' >> extenddb.toml + + - name: Start ExtendDB + env: + AWS_ACCESS_KEY_ID: testing + AWS_SECRET_ACCESS_KEY: testing + AWS_REGION: us-east-1 + AWS_EC2_METADATA_DISABLED: "true" + run: | + # An S3 store is checked after the listener is up and reports + # "checking" until its first check returns; a CreateBackup inside + # that window is refused, so the wait is for the component to be + # healthy, which also names a store that failed its check here + # rather than in a backup cell. + ./target/release/extenddb serve --config extenddb.toml --foreground --write-pid-file & + for i in $(seq 1 30); do + if curl -sk https://127.0.0.1:18443/health | python3 -c 'import json, sys; sys.exit(0 if json.load(sys.stdin).get("components", {}).get("backup_store", {}).get("status") == "healthy" else 1)' 2>/dev/null; then + echo "Server ready with a healthy backup store" + exit 0 + fi + sleep 1 + done + echo "Server failed to start, or its backup store did not become healthy" + curl -sk https://127.0.0.1:18443/health || true + exit 1 + + - name: Run the backup tests against the S3 store + env: + EXTENDDB_TEST_ENDPOINT: https://127.0.0.1:18443 + EXTENDDB_ADMIN_USER: admin + EXTENDDB_ADMIN_PASSWORD: ${{ steps.init.outputs.admin_password }} + EXTENDDB_TEST_BACKUP_STORE: s3 + # run-tests restarts the server daemonized, which needs the store's + # credentials in its environment too. + AWS_ACCESS_KEY_ID: testing + AWS_SECRET_ACCESS_KEY: testing + AWS_REGION: us-east-1 + AWS_EC2_METADATA_DISABLED: "true" + run: devtools/run-tests --extenddb --pytest --parallel --filter "backup" + + # The store configuration suite's S3 cells against moto; the job suite + # against the build's defaulted filesystem root (the job comment above). + - name: Run the store configuration suite's S3 cells and the job suite + env: + EXTENDDB_TEST_S3_ENDPOINT: http://127.0.0.1:9000 + run: python3 -m pytest tests/test_backup_store_config.py tests/test_backup_job.py -v -rP -p no:cacheprovider + + - name: Dump server syslog on failure + if: failure() + run: sudo journalctl -t extenddb --no-pager -n 500 || true + + - name: Print the S3 mock's log + if: failure() + run: cat moto.log || true + run-integration-dev-mode: # dev-mode was shipped with no CI coverage at all, which is how the batch and # transaction authorization regression reached main: the build compiled, so @@ -384,6 +532,9 @@ jobs: # tests mandatory. The refusal suite moves to the novector job, which is # the only place those assertions still mean anything. EXTENDDB_EXPECT_VECTORS: "1" + # This server has no [backup.store]: CreateBackup is the in-catalog + # create and the backup tests restore its backups. + EXTENDDB_TEST_BACKUP_STORE: none run: devtools/run-tests --extenddb --rust-integration --release # The control plane for vector indexes is not reachable over the wire while @@ -479,6 +630,9 @@ jobs: # the backend reports, so without this it could skip every assertion # and still report green. EXTENDDB_EXPECT_VECTORS: "0" + # No [backup.store] on this server either; the filter runs no backup + # test, and the variable says so for whoever widens it. + EXTENDDB_TEST_BACKUP_STORE: none run: >- devtools/run-tests --extenddb --rust-integration --release --filter vector_index_unsupported @@ -560,6 +714,11 @@ jobs: # which is what would happen if the backend lost the capability. Pinning the # expectation turns that skip into a failure. EXTENDDB_EXPECT_VECTORS: "1" + # A file-backed SQLite server without [backup.store] has the defaulted + # filesystem root beside its database file, so CreateBackup is the + # asynchronous, store-backed one: the backup tests wait for AVAILABLE, + # delete what they create, and expect the restore's refusal. + EXTENDDB_TEST_BACKUP_STORE: filesystem run: devtools/run-tests --extenddb --rust-integration --release # The daemonized server logs to syslog; dump it so server-side failures @@ -575,6 +734,7 @@ jobs: [ run-integration, run-integration-sqlite, + run-integration-sqlite-s3, run-integration-dev-mode, run-rust-integration, run-rust-integration-postgres-novector, @@ -585,6 +745,7 @@ jobs: - run: | if [ "${{ needs.run-integration.result }}" != "success" ] || \ [ "${{ needs.run-integration-sqlite.result }}" != "success" ] || \ + [ "${{ needs.run-integration-sqlite-s3.result }}" != "success" ] || \ [ "${{ needs.run-integration-dev-mode.result }}" != "success" ] || \ [ "${{ needs.run-rust-integration.result }}" != "success" ] || \ [ "${{ needs.run-rust-integration-postgres-novector.result }}" != "success" ] || \ diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index c3b4d96c8..05f39c47a 100755 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -47,8 +47,9 @@ cargo build --workspace # Rust unit tests cargo test --workspace -# Integration tests (requires a running extenddb server) -devtools/run-tests --extenddb --pytest +# Integration tests (requires a running extenddb server; the variable says +# where that server writes its backups, see the developer test guide) +EXTENDDB_TEST_BACKUP_STORE=filesystem devtools/run-tests --extenddb --pytest ``` ### Code Style diff --git a/Cargo.lock b/Cargo.lock index 7a5ca02d0..ddceb2933 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1817,6 +1817,7 @@ name = "extenddb-engine" version = "0.1.14" dependencies = [ "base64 0.22.1", + "bytes", "crc32fast", "extenddb-auth", "extenddb-backup-store", @@ -1829,6 +1830,7 @@ dependencies = [ "serde_json", "sha2 0.10.9", "tokio", + "tokio-util", "tracing", "uuid", ] @@ -1999,7 +2001,9 @@ dependencies = [ "bigdecimal", "crc32fast", "extenddb-auth", + "extenddb-backup-store", "extenddb-core", + "extenddb-engine", "extenddb-storage", "extenddb-storage-sqlite", "futures", diff --git a/crates/app/Cargo.toml b/crates/app/Cargo.toml index 099ea21cf..fef29af1e 100644 --- a/crates/app/Cargo.toml +++ b/crates/app/Cargo.toml @@ -13,6 +13,7 @@ license.workspace = true # the thin bin enforces that it is paired with a dev/CI backend. dev-mode = ["extenddb-config/dev-mode"] mongodb-test-hooks = ["extenddb-server/mongodb-test-hooks"] +backup-test-hooks = ["extenddb-server/backup-test-hooks"] [dependencies] extenddb-auth = { workspace = true } diff --git a/crates/app/src/init_helpers.rs b/crates/app/src/init_helpers.rs index 8be891e82..1e93d2777 100755 --- a/crates/app/src/init_helpers.rs +++ b/crates/app/src/init_helpers.rs @@ -303,7 +303,7 @@ key_path = "{tls_key}" # [backup] # Backup and restore jobs. stale_job_after_secs must be at least three # heartbeat periods, the larger of heartbeat_interval_secs and the 15 s the -# jobs of this release heartbeat at (45 at the least); pool_size is raised to +# in-catalog jobs heartbeat at (45 at the least); pool_size is raised to # at least 2 * max_concurrent_jobs + 10 on PostgreSQL; max_concurrent_jobs # is at most 1024. # data_file_target_bytes = 67108864 # Uncompressed bytes per data file (minimum 32 MiB) @@ -317,9 +317,10 @@ key_path = "{tls_key}" # # it defaults to backup-spool/ beside the database file # [backup.store] -# Where CreateBackup's backups will live, from a later release (this release's -# CreateBackup writes in-catalog backups with any store, and the store serves -# restores and deletes of the backups it holds). Absent on file-backed SQLite: a filesystem +# Where CreateBackup writes its backups on PostgreSQL, SQLite and MongoDB +# (CreateBackup answers CREATING and a job of the server uploads; Cassandra +# writes in-catalog backups with or without a store), and where restores and +# deletes of store-backed backups read. Absent on file-backed SQLite: a filesystem # store at backups/ beside the database file. Absent on PostgreSQL, MongoDB, # Cassandra and an in-memory SQLite database: no store, and CreateBackup # writes in-catalog backups with a deprecation warning. kind = "disabled" diff --git a/crates/config/src/backup.rs b/crates/config/src/backup.rs index 83e3a9c83..b31a80ce7 100644 --- a/crates/config/src/backup.rs +++ b/crates/config/src/backup.rs @@ -22,17 +22,24 @@ //! //! `stale_job_after_secs` must be at least three heartbeat periods: a job //! checks before each store write that its latest heartbeat confirmed its -//! claim within one period, and the margin is what makes that detection -//! precede a reconciler's deletion of the job's prefix. The period the floor -//! is computed from is the one the jobs run at: the larger of -//! `heartbeat_interval_secs` and [`JOB_HEARTBEAT_INTERVAL`] (15 s), since -//! until a later release wires the knob to the jobs every backend heartbeats -//! at that constant, and a floor on a shorter configured interval would let -//! the reconciler declare a live job dead between two of its heartbeats. The -//! floor is a floor: on SQLite a heartbeat waits for the engine write lock -//! like any writer, so a long write transaction delays it, and a -//! `stale_job_after_secs` near three periods on a busy file can declare a -//! live restore dead. +//! claim recently enough (within two periods: one for the timer, one for the +//! beat to return), and the margin is what makes that detection precede a +//! reconciler's deletion of the job's prefix. The period the floor is +//! computed from is the one the jobs run at: the larger of +//! `heartbeat_interval_secs` and [`JOB_HEARTBEAT_INTERVAL`] (15 s). The +//! store-backed backup job heartbeats at that period; the in-catalog backup +//! and restore paths heartbeat at the constant whatever the knob says, and a +//! floor on a shorter configured interval would let the reconciler declare +//! one of those dead between two of its heartbeats. The floor is a floor: on +//! SQLite a heartbeat waits for the engine write lock like any writer, so a +//! long write transaction delays it, and a `stale_job_after_secs` near three +//! periods on a busy file can declare a live restore dead. +//! +//! `data_file_target_bytes` is read by the store-backed backup job (a data +//! file closes once its uncompressed bytes reach it, so a file exceeds it by +//! less than one line); `failed_row_retention_secs` by the reconciler, which +//! removes a FAILED or DELETED row whose prefix is empty once its +//! `completed_at` is older than that. //! //! `max_concurrent_jobs` is at most [`MAX_CONCURRENT_JOBS`]: a value over //! the runtime's permit ceiling would stop the server at startup rather than @@ -59,10 +66,11 @@ //! unconfigured and startup logs that. //! - `kind = "disabled"` leaves the store unconfigured on purpose, on any //! backend. -//! - Without a configured store, CreateBackup writes the in-catalog format -//! and logs a deprecation warning naming `[backup.store]` on each call; -//! backups of either format stay readable, listable, restorable and -//! deletable. +//! - With a configured store, CreateBackup on PostgreSQL, SQLite and +//! MongoDB writes the backup to the store and returns CREATING; without +//! one, and on Cassandra always, it writes the in-catalog format and logs +//! a deprecation warning naming `[backup.store]` on each call; backups of +//! either format stay readable, listable, restorable and deletable. //! //! The spool. Restores from an S3 store download to `spool_path`, created //! `0700` with files `0600`. On file-backed SQLite it defaults to @@ -196,9 +204,9 @@ pub struct ResolvedStore { } impl BackupConfig { - /// The period the jobs heartbeat at: the larger of - /// `heartbeat_interval_secs` and [`JOB_HEARTBEAT_INTERVAL`], which is - /// what every backend runs at until the knob reaches the jobs. + /// The period the store-backed backup job heartbeats at, and the floor's + /// unit: the larger of `heartbeat_interval_secs` and + /// [`JOB_HEARTBEAT_INTERVAL`] (the in-catalog paths run at the constant). #[must_use] pub fn effective_heartbeat_interval_secs(&self) -> u64 { self.heartbeat_interval_secs @@ -250,8 +258,8 @@ impl BackupConfig { return Err(format!( "backup.stale_job_after_secs = {} is under {MIN_STALE_HEARTBEATS} * the job \ heartbeat period of {period} s = {floor} (the larger of \ - backup.heartbeat_interval_secs = {} and the {} s the jobs heartbeat at in \ - this release); a reconciler could declare a running job dead between two \ + backup.heartbeat_interval_secs = {} and the {} s the in-catalog jobs \ + heartbeat at); a reconciler could declare a running job dead between two \ of its heartbeats", self.stale_job_after_secs, self.heartbeat_interval_secs, diff --git a/crates/core/src/backup_format/data_file.rs b/crates/core/src/backup_format/data_file.rs index 10054b8e9..bb5e48033 100644 --- a/crates/core/src/backup_format/data_file.rs +++ b/crates/core/src/backup_format/data_file.rs @@ -137,7 +137,8 @@ pub const MAX_LINE_BYTES: usize = 32 * 1024 * 1024; /// the two paths share one encoding by construction. The line is not checked /// against [`MAX_LINE_BYTES`]; [`DataFileEncoder::write_item`] does that. /// [`encode_item_line_into`] writes the same bytes into a buffer the caller -/// keeps. +/// keeps; [`encode_item_line_len_bounded`] measures them against a cap +/// without a buffer. /// /// # Errors /// @@ -165,6 +166,65 @@ pub fn encode_item_line_into(item: &Item, line: &mut Vec) -> Result<(), serd Ok(()) } +/// The length of [`encode_item_line`]'s line for `item` when it is at most +/// `cap` bytes, measured without a buffer: the line is serialized into a +/// writer that counts and refuses the write that would take the count past +/// `cap`, at which `serde_json::to_writer` stops, so an item whose line is +/// many times the cap costs no memory and no more work than the cap's +/// worth. The backup job measures an item over the default item limit this +/// way before the encoder sees it; for a line within the cap the length is +/// the one `encode_item_line_into` would leave. +/// +/// # Errors +/// +/// Returns [`FormatError::LineTooLong`] once the count would pass `cap` and +/// [`FormatError::Json`] when the item cannot be serialized. +pub fn encode_item_line_len_bounded(item: &Item, cap: usize) -> Result { + let mut counter = LineCounter { + written: 0, + cap, + over: false, + }; + let counted = serde_json::to_writer(&mut counter, &ItemLineRef { item }) + .map_err(FormatError::from) + .and_then(|()| { + counter + .write_all(b"\n") + .map_err(|e| FormatError::Encode(e.to_string())) + }); + if counter.over { + return Err(FormatError::LineTooLong { cap }); + } + counted.map(|()| counter.written) +} + +/// The writer behind [`encode_item_line_len_bounded`]: counts the bytes +/// written and refuses the write that would take the count past `cap`, +/// setting `over`; no byte past the cap is counted and nothing is kept. +struct LineCounter { + written: usize, + cap: usize, + over: bool, +} + +impl Write for LineCounter { + fn write(&mut self, buf: &[u8]) -> std::io::Result { + if buf.len() > self.cap - self.written { + self.over = true; + return Err(std::io::Error::new( + std::io::ErrorKind::WriteZero, + "the line cap", + )); + } + self.written += buf.len(); + Ok(buf.len()) + } + + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } +} + /// Lowercase hex encoding of a byte slice. pub(super) fn hex_encode(bytes: &[u8]) -> String { use std::fmt::Write as _; @@ -1991,6 +2051,65 @@ mod tests { assert!(numbers_line < line, "{numbers_line}"); } + /// The bounded measurement is the line's length up to the cap and + /// `LineTooLong` past it, at the byte (the newline counted, as the + /// encoder counts it), and it holds no line: the counter refuses the + /// write that would pass the cap and counts nothing past it, so a + /// 600 KB item of the worst shape (a 40 MB line) is refused with at most + /// the cap's worth of bytes counted and no buffer. + #[test] + fn the_bounded_line_measurement_stops_at_the_cap_without_a_buffer() { + let small = item(vec![("pk", AttributeValue::S("p".into()))]); + let len = encode_item_line(&small).unwrap().len(); + assert_eq!(encode_item_line_len_bounded(&small, len).unwrap(), len); + assert_eq!( + encode_item_line_len_bounded(&small, MAX_LINE_BYTES).unwrap(), + len + ); + assert!(matches!( + encode_item_line_len_bounded(&small, len - 1), + Err(FormatError::LineTooLong { cap }) if cap == len - 1 + )); + // The counter itself: a write that would pass the cap is refused + // and leaves the count where it was. + let mut counter = LineCounter { + written: 0, + cap: 10, + over: false, + }; + counter.write_all(b"12345").unwrap(); + counter.write_all(b"67890").unwrap(); + assert_eq!((counter.written, counter.over), (10, false)); + assert!(counter.write_all(b"x").is_err()); + assert_eq!((counter.written, counter.over), (10, true)); + // The worst shape over the default limit: 3,700 sets of 81 two-digit + // values at `E-130`, a 40 MB line for a 600 KB item. + let set: BTreeSet = (11..100) + .filter(|m| m % 10 != 0) + .map(|m| format!("0.{}{m}", "0".repeat(129))) + .collect(); + let mut worst = item(vec![("pk", AttributeValue::S("p".into()))]); + worst.insert( + "l".to_owned(), + AttributeValue::L(vec![AttributeValue::NS(set); 3_700]), + ); + assert!(crate::types::item_size_bytes(&worst) > crate::limits::DEFAULT_MAX_ITEM_SIZE_BYTES); + assert!(matches!( + encode_item_line_len_bounded(&worst, MAX_LINE_BYTES), + Err(FormatError::LineTooLong { cap }) if cap == MAX_LINE_BYTES + )); + // A cap of 100 bytes stops the serializer after at most 100 bytes + // of a 40 MB line. + let mut counter = LineCounter { + written: 0, + cap: 100, + over: false, + }; + assert!(serde_json::to_writer(&mut counter, &ItemLineRef { item: &worst }).is_err()); + assert!(counter.over); + assert!(counter.written <= 100, "{}", counter.written); + } + /// Dropping an encoder that did not finish writes nothing more, whether /// it is fresh or has items in its buffer: an abandoned encoder must not /// complete a valid gzip file in the sink. diff --git a/crates/core/src/backup_format/key_order.rs b/crates/core/src/backup_format/key_order.rs index b0060e30c..d8990bdde 100644 --- a/crates/core/src/backup_format/key_order.rs +++ b/crates/core/src/backup_format/key_order.rs @@ -151,6 +151,15 @@ pub fn hash_key_text(value: &AttributeValue) -> Option> { } } +/// An untrusted string bounded for an error message, the format module's +/// rule (`error::truncated`) reached from outside the module: the string +/// when short, a prefix and the total length otherwise. Call sites format +/// the result with `{:?}`. +#[must_use] +pub fn shown(s: &str) -> String { + truncated(s) +} + /// A table's key shape, resolved once from its key schema and attribute /// definitions, able to compare and validate items' keys. #[derive(Debug, Clone, PartialEq, Eq)] @@ -433,9 +442,17 @@ fn parse_number(name: &str, n: &str) -> Result { } /// One key attribute's value checked as [`KeyOrder::validate_key_item`] -/// checks it: the declared scalar type, not empty, and for `N` a valid -/// number in normalized form. The error is the reason alone. -fn check_key_value( +/// checks it: not empty, the declared scalar type, and for `N` a valid +/// number in normalized form, in that order, so a text under a key declared +/// `S` is a value of the wrong type and not a malformed number. The error is +/// the reason alone. Public for the restorable-item validator +/// (`crate::validation::restorable_item`), which checks a multi-part key's +/// attributes one by one with the same rule. +/// +/// # Errors +/// +/// [`FormatError::InvalidKey`] naming the attribute and what is wrong. +pub fn check_key_value( name: &str, ty: ScalarAttributeType, value: &AttributeValue, @@ -454,32 +471,7 @@ fn check_key_value( } let actual = match value { AttributeValue::S(_) => ScalarAttributeType::S, - AttributeValue::N(n) => { - let normalized = validate_and_normalize_number(n).map_err(|e| { - // The reason for a malformed number ends in the input, which - // is shown once, bounded, before it; the other reasons - // (overflow, underflow, too many digits) are fixed texts. - let reason = e.to_string(); - let reason = match reason.strip_suffix(n.as_str()) { - Some(rest) => rest.strip_suffix(": ").unwrap_or(rest).to_owned(), - None => reason, - }; - FormatError::InvalidKey(format!( - "key attribute {:?} value {:?} is not a valid number: {reason}", - shown(), - truncated(n), - )) - })?; - if normalized != *n { - return Err(FormatError::InvalidKey(format!( - "key attribute {:?} value {:?} is not in normalized form ({:?})", - shown(), - truncated(n), - truncated(&normalized) - ))); - } - ScalarAttributeType::N - } + AttributeValue::N(_) => ScalarAttributeType::N, AttributeValue::B(_) => ScalarAttributeType::B, _ => { return Err(FormatError::InvalidKey(format!( @@ -494,6 +486,33 @@ fn check_key_value( shown() ))); } + if let AttributeValue::N(n) = value { + let normalized = validate_and_normalize_number(n).map_err(|e| { + // The reason for a malformed number ends in the input, which is + // shown once, bounded, before it, so the input and the colon + // before it are taken off the end (the wire appends the input + // as it is, trailing whitespace included); the other reasons + // (overflow, underflow, too many digits) are fixed texts. + let reason = e.to_string(); + let reason = match reason.strip_suffix(n.as_str()) { + Some(rest) => rest.strip_suffix(": ").unwrap_or(rest).to_owned(), + None => reason, + }; + FormatError::InvalidKey(format!( + "key attribute {:?} value {:?} is not a valid number: {reason}", + shown(), + truncated(n), + )) + })?; + if normalized != *n { + return Err(FormatError::InvalidKey(format!( + "key attribute {:?} value {:?} is not in normalized form ({:?})", + shown(), + truncated(n), + truncated(&normalized) + ))); + } + } Ok(()) } @@ -973,6 +992,48 @@ mod tests { assert!(err.to_string().len() < 400, "{}", err.to_string().len()); } + /// `check_key_value`'s texts: the declared type is compared before an + /// `N` is parsed, so a text under an `S` key is a value of the wrong + /// type; a malformed `N` under an `N` key gives the wire's reason without + /// the input the wire appends (shown once, bounded, before it) and + /// without the colon that preceded it, whatever whitespace the input + /// ends in; the fixed reasons are carried whole. + #[test] + fn check_key_value_compares_the_type_before_parsing_and_strips_the_echoed_input() { + let reason = |ty, value: AttributeValue| match check_key_value("k", ty, &value) { + Err(FormatError::InvalidKey(reason)) => reason, + other => panic!("{other:?}"), + }; + assert_eq!( + reason(ScalarAttributeType::S, AttributeValue::N("abc".into())), + r#"key attribute "k" is N, the schema declares S"# + ); + for bad in ["abc", "5 ", " 5", "5\t", "1e"] { + assert_eq!( + reason(ScalarAttributeType::N, AttributeValue::N(bad.into())), + format!( + "key attribute \"k\" value {bad:?} is not a valid number: The parameter \ + cannot be converted to a numeric value" + ), + "{bad:?}" + ); + } + assert_eq!( + reason(ScalarAttributeType::N, AttributeValue::N("1E126".into())), + "key attribute \"k\" value \"1E126\" is not a valid number: Number overflow. \ + Attempting to store a number with magnitude larger than supported range" + ); + assert_eq!( + reason(ScalarAttributeType::N, AttributeValue::N(String::new())), + "key attribute \"k\" value \"\" is not a valid number: The parameter cannot be \ + converted to a numeric value" + ); + assert_eq!( + reason(ScalarAttributeType::N, AttributeValue::N("5.0".into())), + r#"key attribute "k" value "5.0" is not in normalized form ("5")"# + ); + } + #[test] fn duplicate_attribute_definitions_are_unorderable() { let ks = vec![KeySchemaElement { diff --git a/crates/core/src/backup_format/mod.rs b/crates/core/src/backup_format/mod.rs index ee88a6c93..ad75ed886 100644 --- a/crates/core/src/backup_format/mod.rs +++ b/crates/core/src/backup_format/mod.rs @@ -73,7 +73,7 @@ mod golden_tests; pub use data_file::{ CompressedDigest, CompressedHasher, DataFileChecksums, DataFileDecoder, DataFileEncoder, DataFileExpectations, GZIP_LEVEL, MAX_LINE_BYTES, decode_data_file, encode_data_file, - encode_item_line, encode_item_line_into, + encode_item_line, encode_item_line_into, encode_item_line_len_bounded, }; pub use ddb_export::{ DYNAMODB_JSON_OUTPUT_FORMAT, EXPORT_SUMMARY_VERSION, ExportFileEntry, ExportSummaryManifest, @@ -81,7 +81,7 @@ pub use ddb_export::{ }; pub use documents::{BackupDocuments, ManifestHeader}; pub use error::{FormatError, JsonError}; -pub use key_order::{KeyOrder, hash_key_text}; +pub use key_order::{KeyOrder, check_key_value, hash_key_text, shown}; pub use layout::{ BACKUP_ID_MILLIS_DIGITS, BackupArn, BackupArnError, BackupArnParts, DATA_PREFIX, EXTENDDB_MANIFEST_FILE, FILES_MANIFEST_FILE, MANIFEST_CHECKSUM_FILE, MAX_BACKUP_ARN_LEN, diff --git a/crates/core/src/limits/mod.rs b/crates/core/src/limits/mod.rs index a894f1b83..5805508bb 100755 --- a/crates/core/src/limits/mod.rs +++ b/crates/core/src/limits/mod.rs @@ -2,6 +2,12 @@ // SPDX-License-Identifier: Apache-2.0 use serde::Deserialize; +/// The default `max_item_size_bytes` (400 KB as `DynamoDB` sizes items), the +/// largest item the backup format's line cap is sized for: the backup job +/// measures the data file line of an item over this size and of no item +/// within it. +pub const DEFAULT_MAX_ITEM_SIZE_BYTES: usize = 409_600; + /// All configurable limits with DynamoDB-compatible defaults. #[derive(Debug, Clone, Deserialize)] #[serde(deny_unknown_fields)] @@ -98,7 +104,7 @@ impl Default for LimitsConfig { } fn default_max_item_size() -> usize { - 409_600 + DEFAULT_MAX_ITEM_SIZE_BYTES } fn default_max_pk_size() -> usize { 2048 diff --git a/crates/core/src/metrics/collector.rs b/crates/core/src/metrics/collector.rs index c5981cdcf..24e3d5cfc 100755 --- a/crates/core/src/metrics/collector.rs +++ b/crates/core/src/metrics/collector.rs @@ -348,6 +348,49 @@ impl MetricsCollector { ); } + /// Record the end of a store-backed backup job: one `BackupJobs` under + /// the outcome and its duration in microseconds under the same + /// dimension. The job measures the duration from the snapshot's open to + /// the end of its last catalog or store call: the AVAILABLE write for a + /// job that succeeded, the FAILED write and the prefix's deletion for + /// one that failed. + pub fn record_backup_job(&self, outcome: super::types::BackupJobOutcome, duration_us: f64) { + let outcome = outcome.to_string(); + self.record(MetricName::BackupJobs, 1.0, None, None, Some(&outcome)); + self.record( + MetricName::BackupJobDuration, + duration_us, + None, + None, + Some(&outcome), + ); + } + + /// Record bytes a backup job wrote to the store; the job records them + /// once it ended AVAILABLE, so a failed job's uploaded and then deleted + /// bytes are not counted. + #[allow(clippy::cast_precision_loss)] // byte counts of one backup + pub fn record_backup_bytes_uploaded(&self, bytes: u64) { + self.record( + MetricName::BackupBytesUploaded, + bytes as f64, + None, + None, + None, + ); + } + + /// Record a running backup job's snapshot age in seconds (gauge). + pub fn record_backup_snapshot_age(&self, age_secs: f64) { + self.record(MetricName::BackupSnapshotAge, age_secs, None, None, None); + } + + /// Record a failed store call of a backup job; `op` names the call + /// (`put` or `delete_prefix`). + pub fn record_backup_store_error(&self, op: &str) { + self.record(MetricName::BackupStoreErrors, 1.0, None, None, Some(op)); + } + /// Record per-segment latency breakdown for a request. pub fn record_segments( &self, diff --git a/crates/core/src/metrics/collector_tests.rs b/crates/core/src/metrics/collector_tests.rs index 26a50b7f6..385b07e83 100755 --- a/crates/core/src/metrics/collector_tests.rs +++ b/crates/core/src/metrics/collector_tests.rs @@ -175,3 +175,60 @@ fn segments_multiple_operations() { assert_eq!(s.count, 1); } } + +/// The backup job's recorders land under their names and dimensions: one +/// `BackupJobs` and one `BackupJobDuration` (microseconds) per job with the +/// outcome as the operation dimension, the bytes and the snapshot age +/// undimensioned, a store error under the call's name. +#[test] +fn backup_job_recorders_land_under_the_outcome_and_the_store_call() { + use crate::metrics::types::BackupJobOutcome; + + let c = MetricsCollector::new(); + c.record_backup_job(BackupJobOutcome::Succeeded, 1_500.0); + c.record_backup_job(BackupJobOutcome::Failed, 300.0); + c.record_backup_bytes_uploaded(4096); + c.record_backup_snapshot_age(12.5); + c.record_backup_store_error("put"); + + let by = |metric: MetricName| { + let mut snaps = c.query(&MetricsQuery { + metric: Some(metric), + window: Some(TimeWindow::AllTime), + ..Default::default() + }); + snaps.sort_by(|a, b| format!("{:?}", a.dimensions).cmp(&format!("{:?}", b.dimensions))); + snaps + }; + let jobs = by(MetricName::BackupJobs); + let outcomes: Vec = jobs.iter().map(|s| format!("{:?}", s.dimensions)).collect(); + assert_eq!( + outcomes, + [ + "[Operation(\"failed\")]".to_owned(), + "[Operation(\"succeeded\")]".to_owned() + ] + ); + assert!(jobs.iter().all(|s| s.count == 1 && s.sum == 1.0)); + let durations = by(MetricName::BackupJobDuration); + assert_eq!(durations.len(), 2); + assert_eq!(durations.iter().map(|s| s.sum).sum::(), 1_800.0); + let bytes = by(MetricName::BackupBytesUploaded); + assert_eq!(bytes.len(), 1); + assert!(bytes[0].dimensions.is_empty()); + assert_eq!(bytes[0].sum, 4096.0); + let age = by(MetricName::BackupSnapshotAge); + assert_eq!(age[0].max, 12.5); + let errors = by(MetricName::BackupStoreErrors); + assert_eq!( + format!("{:?}", errors[0].dimensions), + "[Operation(\"put\")]" + ); + for (outcome, text) in [ + (BackupJobOutcome::Succeeded, "succeeded"), + (BackupJobOutcome::Failed, "failed"), + (BackupJobOutcome::Cancelled, "cancelled"), + ] { + assert_eq!(outcome.to_string(), text); + } +} diff --git a/crates/core/src/metrics/mod.rs b/crates/core/src/metrics/mod.rs index bf1ff0272..637a97c29 100755 --- a/crates/core/src/metrics/mod.rs +++ b/crates/core/src/metrics/mod.rs @@ -17,7 +17,7 @@ mod types; pub use collector::MetricsCollector; pub use types::{ - Dimension, FlushBucket, LatencySegments, MetricName, MetricSnapshot, MetricsBucket, - MetricsQuery, MetricsResponse, OperationSegments, Percentiles, QueryCategory, QuerySource, - TimeWindow, auto_granularity, parse_granularity, window_duration_secs, + BackupJobOutcome, Dimension, FlushBucket, LatencySegments, MetricName, MetricSnapshot, + MetricsBucket, MetricsQuery, MetricsResponse, OperationSegments, Percentiles, QueryCategory, + QuerySource, TimeWindow, auto_granularity, parse_granularity, window_duration_secs, }; diff --git a/crates/core/src/metrics/types.rs b/crates/core/src/metrics/types.rs index 85811deaa..3351c46fd 100755 --- a/crates/core/src/metrics/types.rs +++ b/crates/core/src/metrics/types.rs @@ -55,6 +55,54 @@ pub enum MetricName { WorkerCycleLatency, /// P120e: Worker error count. WorkerErrorCount, + /// Store-backed backup jobs ended, one per job; the operation dimension + /// carries the outcome ([`BackupJobOutcome`]: `succeeded`, `failed` or + /// `cancelled`). `/metrics` emits it under this name as JSON, as it does + /// every metric (the endpoint is not Prometheus exposition format); a + /// converter would give it a name like `backup_jobs_total{outcome}`, and + /// nothing in the server renders that. + BackupJobs, + /// Duration of a store-backed backup job in microseconds, from the + /// snapshot's open to the end of its last catalog or store call (the + /// AVAILABLE write, or the FAILED write and the prefix's deletion); + /// recorded for every outcome, with the outcome as the operation + /// dimension. + BackupJobDuration, + /// Bytes a backup job that ended AVAILABLE wrote to the store: every + /// data file and document as the store reported it stored; a failed + /// job's uploaded and then deleted bytes are not counted. + BackupBytesUploaded, + /// Seconds between a running backup job's snapshot instant and now + /// (gauge), recorded at each confirmed heartbeat. + BackupSnapshotAge, + /// Store operations a backup job saw fail; the operation dimension + /// names the store call (`put` by the job's uploads, `delete_prefix` by + /// its failure path). + BackupStoreErrors, +} + +/// How a store-backed backup job ended, the operation dimension of +/// [`MetricName::BackupJobs`] and [`MetricName::BackupJobDuration`]. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum BackupJobOutcome { + /// The manifest was written and the row is AVAILABLE. + Succeeded, + /// The row is FAILED, or the job stopped without a status write of its + /// own (its claim lost). + Failed, + /// The server's shutdown stopped the job; the row is FAILED with reason + /// `shutdown`. + Cancelled, +} + +impl std::fmt::Display for BackupJobOutcome { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(match self { + Self::Succeeded => "succeeded", + Self::Failed => "failed", + Self::Cancelled => "cancelled", + }) + } } impl std::fmt::Display for MetricName { @@ -86,6 +134,11 @@ impl std::fmt::Display for MetricName { Self::WorkerLastSuccess => f.write_str("WorkerLastSuccess"), Self::WorkerCycleLatency => f.write_str("WorkerCycleLatency"), Self::WorkerErrorCount => f.write_str("WorkerErrorCount"), + Self::BackupJobs => f.write_str("BackupJobs"), + Self::BackupJobDuration => f.write_str("BackupJobDuration"), + Self::BackupBytesUploaded => f.write_str("BackupBytesUploaded"), + Self::BackupSnapshotAge => f.write_str("BackupSnapshotAge"), + Self::BackupStoreErrors => f.write_str("BackupStoreErrors"), } } } @@ -131,6 +184,9 @@ pub enum QuerySource { LoginAttemptCleanup, /// Background: the backup reconciler. BackupReconciler, + /// A store-backed backup job (the worker metrics are recorded once per + /// job, at its end). + BackupJob, /// Management CLI commands. Management, } @@ -152,6 +208,7 @@ impl std::fmt::Display for QuerySource { Self::ControlPlane => f.write_str("ControlPlane"), Self::LoginAttemptCleanup => f.write_str("LoginAttemptCleanup"), Self::BackupReconciler => f.write_str("BackupReconciler"), + Self::BackupJob => f.write_str("BackupJob"), Self::Management => f.write_str("Management"), } } diff --git a/crates/core/src/validation/mod.rs b/crates/core/src/validation/mod.rs index 3decb4666..3ebc3d2c2 100755 --- a/crates/core/src/validation/mod.rs +++ b/crates/core/src/validation/mod.rs @@ -1,8 +1,13 @@ // Copyright 2026 ExtendDB contributors // SPDX-License-Identifier: Apache-2.0 pub mod number; +pub mod restorable_item; pub mod vector_item; +pub use restorable_item::{ + RestorableItemError, RestorableItemReason, RestorableKeySchema, RestorableLimits, + SHOWN_VALUE_BYTES, validate_restorable_item, +}; pub use vector_item::{ MAX_HASH_KEY_SIZE, MAX_INLINE_FILTER_SIZE, validate_vector_write, validate_vector_write_changed, vector_components, vector_norm, diff --git a/crates/core/src/validation/restorable_item.rs b/crates/core/src/validation/restorable_item.rs new file mode 100644 index 000000000..07f65d2ff --- /dev/null +++ b/crates/core/src/validation/restorable_item.rs @@ -0,0 +1,1465 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! Whether a stored item can be restored: the check a backup job runs on +//! every item it reads, before the item is written to a data file, and the +//! check PR 9a's restore binds for every item it reads back. +//! +//! The write paths let some items in that a restore's `PutItem`-shaped write +//! would refuse: an `N` value that is not a valid number (`1E126`, `abc`; +//! #618 names the four paths that store one unvalidated: `BatchWriteItem`, +//! `TransactWriteItems`, `UpdateItem` on an absent item, `ImportTable`), an +//! item over the configured size limit stored under a larger one, a key +//! attribute or attribute name over this instance's limits stored under +//! larger ones. The key-shape rules (a key attribute missing, of the wrong +//! type or empty, an `N` key not in normalized form) cover rows of earlier +//! builds and rows edited outside the wire: every write path of this tree +//! refuses such a key, and the snapshot streams hand every valid `N` over +//! normalized, so the normalized-form branch is unreachable from a +//! snapshot; it stays because it is the format's own rule +//! ([`crate::backup_format::check_key_value`]). A backup that carried such +//! an item could not be restored in full, so the job refuses the backup +//! instead and names the item (its key), the attribute path and the reason. +//! The base key alone is checked: an index key attribute of the wrong type +//! or empty is left to the restore's index build. +//! +//! One walk over the item: every `N` and every `NS` member is parsed once +//! on the accepting path (a key `N` that fails the key rule is parsed a +//! second time to classify the failure), the nesting depth is counted as it +//! goes, every attribute name is measured against `max_attribute_name_bytes` +//! and the key attributes are checked with the backup format's own key rule +//! and against the hash and range key size limits, the rules the wire's +//! `check_key_size` and `check_attribute_name_size` apply, so a key that +//! passes here is one a key-ordered file's range can carry and one the +//! target's write accepts. The item's size is one call on the whole item +//! (`item_size_bytes`). The request path's `validate_put_item` stays what it +//! is: it returns the wire error with no attribute path and is not called +//! here. +//! +//! Every untrusted text an error carries (a key value, an attribute name, a +//! path segment, a number the wire's reason echoes) is bounded on its own +//! ([`SHOWN_VALUE_BYTES`]; a key value at [`SHOWN_KEY_VALUE_BYTES`]) and +//! rendered as a JSON string's content, with the line separators U+0085, +//! U+2028 and U+2029 escaped beside the control characters, so neither a +//! control character nor a Unicode line separator in a name or a number +//! can end the log line it is written in, and a key of any +//! legal size appears whole while only an oversized value is cut. The +//! bound is per text, so the whole message is bounded by the item's own +//! size times a small factor (a byte renders as at most six), not by a +//! constant: a path of thirty nested maps with names at the bound is about +//! 127 KB. + +use std::borrow::Cow; +use std::fmt; + +use crate::backup_format::{FormatError, check_key_value, hash_key_text, shown}; +use crate::limits::LimitsConfig; +use crate::types::{ + AttributeDefinition, AttributeValue, Item, KeySchemaElement, KeyType, ScalarAttributeType, + item_size_bytes, +}; +use crate::validation::number::validate_and_normalize_number; + +/// Longest rendering of one untrusted text in a [`RestorableItemError`] +/// other than a key value (an attribute name, a path segment, a set member, +/// the number a reason echoes), in the bytes the message writes: the text +/// as a JSON string's content, quotes not counted. A round figure that +/// keeps a path segment to four times the longest key attribute name +/// (255 bytes), after which a name is cut. +pub const SHOWN_VALUE_BYTES: usize = 4096; + +/// Longest rendering of a key attribute's value, in the same measure: six +/// times the 2048-byte hash key limit, the most a byte escapes to +/// (`\u00XX`), so a key of any legal size renders whole whatever its +/// bytes, and the 2732-character base64 of a 2048-byte `B` key fits. +pub const SHOWN_KEY_VALUE_BYTES: usize = 6 * 2048; + +/// The prefix kept of a text over its bound, in the same measure, followed +/// by `... ` with the text's length. +const SHOWN_VALUE_PREFIX_BYTES: usize = 1024; + +/// The instance's limits a restore would hold an item to, for +/// [`validate_restorable_item`]: the item size, the hash and range key +/// sizes and the attribute name size of `LimitsConfig`. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct RestorableLimits { + pub max_item_size_bytes: usize, + pub max_partition_key_size_bytes: usize, + pub max_sort_key_size_bytes: usize, + pub max_attribute_name_bytes: usize, +} + +impl From<&LimitsConfig> for RestorableLimits { + fn from(limits: &LimitsConfig) -> Self { + Self { + max_item_size_bytes: limits.max_item_size_bytes, + max_partition_key_size_bytes: limits.max_partition_key_size_bytes, + max_sort_key_size_bytes: limits.max_sort_key_size_bytes, + max_attribute_name_bytes: limits.max_attribute_name_bytes, + } + } +} + +/// One key attribute of a [`RestorableKeySchema`]. +#[derive(Debug, Clone, PartialEq, Eq)] +struct KeyAttribute { + name: String, + ty: ScalarAttributeType, + key_type: KeyType, +} + +/// The key attributes of a table, resolved once from its key schema and +/// attribute definitions, for [`validate_restorable_item`]. Unlike +/// `KeyOrder`, which only a hash-only or single-range-key table has, every +/// table has one of these: a multi-part key is a list of more than two +/// attributes. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct RestorableKeySchema { + attributes: Vec, +} + +impl RestorableKeySchema { + /// Resolve the key attributes, in key-schema order. + /// + /// # Errors + /// + /// The name of a key attribute that has no attribute definition, or more + /// than one; `CreateTable` and the manifest reader both refuse such a + /// schema, so a table's own schema never fails here. + pub fn resolve( + key_schema: &[KeySchemaElement], + attribute_definitions: &[AttributeDefinition], + ) -> Result { + let mut attributes = Vec::with_capacity(key_schema.len()); + for element in key_schema { + let name = element.attribute_name.as_str(); + let mut matching = attribute_definitions + .iter() + .filter(|d| d.attribute_name == name) + .map(|d| d.attribute_type); + let ty = matching.next().ok_or_else(|| { + format!( + "key attribute {:?} has no attribute definition", + shown(name) + ) + })?; + if matching.next().is_some() { + return Err(format!( + "key attribute {:?} has more than one attribute definition", + shown(name) + )); + } + attributes.push(KeyAttribute { + name: name.to_owned(), + ty, + key_type: element.key_type, + }); + } + Ok(Self { attributes }) + } + + /// Names of the key attributes, in key-schema order. + pub fn attribute_names(&self) -> impl Iterator { + self.attributes.iter().map(|a| a.name.as_str()) + } + + /// Whether `name` is a key attribute. + fn is_key(&self, name: &str) -> bool { + self.attributes.iter().any(|a| a.name == name) + } + + /// The key attributes of `item` as compact JSON (`{"pk":{"S":"a"}}`) + /// for an error message or a log line, each name bounded on its own + /// ([`SHOWN_VALUE_BYTES`]) and each value on its own + /// ([`SHOWN_KEY_VALUE_BYTES`]), so every key attribute appears and only + /// an oversized value is cut; a key attribute the item lacks is left + /// out. A `B` value is its base64 text, as the format writes it; a value + /// that is not a scalar (a key attribute of the wrong type) is its own + /// JSON, cut to a JSON string when oversized. + #[must_use] + pub fn shown_key(&self, item: &Item) -> String { + let mut out = String::from("{"); + let mut first = true; + for attribute in &self.attributes { + let Some(value) = item.get(&attribute.name) else { + continue; + }; + if !first { + out.push(','); + } + first = false; + push_json_string(&mut out, &attribute.name, SHOWN_VALUE_BYTES); + out.push(':'); + let tag = match value { + AttributeValue::S(_) => Some("S"), + AttributeValue::N(_) => Some("N"), + AttributeValue::B(_) => Some("B"), + _ => None, + }; + match (tag, hash_key_text(value)) { + (Some(tag), Some(text)) => { + out.push_str("{\""); + out.push_str(tag); + out.push_str("\":"); + push_json_string(&mut out, &text, SHOWN_KEY_VALUE_BYTES); + out.push('}'); + } + _ => { + let text = serde_json::to_string(value).unwrap_or_default(); + if text.len() <= SHOWN_KEY_VALUE_BYTES { + out.push_str(&text); + } else { + push_json_string(&mut out, &text, SHOWN_KEY_VALUE_BYTES); + } + } + } + } + out.push('}'); + out + } +} + +/// Append `s` to `out` as a JSON string: quotes around [`bounded_text`] +/// with `bound`. +fn push_json_string(out: &mut String, s: &str, bound: usize) { + out.push('"'); + out.push_str(&bounded_text(s, bound)); + out.push('"'); +} + +/// Bytes a message writes for `c` inside a JSON string: `\"`, `\\` and +/// the five short escapes take two, the other control characters and the +/// three line separators `\u00XX`-style escapes of six, everything else +/// its UTF-8 length. +fn json_escaped_len(c: char) -> usize { + match c { + '"' | '\\' | '\n' | '\r' | '\t' | '\u{8}' | '\u{c}' => 2, + c if (c as u32) < 0x20 || is_line_separator(c) => 6, + c => c.len_utf8(), + } +} + +/// The Unicode line separators `serde_json` leaves raw and a reader that +/// breaks lines the Unicode way (Python's `str.splitlines`) breaks on: +/// NEXT LINE, LINE SEPARATOR and PARAGRAPH SEPARATOR. +fn is_line_separator(c: char) -> bool { + matches!(c, '\u{85}' | '\u{2028}' | '\u{2029}') +} + +/// `s` as a JSON string's content: the escapes `serde_json` writes, then +/// the three line separators as `\u0085`, `\u2028` and `\u2029` (valid JSON +/// escapes `serde_json` does not write), no quotes, so neither a control +/// character nor a line separator in a tenant's text can stand as a line +/// of its own in a log; borrowed when nothing in it escapes. +fn json_escaped(s: &str) -> Cow<'_, str> { + if s.chars().all(|c| json_escaped_len(c) == c.len_utf8()) { + return Cow::Borrowed(s); + } + let mut quoted = serde_json::to_string(s).unwrap_or_default(); + quoted.pop(); + if !quoted.is_empty() { + quoted.remove(0); + } + if quoted.chars().any(is_line_separator) { + quoted = quoted + .replace('\u{85}', r"\u0085") + .replace('\u{2028}', r"\u2028") + .replace('\u{2029}', r"\u2029"); + } + Cow::Owned(quoted) +} + +/// `s` as a message writes it, a JSON string's content ([`json_escaped`]), +/// bounded: whole when its escaped text is at most `bound` bytes, otherwise +/// a prefix of at most [`SHOWN_VALUE_PREFIX_BYTES`] of escaped text and +/// `... ` with the text's length. +fn bounded_text(s: &str, bound: usize) -> Cow<'_, str> { + let mut rendered = 0; + let mut prefix = 0; + for c in s.chars() { + rendered += json_escaped_len(c); + if rendered > bound { + return Cow::Owned(format!( + "{}... <{} bytes>", + json_escaped(&s[..prefix]), + s.len() + )); + } + if rendered <= SHOWN_VALUE_PREFIX_BYTES { + prefix += c.len_utf8(); + } + } + json_escaped(s) +} + +/// Why an item cannot be restored. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum RestorableItemReason { + /// An `N` value, or a member of an `NS`, that is not a valid `DynamoDB` + /// number: the wire's own reason text, with the member appended for an + /// `NS` when the text does not already end in it (the #618 case; a + /// FAILED backup names the issue when its first refused item has this + /// reason). + InvalidNumber(String), + /// A key attribute missing, of the wrong type, empty, an `N` key not in + /// normalized form (the backup format's key rule's reason text), or a + /// key value over the hash or range key size limit. + KeyShape(String), + /// An attribute name (the path's last segment) longer than the + /// attribute name limit. + AttributeNameTooLong { size: usize, max: usize }, + /// The item is larger than the configured limit. + TooLarge { size: usize, max: usize }, + /// A map or list nests deeper than the item nesting limit. + TooDeep, + /// The item's data file line would exceed the format's line cap + /// (`MAX_LINE_BYTES`): the backup job measures the line of an item over + /// the default item limit, the largest the cap is sized for, and stops + /// measuring once the cap is passed, so the line's own length is not + /// known. + LineTooLong { cap: usize }, +} + +impl fmt::Display for RestorableItemReason { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::InvalidNumber(reason) | Self::KeyShape(reason) => f.write_str(reason), + Self::AttributeNameTooLong { size, max } => { + write!( + f, + "attribute name of {size} bytes exceeds the limit of {max} bytes" + ) + } + Self::TooLarge { size, max } => { + write!(f, "item size {size} bytes exceeds the limit of {max} bytes") + } + Self::TooDeep => write!( + f, + "attributes nest beyond {} levels", + super::MAX_ITEM_NESTING_DEPTH + ), + Self::LineTooLong { cap } => { + write!(f, "data file line exceeds the line cap of {cap} bytes") + } + } + } +} + +/// An item [`validate_restorable_item`] refused: which item (its key +/// attributes as compact JSON, each name and value bounded), where (the +/// attribute path, `a.b[2].c` for a member of a list inside a map, each +/// segment bounded, empty for a rule about the whole item) and why. +/// `Display` writes `key {key}, attribute {path}: {reason}`, without the +/// attribute clause when the path is empty. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct RestorableItemError { + pub key: String, + pub path: String, + pub reason: RestorableItemReason, +} + +impl fmt::Display for RestorableItemError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + if self.path.is_empty() { + write!(f, "key {}: {}", self.key, self.reason) + } else { + write!( + f, + "key {}, attribute {}: {}", + self.key, self.path, self.reason + ) + } + } +} + +impl std::error::Error for RestorableItemError {} + +/// Check that `item` is one a restore under `limits` can write: its key +/// attributes present with the declared types, not empty, an `N` key +/// normalized, each within its key size limit; every attribute name within +/// the name limit; every `N` and `NS` member a valid number; no map or list +/// nested beyond the item nesting limit; the item within the item size +/// limit. The first failure, in that order (key attributes in key-schema +/// order, then the other attributes in name order, then the size), is the +/// error. +/// +/// # Errors +/// +/// [`RestorableItemError`] naming the item's key, the attribute path and the +/// reason. +pub fn validate_restorable_item( + item: &Item, + key: &RestorableKeySchema, + limits: &RestorableLimits, +) -> Result<(), RestorableItemError> { + let refuse = |path: String, reason: RestorableItemReason| RestorableItemError { + key: key.shown_key(item), + path, + reason, + }; + for attribute in &key.attributes { + let name = attribute.name.as_str(); + let path = || bounded_text(name, SHOWN_VALUE_BYTES).into_owned(); + if name.len() > limits.max_attribute_name_bytes { + return Err(refuse( + path(), + RestorableItemReason::AttributeNameTooLong { + size: name.len(), + max: limits.max_attribute_name_bytes, + }, + )); + } + let Some(value) = item.get(name) else { + return Err(refuse( + path(), + RestorableItemReason::KeyShape(format!("key attribute {:?} missing", shown(name))), + )); + }; + if let Err(e) = check_key_value(name, attribute.ty, value) { + let reason = match e { + FormatError::InvalidKey(reason) => reason, + other => other.to_string(), + }; + // Classified by the declared type first: a text that is not a + // number under an `N` key is the #618 case; under any other + // declared type it is a key of the wrong type. + let reason = match (value, attribute.ty) { + (AttributeValue::N(n), ScalarAttributeType::N) => { + match validate_and_normalize_number(n) { + Err(wire) => RestorableItemReason::InvalidNumber(number_reason(n, &wire)), + Ok(_) => RestorableItemReason::KeyShape(reason), + } + } + _ => RestorableItemReason::KeyShape(reason), + }; + return Err(refuse(path(), reason)); + } + let (max, kind) = match attribute.key_type { + KeyType::Hash => (limits.max_partition_key_size_bytes, "hash"), + KeyType::Range => (limits.max_sort_key_size_bytes, "range"), + }; + let size = super::key_value_byte_size(value); + if size > max { + return Err(refuse( + path(), + RestorableItemReason::KeyShape(format!( + "key attribute {:?} has {size} bytes; the {kind} key limit is {max} bytes", + shown(name) + )), + )); + } + } + let mut path = String::new(); + for (name, value) in item { + if key.is_key(name) { + continue; + } + path.clear(); + push_segment(&mut path, name); + if let Err(reason) = + check_name(name, limits).and_then(|()| walk(value, &mut path, 0, limits)) + { + return Err(refuse(std::mem::take(&mut path), reason)); + } + } + let size = item_size_bytes(item); + if size > limits.max_item_size_bytes { + return Err(refuse( + String::new(), + RestorableItemReason::TooLarge { + size, + max: limits.max_item_size_bytes, + }, + )); + } + Ok(()) +} + +/// The attribute name rule the wire's `check_attribute_name_size` applies, +/// at the top level and for every map member. +fn check_name(name: &str, limits: &RestorableLimits) -> Result<(), RestorableItemReason> { + if name.len() > limits.max_attribute_name_bytes { + return Err(RestorableItemReason::AttributeNameTooLong { + size: name.len(), + max: limits.max_attribute_name_bytes, + }); + } + Ok(()) +} + +/// Append one attribute name to `path`, bounded and escaped +/// ([`bounded_text`]). +fn push_segment(path: &mut String, name: &str) { + path.push_str(&bounded_text(name, SHOWN_VALUE_BYTES)); +} + +/// Visit `value` and everything under it. On an error `path` names the +/// offending value; otherwise it is left as it was on entry. +fn walk( + value: &AttributeValue, + path: &mut String, + depth: usize, + limits: &RestorableLimits, +) -> Result<(), RestorableItemReason> { + match value { + AttributeValue::N(n) => number(n), + AttributeValue::NS(set) => { + for (i, n) in set.iter().enumerate() { + if let Err(RestorableItemReason::InvalidNumber(text)) = number(n) { + path.push_str(&format!("[{i}]")); + let member = bounded_text(n, SHOWN_VALUE_BYTES); + let text = if text.ends_with(&*member) { + text + } else { + format!("{text} (member {member})") + }; + return Err(RestorableItemReason::InvalidNumber(text)); + } + } + Ok(()) + } + AttributeValue::L(list) => { + let next = depth + 1; + if next >= super::MAX_ITEM_NESTING_DEPTH { + return Err(RestorableItemReason::TooDeep); + } + let len = path.len(); + for (i, v) in list.iter().enumerate() { + path.push_str(&format!("[{i}]")); + walk(v, path, next, limits)?; + path.truncate(len); + } + Ok(()) + } + AttributeValue::M(map) => { + let next = depth + 1; + if next >= super::MAX_ITEM_NESTING_DEPTH { + return Err(RestorableItemReason::TooDeep); + } + let len = path.len(); + for (name, v) in map { + path.push('.'); + push_segment(path, name); + check_name(name, limits)?; + walk(v, path, next, limits)?; + path.truncate(len); + } + Ok(()) + } + AttributeValue::S(_) + | AttributeValue::B(_) + | AttributeValue::SS(_) + | AttributeValue::BS(_) + | AttributeValue::Bool(_) + | AttributeValue::Null => Ok(()), + } +} + +fn number(n: &str) -> Result<(), RestorableItemReason> { + validate_and_normalize_number(n) + .map(drop) + .map_err(|e| RestorableItemReason::InvalidNumber(number_reason(n, &e))) +} + +/// The wire's reason for `n`, which ends in `n` for a malformed number: that +/// echo is bounded and escaped like every other untrusted text +/// ([`bounded_text`]), so the text is the wire's byte for byte up to +/// [`SHOWN_VALUE_BYTES`] for an input with nothing to escape. +fn number_reason(n: &str, e: &crate::error::DynamoDbError) -> String { + let text = e.message(); + match text.strip_suffix(n) { + Some(head) => format!("{head}{}", bounded_text(n, SHOWN_VALUE_BYTES)), + None => text.to_owned(), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::types::KeyType; + use std::collections::{BTreeMap, BTreeSet}; + + const MAX: usize = 400 * 1024; + + /// The default limits with `max_item_size_bytes` at `max`. + fn limits(max: usize) -> RestorableLimits { + RestorableLimits { + max_item_size_bytes: max, + ..RestorableLimits::from(&LimitsConfig::default()) + } + } + + fn element(name: &str, key_type: KeyType) -> KeySchemaElement { + KeySchemaElement { + attribute_name: name.to_owned(), + key_type, + } + } + + fn definition(name: &str, ty: ScalarAttributeType) -> AttributeDefinition { + AttributeDefinition { + attribute_name: name.to_owned(), + attribute_type: ty, + } + } + + /// `pk` S alone. + fn hash_only() -> RestorableKeySchema { + RestorableKeySchema::resolve( + &[element("pk", KeyType::Hash)], + &[definition("pk", ScalarAttributeType::S)], + ) + .unwrap() + } + + /// `pk` S, `sk` N. + fn composite() -> RestorableKeySchema { + RestorableKeySchema::resolve( + &[element("pk", KeyType::Hash), element("sk", KeyType::Range)], + &[ + definition("pk", ScalarAttributeType::S), + definition("sk", ScalarAttributeType::N), + ], + ) + .unwrap() + } + + /// `pk` S, `sk` N, `sk2` B: a key `KeyOrder` cannot resolve. + fn multi_part() -> RestorableKeySchema { + RestorableKeySchema::resolve( + &[ + element("pk", KeyType::Hash), + element("sk", KeyType::Range), + element("sk2", KeyType::Range), + ], + &[ + definition("pk", ScalarAttributeType::S), + definition("sk", ScalarAttributeType::N), + definition("sk2", ScalarAttributeType::B), + ], + ) + .unwrap() + } + + fn s(v: &str) -> AttributeValue { + AttributeValue::S(v.to_owned()) + } + + fn n(v: &str) -> AttributeValue { + AttributeValue::N(v.to_owned()) + } + + fn item(pairs: &[(&str, AttributeValue)]) -> Item { + pairs + .iter() + .map(|(k, v)| ((*k).to_owned(), v.clone())) + .collect() + } + + fn refused(item: &Item, key: &RestorableKeySchema, max: usize) -> RestorableItemError { + validate_restorable_item(item, key, &limits(max)).expect_err("refused") + } + + #[test] + fn a_plain_item_passes_on_every_key_shape() { + let it = item(&[ + ("pk", s("a")), + ("sk", n("1")), + ("sk2", AttributeValue::B(vec![1])), + ("x", n("1E125")), + ( + "y", + AttributeValue::NS(BTreeSet::from(["1".to_owned(), "2.5".to_owned()])), + ), + ]); + validate_restorable_item(&it, &hash_only(), &limits(MAX)).unwrap(); + validate_restorable_item(&it, &composite(), &limits(MAX)).unwrap(); + validate_restorable_item(&it, &multi_part(), &limits(MAX)).unwrap(); + } + + #[test] + fn an_overflowing_number_in_a_non_key_attribute_is_refused_with_its_path() { + let it = item(&[("pk", s("a")), ("amount", n("1E126"))]); + let e = refused(&it, &hash_only(), MAX); + assert_eq!(e.key, r#"{"pk":{"S":"a"}}"#); + assert_eq!(e.path, "amount"); + let RestorableItemReason::InvalidNumber(reason) = &e.reason else { + panic!("{e:?}"); + }; + assert!(reason.starts_with("Number overflow"), "{reason}"); + assert_eq!( + e.to_string(), + format!(r#"key {{"pk":{{"S":"a"}}}}, attribute amount: {reason}"#) + ); + } + + #[test] + fn a_non_numeric_text_is_refused_with_the_wire_reason() { + let it = item(&[("pk", s("a")), ("amount", n("abc"))]); + let e = refused(&it, &hash_only(), MAX); + assert_eq!(e.path, "amount"); + assert_eq!( + e.reason, + RestorableItemReason::InvalidNumber( + "The parameter cannot be converted to a numeric value: abc".to_owned() + ) + ); + } + + #[test] + fn a_nested_number_names_its_path_through_maps_and_lists() { + let inner = AttributeValue::M(BTreeMap::from([("c".to_owned(), n("abc"))])); + let list = AttributeValue::L(vec![n("1"), n("2"), inner]); + let a = AttributeValue::M(BTreeMap::from([("b".to_owned(), list)])); + let it = item(&[("pk", s("a")), ("a", a)]); + let e = refused(&it, &hash_only(), MAX); + assert_eq!(e.path, "a.b[2].c"); + assert!(matches!(e.reason, RestorableItemReason::InvalidNumber(_))); + } + + /// A set member's reason carries the member, since the position is the + /// sorted set's and the overflow text alone does not name the value; a + /// text that already ends in the member (the malformed case) is left as + /// it is. + #[test] + fn a_number_set_member_names_its_position_and_the_member() { + let set = AttributeValue::NS(BTreeSet::from([ + "1".to_owned(), + "2".to_owned(), + "x".to_owned(), + ])); + let it = item(&[("pk", s("a")), ("ns", set)]); + let e = refused(&it, &hash_only(), MAX); + assert_eq!(e.path, "ns[2]"); + assert_eq!( + e.reason, + RestorableItemReason::InvalidNumber( + "The parameter cannot be converted to a numeric value: x".to_owned() + ) + ); + let set = AttributeValue::NS(BTreeSet::from(["01E126".to_owned(), "1E126".to_owned()])); + let it = item(&[("pk", s("a")), ("ns", set)]); + let e = refused(&it, &hash_only(), MAX); + assert_eq!(e.path, "ns[0]"); + let RestorableItemReason::InvalidNumber(reason) = &e.reason else { + panic!("{e:?}"); + }; + assert!(reason.starts_with("Number overflow"), "{reason}"); + assert!(reason.ends_with(" (member 01E126)"), "{reason}"); + } + + /// The path is rebuilt from the attribute for each sibling: a list whose + /// first member is a valid map and whose second member is bad names the + /// second, not a path that still carries the first's members. + #[test] + fn the_path_does_not_carry_a_sibling_that_passed() { + let ok = AttributeValue::M(BTreeMap::from([("deep".to_owned(), n("1"))])); + let list = AttributeValue::L(vec![ok, n("abc")]); + let it = item(&[("pk", s("a")), ("l", list)]); + let e = refused(&it, &hash_only(), MAX); + assert_eq!(e.path, "l[1]"); + } + + #[test] + fn validation_does_not_stop_at_the_first_valid_number() { + // Three valid numbers before the bad one, in name order. + let it = item(&[ + ("pk", s("a")), + ("n1", n("1")), + ("n2", n("2")), + ("n3", n("3")), + ("n4", n("1E126")), + ]); + assert_eq!(refused(&it, &hash_only(), MAX).path, "n4"); + } + + #[test] + fn an_item_over_the_size_limit_is_refused_without_a_path() { + let it = item(&[("pk", s("a")), ("blob", s(&"x".repeat(100)))]); + let size = item_size_bytes(&it); + validate_restorable_item(&it, &hash_only(), &limits(size)).unwrap(); + let e = refused(&it, &hash_only(), size - 1); + assert_eq!(e.path, ""); + assert_eq!( + e.reason, + RestorableItemReason::TooLarge { + size, + max: size - 1 + } + ); + assert_eq!( + e.to_string(), + format!( + r#"key {{"pk":{{"S":"a"}}}}: item size {size} bytes exceeds the limit of {} bytes"#, + size - 1 + ) + ); + } + + #[test] + fn nesting_at_the_limit_is_refused_and_one_short_of_it_passes() { + let nest = |levels: usize| { + let mut v = n("1"); + for _ in 0..levels { + v = AttributeValue::M(BTreeMap::from([("m".to_owned(), v)])); + } + item(&[("pk", s("a")), ("deep", v)]) + }; + let limit = super::super::MAX_ITEM_NESTING_DEPTH; + validate_restorable_item(&nest(limit - 1), &hash_only(), &limits(MAX)).unwrap(); + let e = refused(&nest(limit), &hash_only(), MAX); + assert_eq!(e.reason, RestorableItemReason::TooDeep); + assert_eq!(e.path, format!("deep{}", ".m".repeat(limit - 1))); + // The same rule the request path applies. + super::super::validate_item_nesting_depth(&nest(limit - 1)).unwrap(); + super::super::validate_item_nesting_depth(&nest(limit)).unwrap_err(); + } + + #[test] + fn a_missing_key_attribute_is_refused() { + let it = item(&[("pk", s("a"))]); + let e = refused(&it, &composite(), MAX); + assert_eq!(e.path, "sk"); + assert_eq!(e.key, r#"{"pk":{"S":"a"}}"#); + assert_eq!( + e.reason, + RestorableItemReason::KeyShape(r#"key attribute "sk" missing"#.to_owned()) + ); + } + + #[test] + fn a_key_attribute_of_the_wrong_type_is_refused() { + let it = item(&[("pk", s("a")), ("sk", s("1"))]); + let e = refused(&it, &composite(), MAX); + assert_eq!(e.path, "sk"); + assert_eq!( + e.reason, + RestorableItemReason::KeyShape( + r#"key attribute "sk" is S, the schema declares N"#.to_owned() + ) + ); + assert_eq!(e.key, r#"{"pk":{"S":"a"},"sk":{"S":"1"}}"#); + } + + /// A text that is not a number under a key declared `S` is a key of the + /// wrong type, not the #618 case: the declared type decides first. + #[test] + fn a_non_numeric_n_under_an_s_key_is_a_key_shape_error() { + let it = item(&[("pk", n("abc"))]); + let e = refused(&it, &hash_only(), MAX); + assert_eq!(e.path, "pk"); + assert_eq!( + e.reason, + RestorableItemReason::KeyShape( + r#"key attribute "pk" is N, the schema declares S"#.to_owned() + ) + ); + } + + #[test] + fn an_empty_key_value_is_refused() { + let it = item(&[("pk", s(""))]); + let e = refused(&it, &hash_only(), MAX); + assert_eq!(e.path, "pk"); + assert_eq!( + e.reason, + RestorableItemReason::KeyShape(r#"key attribute "pk" is empty"#.to_owned()) + ); + } + + /// A bad number in the range key is the #618 case, not a key-shape one: + /// the backup's FAILED reason names the issue by this variant. + #[test] + fn an_invalid_number_in_a_range_key_is_an_invalid_number() { + for bad in ["abc", "1E126"] { + let it = item(&[("pk", s("a")), ("sk", n(bad))]); + let e = refused(&it, &composite(), MAX); + assert_eq!(e.path, "sk"); + assert!( + matches!(e.reason, RestorableItemReason::InvalidNumber(_)), + "{bad}: {e:?}" + ); + } + } + + #[test] + fn a_valid_but_unnormalized_number_key_is_a_key_shape_error() { + let it = item(&[("pk", s("a")), ("sk", n("1.0"))]); + let e = refused(&it, &composite(), MAX); + assert_eq!(e.path, "sk"); + let RestorableItemReason::KeyShape(reason) = &e.reason else { + panic!("{e:?}"); + }; + assert!(reason.contains("not in normalized form"), "{reason}"); + } + + #[test] + fn a_multi_part_key_is_checked_attribute_by_attribute() { + let it = item(&[ + ("pk", s("a")), + ("sk", n("1")), + ("sk2", AttributeValue::B(vec![])), + ]); + let e = refused(&it, &multi_part(), MAX); + assert_eq!(e.path, "sk2"); + assert_eq!( + e.reason, + RestorableItemReason::KeyShape(r#"key attribute "sk2" is empty"#.to_owned()) + ); + assert_eq!(e.key, r#"{"pk":{"S":"a"},"sk":{"N":"1"},"sk2":{"B":""}}"#); + } + + #[test] + fn key_attributes_are_checked_before_the_other_attributes() { + let it = item(&[("amount", n("abc")), ("pk", s(""))]); + assert_eq!(refused(&it, &hash_only(), MAX).path, "pk"); + } + + #[test] + fn a_schema_whose_key_attribute_lacks_a_definition_does_not_resolve() { + let err = RestorableKeySchema::resolve(&[element("pk", KeyType::Hash)], &[]).unwrap_err(); + assert_eq!(err, r#"key attribute "pk" has no attribute definition"#); + let err = RestorableKeySchema::resolve( + &[element("pk", KeyType::Hash)], + &[ + definition("pk", ScalarAttributeType::S), + definition("pk", ScalarAttributeType::N), + ], + ) + .unwrap_err(); + assert_eq!( + err, + r#"key attribute "pk" has more than one attribute definition"# + ); + } + + /// The key sizes against the instance's limits, the rule the wire's + /// `check_key_size` applies: at the limit the key passes, one byte over + /// it is refused, hash and range each against their own limit; a `B` + /// key's size is its bytes. + #[test] + fn a_key_over_its_size_limit_is_refused_at_the_boundary() { + let limits = RestorableLimits { + max_partition_key_size_bytes: 10, + max_sort_key_size_bytes: 4, + ..limits(MAX) + }; + let ok = item(&[("pk", s(&"p".repeat(10))), ("sk", n("1234"))]); + validate_restorable_item(&ok, &composite(), &limits).unwrap(); + let hash = item(&[("pk", s(&"p".repeat(11))), ("sk", n("1"))]); + let e = validate_restorable_item(&hash, &composite(), &limits).unwrap_err(); + assert_eq!(e.path, "pk"); + assert_eq!( + e.reason, + RestorableItemReason::KeyShape( + r#"key attribute "pk" has 11 bytes; the hash key limit is 10 bytes"#.to_owned() + ) + ); + let range = item(&[("pk", s("p")), ("sk", n("12345"))]); + let e = validate_restorable_item(&range, &composite(), &limits).unwrap_err(); + assert_eq!(e.path, "sk"); + assert_eq!( + e.reason, + RestorableItemReason::KeyShape( + r#"key attribute "sk" has 5 bytes; the range key limit is 4 bytes"#.to_owned() + ) + ); + let binary = item(&[ + ("pk", s("p")), + ("sk", n("1")), + ("sk2", AttributeValue::B(vec![0; 5])), + ]); + let e = validate_restorable_item(&binary, &multi_part(), &limits).unwrap_err(); + assert_eq!(e.path, "sk2"); + assert!( + e.to_string().contains("has 5 bytes; the range key limit"), + "{e}" + ); + // The defaults: a 2048-byte hash key and a 1024-byte range key pass. + let largest = item(&[("pk", s(&"p".repeat(2048))), ("sk", n(&"1".repeat(38)))]); + validate_restorable_item(&largest, &composite(), &self::limits(MAX)).unwrap(); + let over = item(&[("pk", s(&"p".repeat(2049))), ("sk", n("1"))]); + assert!(validate_restorable_item(&over, &composite(), &self::limits(MAX)).is_err()); + } + + /// The attribute name rule the wire's `check_attribute_name_size` + /// applies: a name at the limit passes, one byte over is refused, at the + /// top level, inside a map and for a key attribute. + #[test] + fn an_attribute_name_over_its_limit_is_refused_at_the_boundary() { + let limits = RestorableLimits { + max_attribute_name_bytes: 6, + ..limits(MAX) + }; + let ok = item(&[("pk", s("a")), ("abcdef", n("1"))]); + validate_restorable_item(&ok, &hash_only(), &limits).unwrap(); + let top = item(&[("pk", s("a")), ("abcdefg", n("1"))]); + let e = validate_restorable_item(&top, &hash_only(), &limits).unwrap_err(); + assert_eq!(e.path, "abcdefg"); + assert_eq!( + e.reason, + RestorableItemReason::AttributeNameTooLong { size: 7, max: 6 } + ); + assert_eq!( + e.to_string(), + r#"key {"pk":{"S":"a"}}, attribute abcdefg: attribute name of 7 bytes exceeds the limit of 6 bytes"# + ); + let nested = AttributeValue::M(BTreeMap::from([("abcdefg".to_owned(), n("1"))])); + let it = item(&[("pk", s("a")), ("m", nested)]); + let e = validate_restorable_item(&it, &hash_only(), &limits).unwrap_err(); + assert_eq!(e.path, "m.abcdefg"); + assert_eq!( + e.reason, + RestorableItemReason::AttributeNameTooLong { size: 7, max: 6 } + ); + let key = RestorableKeySchema::resolve( + &[element("abcdefg", KeyType::Hash)], + &[definition("abcdefg", ScalarAttributeType::S)], + ) + .unwrap(); + let it = item(&[("abcdefg", s("a"))]); + let e = validate_restorable_item(&it, &key, &limits).unwrap_err(); + assert_eq!(e.path, "abcdefg"); + assert_eq!( + e.reason, + RestorableItemReason::AttributeNameTooLong { size: 7, max: 6 } + ); + } + + /// The limits are the instance's: `RestorableLimits` carries the four + /// sizes of `LimitsConfig`. + #[test] + fn restorable_limits_are_built_from_the_limits_config() { + let config = LimitsConfig { + max_item_size_bytes: 1, + max_partition_key_size_bytes: 2, + max_sort_key_size_bytes: 3, + max_attribute_name_bytes: 4, + ..LimitsConfig::default() + }; + assert_eq!( + RestorableLimits::from(&config), + RestorableLimits { + max_item_size_bytes: 1, + max_partition_key_size_bytes: 2, + max_sort_key_size_bytes: 3, + max_attribute_name_bytes: 4, + } + ); + } + + /// Keys of ordinary size are shown whole, every attribute present, as + /// `Display` writes them: a UUID hash key with a ten-digit range key, a + /// 58-byte hash key with range `1`, a 2048-byte hash key with a + /// 1024-byte range key (the largest legal key), and a hash key of + /// quotes, whose escaped text is twice its bytes. + #[test] + fn the_shown_key_carries_every_attribute_of_a_legal_key_whole() { + let uuid = "39518456-7a1b-4c2d-9e3f-0a1b2c3d832c"; + let it = item(&[ + ("pk", s(uuid)), + ("sk", n("1761955200")), + ("amount", n("abc")), + ]); + let e = refused(&it, &composite(), MAX); + assert_eq!( + e.key, + format!(r#"{{"pk":{{"S":"{uuid}"}},"sk":{{"N":"1761955200"}}}}"#) + ); + assert_eq!( + e.to_string(), + format!( + r#"key {{"pk":{{"S":"{uuid}"}},"sk":{{"N":"1761955200"}}}}, attribute amount: The parameter cannot be converted to a numeric value: abc"# + ) + ); + let tenant = "tenant-42/orders/2026-10-31/xxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"; + assert_eq!(tenant.len(), 58); + let it = item(&[("pk", s(tenant)), ("sk", n("1")), ("amount", n("abc"))]); + let e = refused(&it, &composite(), MAX); + assert_eq!( + e.key, + format!(r#"{{"pk":{{"S":"{tenant}"}},"sk":{{"N":"1"}}}}"#) + ); + let hash = "h".repeat(2048); + let range = "r".repeat(1024); + let key = RestorableKeySchema::resolve( + &[element("pk", KeyType::Hash), element("sk", KeyType::Range)], + &[ + definition("pk", ScalarAttributeType::S), + definition("sk", ScalarAttributeType::S), + ], + ) + .unwrap(); + let it = item(&[("pk", s(&hash)), ("sk", s(&range)), ("amount", n("abc"))]); + let e = refused(&it, &key, MAX); + assert_eq!( + e.key, + format!(r#"{{"pk":{{"S":"{hash}"}},"sk":{{"S":"{range}"}}}}"#) + ); + let quotes = "\"".repeat(2048); + let it = item(&[("pk", s("es)), ("amount", n("abc"))]); + let e = refused(&it, &hash_only(), MAX); + assert_eq!( + e.key, + format!( + r#"{{"pk":{{"S":{}}}}}"#, + serde_json::to_string("es).unwrap() + ) + ); + // A `B` key is its base64 text, the format's rendering. + let it = item(&[ + ("pk", s("a")), + ("sk", n("1")), + ("sk2", AttributeValue::B(vec![0xff; 2048])), + ("amount", n("abc")), + ]); + let e = refused(&it, &multi_part(), MAX); + assert!( + e.key + .ends_with(&format!(r#""sk2":{{"B":"{}"}}}}"#, "/".repeat(2730) + "8=")), + "{}", + e.key + ); + } + + /// Only an oversized value is cut, and on its own: a 13,000-byte hash + /// key (over `SHOWN_KEY_VALUE_BYTES`, under a raised key limit) is cut + /// to its prefix and length while the range key beside it is shown + /// whole; a 3000-byte key, over the service's limit but under the + /// bound, is whole; the bound is on the escaped text, so a key of 6145 + /// quotes (12,290 escaped bytes) is cut and a key of 6144 is not; a cut + /// value is still a JSON string. A key attribute's name is bounded at + /// `SHOWN_VALUE_BYTES` (one of 4097 bytes is cut), and a key attribute + /// holding a list is its JSON, cut to a JSON string when oversized. + #[test] + fn an_oversized_key_value_is_cut_on_its_own() { + let limits = RestorableLimits { + max_partition_key_size_bytes: 20_000, + ..limits(10) + }; + let long = "k".repeat(13_000); + let it = item(&[("pk", s(&long)), ("sk", n("7"))]); + let e = validate_restorable_item(&it, &composite(), &limits).unwrap_err(); + assert!(matches!( + e.reason, + RestorableItemReason::TooLarge { max: 10, .. } + )); + assert_eq!( + e.key, + format!( + r#"{{"pk":{{"S":"{}... <13000 bytes>"}},"sk":{{"N":"7"}}}}"#, + "k".repeat(SHOWN_VALUE_PREFIX_BYTES) + ) + ); + let over_the_service_limit = "k".repeat(3000); + let it = item(&[("pk", s(&over_the_service_limit)), ("sk", n("7"))]); + let e = validate_restorable_item(&it, &composite(), &limits).unwrap_err(); + assert_eq!( + e.key, + format!(r#"{{"pk":{{"S":"{over_the_service_limit}"}},"sk":{{"N":"7"}}}}"#) + ); + let parsed: serde_json::Value = serde_json::from_str(&e.key).unwrap(); + assert_eq!(parsed["sk"]["N"], "7"); + let plain = "p".repeat(SHOWN_KEY_VALUE_BYTES); + let it = item(&[("pk", s(&plain)), ("sk", n("7"))]); + let e = validate_restorable_item(&it, &composite(), &limits).unwrap_err(); + assert_eq!( + e.key, + format!(r#"{{"pk":{{"S":"{plain}"}},"sk":{{"N":"7"}}}}"#) + ); + let quotes = "\"".repeat(SHOWN_KEY_VALUE_BYTES / 2); + let it = item(&[("pk", s("es)), ("sk", n("7"))]); + let e = validate_restorable_item(&it, &composite(), &limits).unwrap_err(); + assert_eq!( + e.key, + format!( + r#"{{"pk":{{"S":{}}},"sk":{{"N":"7"}}}}"#, + serde_json::to_string("es).unwrap() + ) + ); + let quotes = "\"".repeat(SHOWN_KEY_VALUE_BYTES / 2 + 1); + let it = item(&[("pk", s("es)), ("sk", n("7"))]); + let e = validate_restorable_item(&it, &composite(), &limits).unwrap_err(); + assert!(e.key.contains("... <6145 bytes>"), "{}", e.key); + assert!(e.key.ends_with(r#","sk":{"N":"7"}}"#), "{}", e.key); + let parsed: serde_json::Value = serde_json::from_str(&e.key).unwrap(); + assert_eq!( + parsed["pk"]["S"].as_str().unwrap().len(), + 512 + "... <6145 bytes>".len() + ); + // The whole key stays bounded whatever the key: two 100,000-byte + // values and a 100,000-byte name. + let huge = "x".repeat(100_000); + let key = RestorableKeySchema::resolve( + &[element(&huge, KeyType::Hash), element("sk", KeyType::Range)], + &[ + definition(&huge, ScalarAttributeType::S), + definition("sk", ScalarAttributeType::S), + ], + ) + .unwrap(); + let it = item(&[(&huge, s(&huge)), ("sk", s(&huge))]); + let limits = RestorableLimits { + max_partition_key_size_bytes: 200_000, + max_sort_key_size_bytes: 200_000, + max_attribute_name_bytes: 200_000, + ..self::limits(10) + }; + let e = validate_restorable_item(&it, &key, &limits).unwrap_err(); + assert!( + e.key.len() < 3 * (SHOWN_VALUE_PREFIX_BYTES + 40), + "{}", + e.key.len() + ); + assert_eq!(e.key.matches("... <100000 bytes>").count(), 3); + // A key attribute's name is bounded at `SHOWN_VALUE_BYTES`, the + // bound a name gets, not the key value's: one of 4097 bytes is cut. + let long_name = "k".repeat(SHOWN_VALUE_BYTES + 1); + let key = RestorableKeySchema::resolve( + &[element(&long_name, KeyType::Hash)], + &[definition(&long_name, ScalarAttributeType::S)], + ) + .unwrap(); + let it = item(&[(long_name.as_str(), s("a")), ("amount", n("abc"))]); + let limits = RestorableLimits { + max_attribute_name_bytes: 10_000, + ..self::limits(MAX) + }; + let e = validate_restorable_item(&it, &key, &limits).unwrap_err(); + assert_eq!(e.path, "amount"); + assert_eq!( + e.key, + format!( + r#"{{"{}... <4097 bytes>":{{"S":"a"}}}}"#, + "k".repeat(SHOWN_VALUE_PREFIX_BYTES) + ) + ); + // A key attribute holding a value that is not a scalar (a list of + // 20,000 bytes under `pk`) renders as its JSON cut to a JSON string: + // the key still parses and stays under 1,200 bytes. + let list = AttributeValue::L(vec![s(&"x".repeat(20_000))]); + let list_json_len = serde_json::to_string(&list).unwrap().len(); + assert!(list_json_len > SHOWN_KEY_VALUE_BYTES); + let it = item(&[("pk", list), ("sk", n("7"))]); + let e = validate_restorable_item(&it, &composite(), &self::limits(MAX)).unwrap_err(); + assert!(e.key.len() < 1200, "{}", e.key.len()); + let parsed: serde_json::Value = serde_json::from_str(&e.key).unwrap(); + assert!( + parsed["pk"] + .as_str() + .unwrap() + .ends_with(&format!("... <{list_json_len} bytes>")), + "{}", + e.key + ); + assert_eq!(parsed["sk"]["N"], "7"); + } + + /// The attribute path is bounded segment by segment with the same rule: + /// a 60,000-byte attribute name holding a map with a 60,000-byte member + /// gives a path of two cut segments, each with its length; a name of + /// `SHOWN_VALUE_BYTES` bytes is whole and one byte more is cut. + #[test] + fn the_attribute_path_is_bounded_segment_by_segment() { + let limits = RestorableLimits { + max_attribute_name_bytes: 100_000, + ..limits(MAX) + }; + let long_name = "a".repeat(60_000); + let long_member = "b".repeat(60_000); + let inner = AttributeValue::M(BTreeMap::from([(long_member, n("abc"))])); + let it = item(&[("pk", s("a")), (&long_name, inner)]); + let e = validate_restorable_item(&it, &hash_only(), &limits).unwrap_err(); + assert_eq!( + e.path, + format!( + "{}... <60000 bytes>.{}... <60000 bytes>", + "a".repeat(SHOWN_VALUE_PREFIX_BYTES), + "b".repeat(SHOWN_VALUE_PREFIX_BYTES) + ) + ); + assert!(e.to_string().len() < 3000, "{}", e.to_string().len()); + let whole = "w".repeat(SHOWN_VALUE_BYTES); + let it = item(&[("pk", s("a")), (&whole, n("abc"))]); + assert_eq!( + validate_restorable_item(&it, &hash_only(), &limits) + .unwrap_err() + .path, + whole + ); + let over = "w".repeat(SHOWN_VALUE_BYTES + 1); + let it = item(&[("pk", s("a")), (&over, n("abc"))]); + assert_eq!( + validate_restorable_item(&it, &hash_only(), &limits) + .unwrap_err() + .path, + format!("{}... <4097 bytes>", "w".repeat(SHOWN_VALUE_PREFIX_BYTES)) + ); + // A set member over the bound is cut in the reason, as a plain `N` + // over it is: the wire's echo of the input is the one text that can + // grow with the item. + let member = format!("1E{}", "9".repeat(SHOWN_VALUE_BYTES)); + let set = AttributeValue::NS(BTreeSet::from([member.clone()])); + let it = item(&[("pk", s("a")), ("ns", set)]); + let e = validate_restorable_item(&it, &hash_only(), &limits).unwrap_err(); + assert!(e.to_string().len() < 2000, "{}", e.to_string().len()); + assert!( + e.to_string() + .ends_with(&format!("<{} bytes>", member.len())), + "{e}" + ); + let it = item(&[("pk", s("a")), ("n", n(&member))]); + let e = validate_restorable_item(&it, &hash_only(), &limits).unwrap_err(); + assert_eq!( + e.reason, + RestorableItemReason::InvalidNumber(format!( + "The parameter cannot be converted to a numeric value: {}... <{} bytes>", + &member[..SHOWN_VALUE_PREFIX_BYTES], + member.len() + )) + ); + // In a key the value is echoed twice: whole in the key (under the + // key bound) and cut in the reason. + let it = item(&[("pk", s("a")), ("sk", n(&member))]); + let e = validate_restorable_item(&it, &composite(), &limits).unwrap_err(); + assert!( + e.to_string().len() < member.len() + SHOWN_VALUE_PREFIX_BYTES + 400, + "{}", + e.to_string().len() + ); + assert!(e.key.contains(&member), "{}", e.key); + assert_eq!(e.to_string().matches("... <4098 bytes>").count(), 1, "{e}"); + } + + /// `LineTooLong`, the reason the backup job's feeder builds for an item + /// whose data file line would exceed the cap, names the cap: the + /// measurement stops at the cap, so the line's own length is unknown. + #[test] + fn a_line_too_long_reason_names_the_cap() { + let e = RestorableItemError { + key: r#"{"pk":{"S":"a"}}"#.to_owned(), + path: String::new(), + reason: RestorableItemReason::LineTooLong { cap: 33_554_432 }, + }; + assert_eq!( + e.to_string(), + r#"key {"pk":{"S":"a"}}: data file line exceeds the line cap of 33554432 bytes"# + ); + } + + /// A control character, or one of the line separators U+0085, U+2028 and + /// U+2029, in an attribute name or in the number the wire's reason + /// echoes is written escaped, as a JSON string's content (`\n`, `\t`, + /// `\"`, `\\`, `\u00XX`, `\u2028`), so a tenant's item cannot add a + /// line of its own to the log: the message holds no raw newline and no + /// separator, and the key beside it is the same escaped rendering. + #[test] + fn a_control_character_in_a_name_or_a_number_is_escaped_in_the_message() { + let it = item(&[ + ("pk", s("a\nb")), + ("amount", n("abc\nFORGED LINE: looks like another record")), + ]); + let e = refused(&it, &hash_only(), MAX); + let message = e.to_string(); + assert!(!message.contains('\n'), "{message:?}"); + assert_eq!( + message, + r#"key {"pk":{"S":"a\nb"}}, attribute amount: The parameter cannot be converted to a numeric value: abc\nFORGED LINE: looks like another record"# + ); + let name = "att\nFORGED LINE TWO\t\"quoted\" \\ \u{1}"; + let it = item(&[("pk", s("a")), (name, n("abc"))]); + let e = refused(&it, &hash_only(), MAX); + assert_eq!(e.path, r#"att\nFORGED LINE TWO\t\"quoted\" \\ \u0001"#); + assert!(!e.to_string().contains('\n'), "{e}"); + // In a map, in a set member and in a key attribute's name the same. + let set = AttributeValue::NS(BTreeSet::from(["1".to_owned(), "x\ny".to_owned()])); + let it = item(&[("pk", s("a")), ("ns", set)]); + let e = refused(&it, &hash_only(), MAX); + assert_eq!(e.path, "ns[1]"); + assert_eq!( + e.reason, + RestorableItemReason::InvalidNumber( + r"The parameter cannot be converted to a numeric value: x\ny".to_owned() + ) + ); + let inner = AttributeValue::M(BTreeMap::from([("c\rd".to_owned(), n("abc"))])); + let it = item(&[("pk", s("a")), ("m\u{c}", inner)]); + let e = refused(&it, &hash_only(), MAX); + assert_eq!(e.path, r"m\f.c\rd"); + let key = RestorableKeySchema::resolve( + &[element("p\nk", KeyType::Hash)], + &[definition("p\nk", ScalarAttributeType::S)], + ) + .unwrap(); + let it = item(&[("p\nk", s(""))]); + let e = validate_restorable_item(&it, &key, &limits(MAX)).unwrap_err(); + assert_eq!(e.path, r"p\nk"); + assert_eq!(e.key, r#"{"p\nk":{"S":""}}"#); + // The three Unicode line separators, which `serde_json` leaves raw + // and a reader that breaks lines the Unicode way breaks on, in a + // key value, a name and a number: each is written as its `\uXXXX` + // escape and none reaches the message. + let separators = "\u{85}\u{2028}\u{2029}"; + let name = format!("att{separators}"); + let it = item(&[ + ("pk", s(&format!("a{separators}b"))), + (name.as_str(), n(&format!("abc{separators}FORGED LINE"))), + ]); + let e = refused(&it, &hash_only(), MAX); + let message = e.to_string(); + assert!( + !message + .chars() + .any(|c| matches!(c, '\u{85}' | '\u{2028}' | '\u{2029}')), + "{message:?}" + ); + assert_eq!(e.key, r#"{"pk":{"S":"a\u0085\u2028\u2029b"}}"#); + assert_eq!(e.path, r"att\u0085\u2028\u2029"); + assert_eq!( + e.reason, + RestorableItemReason::InvalidNumber( + r"The parameter cannot be converted to a numeric value: abc\u0085\u2028\u2029FORGED LINE" + .to_owned() + ) + ); + } + + /// A key value is bounded at `SHOWN_KEY_VALUE_BYTES`, six times the hash + /// key limit: a 700-byte hash key of U+0001 (4,200 escaped bytes, over + /// the bound a name gets) is shown whole, and a 2049-byte one (12,294 + /// escaped bytes, over the legal key size) is cut. + #[test] + fn a_legal_key_of_control_characters_is_shown_whole() { + let limits = RestorableLimits { + max_partition_key_size_bytes: 4096, + ..limits(MAX) + }; + let legal = "\u{1}".repeat(700); + let it = item(&[("pk", s(&legal)), ("amount", n("abc"))]); + let e = validate_restorable_item(&it, &hash_only(), &limits).unwrap_err(); + assert_eq!(e.path, "amount"); + assert_eq!( + e.key, + format!(r#"{{"pk":{{"S":"{}"}}}}"#, r"\u0001".repeat(700)) + ); + let largest = "\u{1}".repeat(2048); + let it = item(&[("pk", s(&largest)), ("amount", n("abc"))]); + let e = validate_restorable_item(&it, &hash_only(), &limits).unwrap_err(); + assert_eq!( + e.key, + format!(r#"{{"pk":{{"S":"{}"}}}}"#, r"\u0001".repeat(2048)) + ); + let over = "\u{1}".repeat(2049); + let it = item(&[("pk", s(&over)), ("amount", n("abc"))]); + let e = validate_restorable_item(&it, &hash_only(), &limits).unwrap_err(); + assert_eq!( + e.key, + format!( + r#"{{"pk":{{"S":"{}... <2049 bytes>"}}}}"#, + r"\u0001".repeat(SHOWN_VALUE_PREFIX_BYTES / 6) + ) + ); + let parsed: serde_json::Value = serde_json::from_str(&e.key).unwrap(); + assert_eq!(parsed["pk"]["S"].as_str().unwrap().len(), 170 + 16); + } +} diff --git a/crates/engine/Cargo.toml b/crates/engine/Cargo.toml index d8e20bc15..6800acfdd 100755 --- a/crates/engine/Cargo.toml +++ b/crates/engine/Cargo.toml @@ -7,6 +7,11 @@ edition.workspace = true rust-version.workspace = true license.workspace = true +[features] +# Kill-point pauses in the backup job (`EXTENDDB_BACKUP_PAUSE`), for the +# binary-level crash tests; never enabled in a release build. +backup-test-hooks = [] + [dependencies] extenddb-auth = { workspace = true } extenddb-core = { workspace = true } @@ -22,6 +27,9 @@ hmac = "0.12" sha2 = "0.10" hex = "0.4" tokio = { workspace = true } +# `TaskTracker` for the backup jobs the server drains at shutdown. +tokio-util = { workspace = true, features = ["rt"] } +bytes = { workspace = true } futures = { workspace = true } [dev-dependencies] diff --git a/crates/engine/src/backup.rs b/crates/engine/src/backup.rs index 432fcb8b6..dfc636b32 100755 --- a/crates/engine/src/backup.rs +++ b/crates/engine/src/backup.rs @@ -29,16 +29,17 @@ use extenddb_storage::backup_catalog::{ }; use extenddb_storage::error::StorageError; use extenddb_storage::{ - BackupCatalog, BackupRestoreSource, LegacyBackupReader, RestoreRequest, RestoreTableOverrides, - TableEngine, + BackupCatalog, BackupRestoreSource, LegacyBackupReader, MetadataEngine, RestoreRequest, + RestoreTableOverrides, SnapshotEngine, TableEngine, }; use futures::TryStreamExt; use serde_json::{Value, json}; +pub use job::KillPoint; pub use maintenance::HeldClaim; pub use runtime::{ - BackupRuntime, ConfiguredStore, LOCAL_CHECK_TIMEOUT, S3_CHECK_TIMEOUT, Spool, StoreHealth, - StoreHealthReport, check_configured_store, read_identity_bounded, + BackupRuntime, ConfiguredStore, JobSettings, LOCAL_CHECK_TIMEOUT, S3_CHECK_TIMEOUT, Spool, + StoreHealth, StoreHealthReport, check_configured_store, read_identity_bounded, }; pub use store_identity::{STORE_IDENTITY_KEY, StoreIdentityFile, StoreProblem}; @@ -400,10 +401,11 @@ pub(crate) async fn handle_create_backup( ) .await; let details = create_backup( - &*ctx.storage, + &ctx.storage, &ctx.backup, &ctx.account_id, &ctx.region, + &ctx.limits, table_name, backup_name, probe, @@ -419,23 +421,45 @@ pub(crate) async fn handle_create_backup( /// fit a backup ARN and its store prefix (`check_backup_arn_fits`: a /// `server.region` outside `[a-z0-9-]` is a `ValidationException` naming the /// setting, a name over 255 bytes or made only of dots one naming the rule); -/// the table has no vector index -/// (the in-catalog format cannot restore one); then the job permit -/// (`LimitExceededException` when none is free). `probe` is the -/// `DescribeTable` of the table, which the handler ran. +/// then the path. /// -/// This is the in-catalog create, the path while no `[backup.store]` is -/// configured; each call logs that the format is deprecated. Last before the -/// permit, the maintenance claim ([`refuse_while_maintenance_held`]). -async fn create_backup( - storage: &S, +/// With a store configured and the backend not bound to the in-catalog +/// create, the create is store-backed: the store must be healthy +/// ([`BackupRuntime::require_healthy_store`], refused before a slot is +/// taken as every operation that touches the store is), then the permit +/// (`LimitExceededException` when none is free), then the detached task of +/// [`job::start`] (the snapshot's open, the row under the job's claim, the +/// maintenance check once the row exists, the response CREATING) whose +/// answer the request awaits. A vector table is backed up on this path (the +/// manifest records its indexes). +/// +/// Otherwise the in-catalog create: the table has no vector index (the +/// in-catalog format cannot restore one), the maintenance claim is not held +/// ([`refuse_while_maintenance_held`], last before the permit), the permit, +/// then `LegacyBackupReader::create`; each call logs that the format is +/// deprecated. `probe` is the `DescribeTable` of the table, which the +/// handler ran. +#[allow(clippy::too_many_arguments)] +async fn create_backup( + storage: &Arc, runtime: &BackupRuntime, account_id: &str, region: &str, + limits: &LimitsConfig, table_name: &str, backup_name: &str, probe: Result, -) -> Result { +) -> Result +where + S: BackupCatalog + + LegacyBackupReader + + SnapshotEngine + + MetadataEngine + + ?Sized + + Send + + Sync + + 'static, +{ let desc = match probe { Ok(desc) if desc.table_status == TableStatus::Active => desc, Ok(_) => { @@ -447,20 +471,32 @@ async fn create_backup( Err(e) => return Err(create_backup_storage_err_to_dynamo(e, table_name)), }; check_backup_arn_fits(region, account_id, table_name)?; - refuse_backup_of_vector_table(&desc)?; - // A create that writes to the configured store needs it healthy, and is - // refused before a slot is taken when it is not, as every operation that - // touches the store is; the in-catalog create touches no store, so the - // store's health does not gate it (a backend bound to the in-catalog - // create, or a server with an unhealthy store, creates as a server - // without a store does). - let store_backed = runtime.configured_store().is_some() - && !runtime.legacy_create_only() - && STORE_BACKED_CREATE; + let store_backed = runtime.configured_store().is_some() && !runtime.legacy_create_only(); if store_backed { - runtime.require_healthy_store()?; + let store = runtime.require_healthy_store()?; + let env = job::JobEnv::new(runtime, store)?; + let permit = runtime.take_permit()?; + let (reply, answer) = tokio::sync::oneshot::channel(); + let request = job::CreateRequest { + account_id: account_id.to_owned(), + region: region.to_owned(), + table_name: table_name.to_owned(), + backup_name: backup_name.to_owned(), + limits: limits.into(), + }; + runtime.spawn_job(job::start(Arc::clone(storage), env, permit, request, reply)); + return answer.await.unwrap_or_else(|_| { + tracing::error!( + "CreateBackup of {table_name} (account {account_id}): the job task ended \ + without answering" + ); + Err(DynamoDbError::InternalServerError( + "Internal server error".to_owned(), + )) + }); } - refuse_while_maintenance_held(storage, runtime).await?; + refuse_backup_of_vector_table(&desc)?; + refuse_while_maintenance_held(&**storage, runtime).await?; let permit = runtime.take_permit()?; match runtime.configured_store() { None => tracing::warn!( @@ -470,8 +506,8 @@ async fn create_backup( ), Some(store) => tracing::info!( "CreateBackup of {table_name} (account {account_id}) is writing the in-catalog \ - backup format; the configured {} backup store serves restores and deletes of the \ - backups it holds, and a later release's CreateBackup writes to it", + backup format: this backend is bound to it; the configured {} backup store serves \ + restores and deletes of the backups it holds", store.kind ), } @@ -481,20 +517,20 @@ async fn create_backup( .map_err(|e| create_backup_storage_err_to_dynamo(e, table_name)) } -/// The jobs' side of the catalog-level maintenance claim: a backup or -/// restore job is refused with `LimitExceededException` while a maintenance -/// command (`extenddb backup purge`) holds the claim, read with -/// `stale_job_after_secs` so a dead command's claim expires. The check runs -/// after every other refusal and before the permit, which on this head is -/// before the job's row exists: the in-catalog create and restore commit -/// their row inside the backend and detach, so the engine has no point -/// between the row and the copy to check at. A claim taken between this -/// check and the row's insert is therefore not seen by the job; the command -/// create that slips through writes no object of the store the command -/// deletes from, and a restore that slips through is a RUNNING record the -/// command's wait lists, so the window costs nothing in this release. The -/// store-backed job of a later release owns its row from the -/// engine and checks after the row exists, as the design's 5.1 step 2 says. +/// The in-catalog jobs' side of the catalog-level maintenance claim: a +/// backup or restore job is refused with `LimitExceededException` +/// ([`maintenance_held`]) while a maintenance command (`extenddb backup +/// purge`) holds the claim, read with `stale_job_after_secs` so a dead +/// command's claim expires. The check runs after every other refusal and +/// before the permit, which is before the job's row exists: the in-catalog +/// create and restore commit their row inside the backend and detach, so the +/// engine has no point between the row and the copy to check at. A claim +/// taken between this check and the row's insert is therefore not seen by +/// the job; the command create that slips through writes no object of the +/// store the command deletes from, and a restore that slips through is a +/// RUNNING record the command's wait lists, so the window costs nothing. +/// The store-backed job owns its row from the engine and checks after the +/// row exists ([`job`]), as the design's 5.1 step 2 says. async fn refuse_while_maintenance_held( storage: &S, runtime: &BackupRuntime, @@ -504,21 +540,19 @@ async fn refuse_while_maintenance_held( .await .map_err(storage_err_to_dynamo)?; if held { - return Err(DynamoDbError::LimitExceededException( - "A maintenance command (extenddb backup purge) holds this catalog's maintenance \ - claim; backup and restore jobs are refused until it finishes" - .to_owned(), - )); + return Err(maintenance_held()); } Ok(()) } -/// Whether CreateBackup writes to the configured store. No store-backed -/// create exists on this head: every create writes the in-catalog format, -/// whatever the store configuration, and the release that adds the -/// store-backed job turns this on for a server with a store configured and -/// not bound to the in-catalog create. -const STORE_BACKED_CREATE: bool = false; +/// The refusal every job answers while the maintenance claim is held. +fn maintenance_held() -> DynamoDbError { + DynamoDbError::LimitExceededException( + "A maintenance command (extenddb backup purge) holds this catalog's maintenance \ + claim; backup and restore jobs are refused until it finishes" + .to_owned(), + ) +} /// The backup ARN rules a table name has to fit before a backup of it can be /// created, each refusal naming its cause: first the server's own region @@ -577,7 +611,8 @@ fn create_backup_storage_err_to_dynamo(e: StorageError, table_name: &str) -> Dyn } } -/// The CreateBackup refusal for a table with vector indexes. +/// The in-catalog create's refusal for a table with vector indexes (the +/// in-catalog format cannot restore one); the store-backed path has none. fn refuse_backup_of_vector_table(desc: &TableDescription) -> Result<(), DynamoDbError> { let count = desc.vector_indexes.as_ref().map_or(0, Vec::len); if count == 0 { @@ -1817,6 +1852,7 @@ fn storage_err_to_dynamo(e: StorageError) -> DynamoDbError { } } +mod job; pub mod maintenance; mod overrides; pub mod reconciler; diff --git a/crates/engine/src/backup/job.rs b/crates/engine/src/backup/job.rs new file mode 100644 index 000000000..df6120eaa --- /dev/null +++ b/crates/engine/src/backup/job.rs @@ -0,0 +1,641 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! The store-backed CreateBackup job: the detached task the request spawns +//! once its checks have passed and the permit is held. +//! +//! The task ([`start`]) opens the table snapshot, mints the backup id from +//! the millisecond the open returned, reads the table's TTL, builds the +//! manifest's `source_table` from the snapshot's description +//! ([`source_table_schema`]), inserts the CREATING row under the job's +//! claim (`claim_backup`), checks the maintenance claim, answers the request +//! through a one-shot channel (`BackupDetails` CREATING, or the refusal) and +//! then goes on as the job ([`run`]) with the same owned values: nothing +//! crosses a thread or a channel between the request and the job but the +//! response (the pipeline's item and chunk channels are the job's own), and +//! a client that disconnects mid-request cannot leave a row whose owner was +//! never spawned. A maintenance claim held when the row exists fails the +//! row with reason `maintenance` and answers `LimitExceededException`; one +//! that cannot be read fails the row with the read's error as the reason +//! and answers that error. +//! +//! The job: the data files through the pipeline ([`writer`]), then the +//! three documents `BackupDocuments::build` returns in its order, a re-check +//! of the claim, the manifest (the commit record), and the AVAILABLE write +//! with the size, the count and the manifest's `source_table` +//! ([`failure::write_status`] gives every status write one meaning; an +//! AVAILABLE write whose outcome is unknown is re-read, one that matches no +//! row ends the job with no store access). The heartbeat runs on a timer for +//! the job's life ([`heartbeat`]); a heartbeat that does not confirm the +//! claim, or does not return within one period, ends the job with no +//! further store or catalog access. Any other failure, and the shutdown +//! token (reason `shutdown`), runs the failure path ([`failure::fail_row`]): +//! FAILED, then the prefix's deletion only when that write matched. +//! +//! Each job records `BackupJobs` and `BackupJobDuration` (microseconds, +//! from the snapshot's open to the end of its last catalog or store call: +//! the AVAILABLE write, or the FAILED write and the prefix's deletion) +//! under its outcome, `BackupBytesUploaded` once it ended AVAILABLE, +//! `BackupSnapshotAge` at each heartbeat, and the worker metrics under +//! `QuerySource::BackupJob` (`WorkerLastSuccess` on success, +//! `WorkerErrorCount` on a failure; a job the shutdown cancelled is neither); +//! one info line at start (ARN, the table's item count as the stats pass +//! last counted it, the order) and at the end (ARN, the items written, +//! bytes, duration), one error line per failure. +//! +//! Kill points ([`kill_point`]) pause the job for the binary-level crash +//! tests under the `backup-test-hooks` feature and are no-ops without it. + +pub(crate) mod failure; +pub(crate) mod heartbeat; +pub(crate) mod writer; + +use std::sync::Arc; + +use extenddb_backup_store::{BackupStore, byte_stream_from}; +use extenddb_core::backup_format::{ + BackupArn, BackupDocuments, ItemOrder, KeyOrder, KeyRangeTracker, ManifestHeader, + SourceTableSchema, backup_creation_date_time, +}; +use extenddb_core::error::DynamoDbError; +use extenddb_core::metrics::{BackupJobOutcome, MetricsCollector, QuerySource}; +use extenddb_core::types::BackupDetails; +use extenddb_core::validation::{RestorableKeySchema, RestorableLimits}; +use extenddb_storage::backup_catalog::{ + BACKUP_TYPE_USER, BackupRow, BackupStatus, JobStatus, StatusFields, StoreIdentity, + UNOWNED_GENERATION, +}; +use extenddb_storage::error::StorageError; +use extenddb_storage::snapshot::source_table_schema; +use extenddb_storage::{ + BackupCatalog, CancellationToken, JobClaim, JobPermit, MetadataEngine, SnapshotEngine, + TableSnapshot, +}; +use tokio::sync::oneshot; +use tokio::time::Instant; + +use super::runtime::{BackupRuntime, ConfiguredStore, JobSettings}; +use super::{create_backup_storage_err_to_dynamo, storage_err_to_dynamo}; +use failure::{FailurePath, Written}; +use heartbeat::HeartbeatHandle; +use writer::{DataFiles, Pipeline, Stop}; + +/// The reason a row carries when the maintenance claim was held as the +/// row was inserted. +pub(crate) const MAINTENANCE: &str = "maintenance"; +/// The reason a row carries when the server's shutdown stopped the job. +pub(crate) const SHUTDOWN: &str = "shutdown"; + +/// What the runtime hands a job: the store, its identity and manifest +/// names, the settings, the collector, the shutdown token and the source +/// names. +#[derive(Clone)] +pub(crate) struct JobEnv { + pub store: Arc, + pub identity: StoreIdentity, + pub bucket: String, + pub key_prefix: String, + pub settings: JobSettings, + pub metrics: Option>, + pub shutdown: CancellationToken, + pub source_backend: String, + pub source_extenddb_version: String, +} + +impl JobEnv { + /// From the runtime and its configured store, whose identity must be + /// known (it is once a check has passed, which `require_healthy_store` + /// has established). + pub(crate) fn new( + runtime: &BackupRuntime, + store: &ConfiguredStore, + ) -> Result { + let identity = store.identity().ok_or_else(|| { + tracing::error!("the configured store is healthy but its id is unknown"); + DynamoDbError::InternalServerError("Internal server error".to_owned()) + })?; + let (source_backend, source_extenddb_version) = runtime.source(); + Ok(Self { + store: Arc::clone(&store.store), + identity, + bucket: store.bucket.clone(), + key_prefix: store.key_prefix.clone(), + settings: runtime.job_settings(), + metrics: runtime.metrics().cloned(), + shutdown: runtime.shutdown().clone(), + source_backend: source_backend.to_owned(), + source_extenddb_version: source_extenddb_version.to_owned(), + }) + } +} + +/// The request's members the task needs. +pub(crate) struct CreateRequest { + pub account_id: String, + pub region: String, + pub table_name: String, + pub backup_name: String, + /// The instance's limits the validator holds every item to. + pub limits: RestorableLimits, +} + +/// The one-shot the task answers the request through. +pub(crate) type Reply = oneshot::Sender>; + +/// The detached task (module doc): everything from the open to the +/// response, then the job. +pub(crate) async fn start( + storage: Arc, + env: JobEnv, + permit: JobPermit, + request: CreateRequest, + reply: Reply, +) where + S: BackupCatalog + SnapshotEngine + MetadataEngine + ?Sized + Send + Sync + 'static, +{ + let job = match open_and_claim(&*storage, &env, &request).await { + Ok(Started::Job(job)) => { + let details = job.details(); + // A request whose client went away still has its job. + let _ = reply.send(Ok(details)); + *job + } + Ok(Started::Refused(e)) | Err(e) => { + drop(permit); + let _ = reply.send(Err(e)); + return; + } + }; + run(job).await; + drop(permit); +} + +/// The job once its row exists. +pub(crate) struct Job { + claim: Arc, + snapshot: TableSnapshot, + arn: BackupArn, + backup_name: String, + source_table: SourceTableSchema, + key_schema: RestorableKeySchema, + /// The table's key order when the stream is key-ordered. + key_order: Option, + limits: RestorableLimits, + env: JobEnv, + /// The id's instant, the row's `created_at`. + created_at: f64, + started: Instant, + /// The instant taken before the claim's insert, the heartbeat's first + /// confirmed beat. + claimed_at: Instant, +} + +impl Job { + /// The response the request answers: the row as inserted, CREATING. + fn details(&self) -> BackupDetails { + BackupDetails { + backup_arn: self.arn.to_arn_string(), + backup_name: self.backup_name.clone(), + backup_status: BackupStatus::Creating.as_str().to_owned(), + backup_type: BACKUP_TYPE_USER.to_owned(), + backup_size_bytes: 0, + backup_creation_date_time: self.created_at, + } + } +} + +enum Started { + Job(Box), + /// The row exists FAILED (reason `maintenance`) and the request is + /// refused. + Refused(DynamoDbError), +} + +/// The open, the id, the TTL read, the row, the claim and the maintenance +/// check (module doc): a claim held, or one that cannot be read, fails the +/// row before the job starts and refuses the request. +async fn open_and_claim( + storage: &S, + env: &JobEnv, + request: &CreateRequest, +) -> Result +where + S: BackupCatalog + SnapshotEngine + MetadataEngine + ?Sized, +{ + let CreateRequest { + account_id, + region, + table_name, + backup_name, + limits, + } = request; + let started = Instant::now(); + let snapshot = storage + .open_table_snapshot(account_id, table_name) + .await + .map_err(|e| create_backup_storage_err_to_dynamo(e, table_name))?; + // The id carries the millisecond the open returned. + let backup_id = extenddb_storage::util::new_backup_id(); + let created_at = backup_creation_date_time(&backup_id) + .ok_or_else(|| internal("the backup id carries no instant"))?; + let arn = BackupArn::new( + extenddb_storage::util::ARN_PARTITION, + region, + account_id, + table_name, + &backup_id, + ) + .map_err(|e| internal(&format!("no backup ARN for {table_name}: {e}")))?; + let ttl = storage + .describe_ttl(account_id, table_name) + .await + .map_err(storage_err_to_dynamo)?; + let source_table = + source_table_schema(&snapshot.description, Some(ttl)).map_err(storage_err_to_dynamo)?; + let key_schema = RestorableKeySchema::resolve( + &source_table.key_schema, + &source_table.attribute_definitions, + ) + .map_err(|e| { + internal(&format!( + "the key schema of {table_name} cannot be checked: {e}" + )) + })?; + let key_order = snapshot + .key_ordered + .then(|| { + KeyOrder::resolve( + &source_table.key_schema, + &source_table.attribute_definitions, + ) + .ok() + }) + .flatten(); + let row = BackupRow { + backup_id: backup_id.clone(), + account_id: account_id.clone(), + table_name: table_name.clone(), + backup_arn: arn.to_arn_string(), + backup_name: backup_name.clone(), + status: BackupStatus::Creating, + store: env.identity.clone(), + size_bytes: 0, + item_count: 0, + created_at, + completed_at: None, + failure_reason: None, + source_table_details_json: serde_json::to_value(&source_table) + .map_err(|e| internal(&format!("the source table does not serialize: {e}")))?, + owner_generation: UNOWNED_GENERATION, + heartbeat_at: 0.0, + }; + let claimed_at = Instant::now(); + let claim: Arc = match storage.claim_backup(&row).await { + Ok(claim) => Arc::from(claim), + Err(StorageError::AlreadyExists(what)) => { + return Err(internal(&format!( + "backup id {backup_id} is already claimed: {what}" + ))); + } + Err(e) => return Err(storage_err_to_dynamo(e)), + }; + let job = Job { + claim, + snapshot, + arn, + backup_name: backup_name.clone(), + source_table, + key_schema, + key_order, + limits: *limits, + env: env.clone(), + created_at, + started, + claimed_at, + }; + // The maintenance claim, once the row exists so the command's wait sees + // it: held, the row is FAILED `maintenance` and the request refused. + match storage + .check_maintenance_claim(env.settings.stale_job_after) + .await + { + Ok(false) => Ok(Started::Job(Box::new(job))), + Ok(true) => { + fail_before_start(&*job.claim, &job.arn, MAINTENANCE).await; + Ok(Started::Refused(super::maintenance_held())) + } + Err(e) => { + fail_before_start( + &*job.claim, + &job.arn, + &format!("the maintenance claim could not be read: {e}"), + ) + .await; + Err(storage_err_to_dynamo(e)) + } + } +} + +fn internal(why: &str) -> DynamoDbError { + tracing::error!("CreateBackup: {why}"); + DynamoDbError::InternalServerError("Internal server error".to_owned()) +} + +/// The FAILED write of a row whose job never started (the maintenance +/// check); nothing is in the store, so there is nothing to delete. +async fn fail_before_start(claim: &dyn JobClaim, arn: &BackupArn, reason: &str) { + let fields = StatusFields { + failure_reason: Some(reason.to_owned()), + completed: true, + ..StatusFields::default() + }; + match failure::write_status(claim, JobStatus::Failed, &fields).await { + Written::Matched => {} + Written::NotMatched(why) | Written::Error(why) => tracing::error!( + "backup {}: the row could not be marked FAILED ({reason}): {why}", + arn.to_arn_string() + ), + } +} + +/// The job (module doc), from the data files to the final status write. +pub(crate) async fn run(job: Job) -> BackupJobOutcome { + let Job { + claim, + snapshot, + arn, + backup_name, + mut source_table, + key_schema, + key_order, + limits, + env, + created_at, + started, + claimed_at, + } = job; + let arn_text = arn.to_arn_string(); + let prefix = arn.store_prefix(); + let metrics = env.metrics.as_deref(); + let heartbeat = HeartbeatHandle::start( + Arc::clone(&claim), + env.settings.heartbeat_interval, + env.metrics.clone(), + created_at, + claimed_at, + ); + tracing::info!( + "backup {arn_text}: started, {} items in the table as last counted by the stats pass, {}", + snapshot.description.item_count, + if key_order.is_some() { + "key-ordered" + } else { + "unordered" + } + ); + let TableSnapshot { info, items, .. } = snapshot; + let pipeline = Pipeline { + store: &*env.store, + prefix: &prefix, + arn: &arn_text, + heartbeat: &heartbeat, + shutdown: &env.shutdown, + metrics, + target_bytes: env.settings.data_file_target_bytes, + heartbeat_period: env.settings.heartbeat_interval, + key_prefix: &env.key_prefix, + bucket: &env.bucket, + }; + let tracker = key_order.as_ref().map(KeyRangeTracker::new); + let mut item_count = 0; + let end = match writer::write_data_files(&pipeline, items, key_schema, limits, tracker).await { + Ok(files) => { + item_count = files.item_count; + source_table.item_count = i64::try_from(files.item_count).unwrap_or(i64::MAX); + let header = ManifestHeader { + backup_arn: arn.clone(), + backup_name, + source_backend: env.source_backend.clone(), + source_extenddb_version: env.source_extenddb_version.clone(), + snapshot: info, + item_order: if key_order.is_some() { + ItemOrder::KeyOrdered + } else { + ItemOrder::Unordered + }, + source_table, + }; + write_documents(&pipeline, &claim, header, files).await + } + Err(stop) => Err(stop), + }; + let (outcome, bytes) = match end { + Ok(bytes) => { + tracing::info!( + "backup {arn_text}: AVAILABLE, {item_count} items, {bytes} bytes written in {:.1} s", + started.elapsed().as_secs_f64() + ); + (BackupJobOutcome::Succeeded, bytes) + } + Err(Stop::ClaimLost(why)) => { + tracing::error!( + "backup {arn_text}: stopped with no further store or catalog access: {why}; the \ + row is left to its owner or to the reconciler" + ); + (BackupJobOutcome::Failed, 0) + } + Err(stop) => { + let (reason, outcome) = match stop { + Stop::Cancelled => (SHUTDOWN.to_owned(), BackupJobOutcome::Cancelled), + Stop::Fail(reason) => (reason, BackupJobOutcome::Failed), + Stop::ClaimLost(_) => unreachable!("matched above"), + }; + match failure::fail_row(&*claim, &*env.store, &prefix, &reason, metrics).await { + FailurePath::Deleted => tracing::error!( + "backup {arn_text}: FAILED ({reason}); its objects were deleted" + ), + FailurePath::DeleteFailed(e) => tracing::error!( + "backup {arn_text}: FAILED ({reason}); its objects could not be deleted and \ + the reconciler deletes them once the row's heartbeat is stale: {e}" + ), + FailurePath::NoStoreAccess(why) => tracing::error!( + "backup {arn_text}: failed ({reason}) and the row is not this job's to \ + write ({why}); no store access was made" + ), + } + (outcome, 0) + } + }; + // After the failure path too, so a FAILED job's duration covers its + // FAILED write and the prefix's deletion. + let duration = started.elapsed(); + drop(heartbeat); + if let Some(metrics) = metrics { + #[allow(clippy::cast_precision_loss)] + let duration_us = duration.as_micros() as f64; + metrics.record_backup_job(outcome, duration_us); + if bytes > 0 { + metrics.record_backup_bytes_uploaded(bytes); + } + match outcome { + BackupJobOutcome::Succeeded => { + metrics.record_worker_success(QuerySource::BackupJob, duration_us); + } + BackupJobOutcome::Failed => { + metrics.record_worker_error(QuerySource::BackupJob); + } + // A clean shutdown is not a worker error. + BackupJobOutcome::Cancelled => {} + } + } + outcome +} + +/// After the data files: the three documents, the claim re-check, the +/// manifest, the AVAILABLE write. Returns the bytes written over the whole +/// backup. +async fn write_documents( + p: &Pipeline<'_>, + claim: &Arc, + header: ManifestHeader, + files: DataFiles, +) -> Result { + let DataFiles { + entries, + item_count, + mut bytes_uploaded, + } = files; + let docs = BackupDocuments::build(header, entries, p.key_prefix, p.bucket) + .map_err(|e| Stop::Fail(format!("building the manifest: {e}")))?; + for (name, bytes) in docs.documents_before_manifest() { + bytes_uploaded += put_document(p, name, bytes).await?; + } + // The re-check narrows the window in which a fenced-out job can still + // write a manifest. + match tokio::time::timeout(p.heartbeat_period, claim.heartbeat()).await { + Ok(Ok(extenddb_storage::backup_catalog::Heartbeat::Confirmed)) => {} + Ok(Ok(extenddb_storage::backup_catalog::Heartbeat::Lost)) => { + return Err(Stop::ClaimLost( + "the claim was lost before the manifest".to_owned(), + )); + } + Ok(Err(e)) => return Err(Stop::ClaimLost(format!("the claim re-check failed: {e}"))), + Err(_) => { + return Err(Stop::ClaimLost( + "the claim re-check did not return within one heartbeat period".to_owned(), + )); + } + } + kill_point(KillPoint::BeforeManifest, p.shutdown).await; + let (name, bytes) = docs.manifest_document(); + bytes_uploaded += put_document(p, name, bytes).await?; + kill_point(KillPoint::AfterManifest, p.shutdown).await; + let manifest = docs.manifest(); + let fields = StatusFields { + size_bytes: Some(manifest.backup_size_bytes), + item_count: Some(i64::try_from(item_count).unwrap_or(i64::MAX)), + failure_reason: None, + source_table_details_json: serde_json::to_value(&manifest.source_table).ok(), + completed: true, + }; + match failure::write_status(&**claim, JobStatus::Available, &fields).await { + Written::Matched => Ok(bytes_uploaded), + Written::NotMatched(why) | Written::Error(why) => Err(Stop::ClaimLost(format!( + "the AVAILABLE write did not settle: {why}" + ))), + } +} + +/// One document's put under the backup prefix, after the pre-put check; +/// returns the bytes the store reported. +pub(crate) async fn put_document(p: &Pipeline<'_>, name: &str, bytes: &[u8]) -> Result { + if p.shutdown.is_cancelled() { + return Err(Stop::Cancelled); + } + if !p.heartbeat.confirms_the_claim() { + return Err(Stop::ClaimLost(format!( + "no heartbeat confirmed the claim recently enough for {name}" + ))); + } + let key = format!("{}{name}", p.prefix); + tokio::select! { + biased; + () = p.shutdown.cancelled() => Err(Stop::Cancelled), + () = p.heartbeat.lost().cancelled() => Err(Stop::ClaimLost(p.heartbeat.reason())), + result = p.store.put(&key, byte_stream_from(bytes.to_vec())) => match result { + Ok(meta) => Ok(meta.size), + Err(e) => { + if let Some(metrics) = p.metrics { + metrics.record_backup_store_error("put"); + } + Err(Stop::Fail(format!("writing {name}: {e}"))) + } + }, + } +} + +/// The points at which a job pauses under `backup-test-hooks` when +/// `EXTENDDB_BACKUP_PAUSE` names one, until the file +/// `EXTENDDB_BACKUP_PAUSE_RELEASE` names exists (or for good, for a test +/// that kills the server there). Each is logged as `backup job reached kill +/// point `. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum KillPoint { + /// After the put of the first data file returned, before the next file + /// or the documents. + AfterDataFile, + /// After the three documents, before `extenddb-manifest.json`. + BeforeManifest, + /// After the manifest's put returned, before the AVAILABLE write. + AfterManifest, +} + +impl KillPoint { + /// The value of `EXTENDDB_BACKUP_PAUSE` that names this point. + #[must_use] + pub fn name(self) -> &'static str { + match self { + Self::AfterDataFile => "after_data_file", + Self::BeforeManifest => "before_manifest", + Self::AfterManifest => "after_manifest", + } + } +} + +/// Pause at `point` if the environment names it (type doc); a ready future +/// without `backup-test-hooks`. +#[cfg_attr(not(feature = "backup-test-hooks"), allow(clippy::unused_async))] +pub(crate) async fn kill_point(point: KillPoint, shutdown: &CancellationToken) { + #[cfg(feature = "backup-test-hooks")] + { + if std::env::var("EXTENDDB_BACKUP_PAUSE").ok().as_deref() != Some(point.name()) { + return; + } + let release = std::env::var("EXTENDDB_BACKUP_PAUSE_RELEASE").ok(); + tracing::warn!( + "backup job reached kill point {}; paused until {} exists", + point.name(), + release + .as_deref() + .unwrap_or("no release file (the pause does not end)") + ); + loop { + if let Some(release) = &release + && std::path::Path::new(release).exists() + { + tracing::warn!("backup job released from kill point {}", point.name()); + return; + } + if shutdown.is_cancelled() { + tracing::warn!( + "backup job at kill point {} released by the shutdown", + point.name() + ); + return; + } + tokio::time::sleep(std::time::Duration::from_millis(100)).await; + } + } + #[cfg(not(feature = "backup-test-hooks"))] + let _ = (point, shutdown); +} diff --git a/crates/engine/src/backup/job/failure.rs b/crates/engine/src/backup/job/failure.rs new file mode 100644 index 000000000..306fbc416 --- /dev/null +++ b/crates/engine/src/backup/job/failure.rs @@ -0,0 +1,137 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! The conditional status writes of a backup row and the job's failure +//! path. +//! +//! Every status write goes through [`write_status`]: the backend writes +//! conditioned on the claim's generation and on the row being CREATING, and +//! this module gives the three outcomes one meaning. `Matched` is the +//! write. `NotMatched` means the row is no longer this claim's (gone, +//! another generation, or not CREATING) and the caller makes no further +//! store or table access of any kind. `Unknown` (the connection failed +//! after the statement was sent) is followed by a re-read of the row, not +//! treated as a failure: a row that shows the write landed is `Matched`; a +//! row still CREATING under this generation did not take the write, which +//! is sent once more, at most; anything else is `NotMatched`. Each unknown +//! outcome has its re-read, the second write's included, so a second +//! unknown outcome whose write landed is `Matched` (the row is in the +//! target status under this generation) and one whose write was lost is +//! `NotMatched`. An error on the claim connection, on the write or the +//! re-read, ends the caller's work the way `NotMatched` does. +//! +//! The failure path ([`fail_row`]) is the FAILED write followed, only when +//! it matched, by `delete_prefix` of the backup's prefix; the holder of the +//! claim owns the prefix, so a job whose FAILED write did not match never +//! deletes. A deletion that fails leaves the prefix to the reconciler, +//! which deletes a FAILED row's prefix once its heartbeat is stale. + +use extenddb_backup_store::BackupStore; +use extenddb_core::metrics::MetricsCollector; +use extenddb_storage::JobClaim; +use extenddb_storage::backup_catalog::{JobStatus, StatusFields, WriteOutcome}; + +/// What a conditional status write settled to (module doc). +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) enum Written { + /// The row was moved. + Matched, + /// The row is not this claim's to write; the text says what the row + /// showed. + NotMatched(String), + /// The claim connection failed on the write or the re-read. + Error(String), +} + +/// Move the row from CREATING to `to` through `claim` (module doc). +pub(crate) async fn write_status( + claim: &dyn JobClaim, + to: JobStatus, + fields: &StatusFields, +) -> Written { + for attempt in 0..2 { + let outcome = match claim.set_status(JobStatus::Creating, to, fields).await { + Ok(outcome) => outcome, + Err(e) => return Written::Error(format!("the {to} write failed: {e}")), + }; + match outcome { + WriteOutcome::Matched => return Written::Matched, + WriteOutcome::NotMatched => { + return Written::NotMatched( + "the row is no longer CREATING under this claim's generation".to_owned(), + ); + } + WriteOutcome::Unknown => {} + } + let row = match claim.read().await { + Ok(row) => row, + Err(e) => { + return Written::Error(format!( + "the {to} write's outcome is unknown and the row could not be re-read: {e}" + )); + } + }; + if row.owner_generation != claim.generation() { + return Written::NotMatched(format!( + "the row carries generation {} (this claim's is {})", + row.owner_generation, + claim.generation() + )); + } + if row.status == to { + return Written::Matched; + } + if row.status != JobStatus::Creating || attempt == 1 { + return Written::NotMatched(format!( + "the row is {} after a {to} write whose outcome was unknown", + row.status + )); + } + tracing::warn!( + "the {to} write's outcome was unknown and the row is still CREATING under this \ + claim; writing once more" + ); + } + unreachable!("the loop returns on its second attempt") +} + +/// The end of the failure path. +#[derive(Debug)] +pub(crate) enum FailurePath { + /// FAILED written and the prefix deleted. + Deleted, + /// FAILED written; the deletion failed and is the reconciler's. + DeleteFailed(String), + /// FAILED not written by this claim (the row is another's, or the claim + /// connection failed): no store access was made. + NoStoreAccess(String), +} + +/// The failure path (module doc): FAILED with `reason`, then the deletion +/// of `prefix` when the write matched. +pub(crate) async fn fail_row( + claim: &dyn JobClaim, + store: &dyn BackupStore, + prefix: &str, + reason: &str, + metrics: Option<&MetricsCollector>, +) -> FailurePath { + let fields = StatusFields { + failure_reason: Some(reason.to_owned()), + completed: true, + ..StatusFields::default() + }; + match write_status(claim, JobStatus::Failed, &fields).await { + Written::Matched => {} + Written::NotMatched(why) | Written::Error(why) => return FailurePath::NoStoreAccess(why), + } + match store.delete_prefix(prefix).await { + Ok(_) => FailurePath::Deleted, + Err(e) => { + if let Some(metrics) = metrics { + metrics.record_backup_store_error("delete_prefix"); + } + FailurePath::DeleteFailed(e.to_string()) + } + } +} diff --git a/crates/engine/src/backup/job/heartbeat.rs b/crates/engine/src/backup/job/heartbeat.rs new file mode 100644 index 000000000..93c85f782 --- /dev/null +++ b/crates/engine/src/backup/job/heartbeat.rs @@ -0,0 +1,142 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! The job's heartbeat: a timer task that refreshes the claim every period +//! for as long as the job runs, independent of upload progress, and the +//! two things the job reads from it: the instant of the last confirmed beat +//! (checked before every put) and the signal that the claim is lost. +//! +//! One rule decides both: a beat that does not confirm the claim +//! ([`Heartbeat::Lost`] or an error) or that has not returned within one +//! period ends the timer and raises the lost signal, after which the job +//! makes no further store or catalog access. The pre-put check is the same +//! rule read the other way: the last confirmed beat is at most two periods +//! old (one for the timer to fire, one for the beat to return, the bound +//! the timer itself enforces) and no loss has been raised. The claim's +//! insert stands for the first confirmed beat: the check measures from the +//! instant the job took before `claim_backup`, so the maintenance check +//! that runs between the insert and the handle's start (one catalog query +//! with no bound of its own) counts against the two periods, and a job +//! whose first put comes within two periods of the insert needs no beat of +//! its own; `stale_job_after_secs` is at least three periods, so a row that +//! passes the check is never one a reconciler may take over. + +use std::sync::{Arc, Mutex, PoisonError}; +use std::time::Duration; + +use extenddb_core::metrics::MetricsCollector; +use extenddb_storage::CancellationToken; +use extenddb_storage::JobClaim; +use extenddb_storage::backup_catalog::Heartbeat; +use tokio::sync::watch; +use tokio::time::Instant; + +/// The job's side of the heartbeat timer. Dropping it stops the timer. +pub(crate) struct HeartbeatHandle { + last_confirmed: watch::Receiver, + lost: CancellationToken, + reason: Arc>>, + period: Duration, + timer: tokio::task::JoinHandle<()>, +} + +impl HeartbeatHandle { + /// Start the timer on `claim`, beating every `period` from one period + /// after now; `claimed_at`, the instant of the claim's insert, stands + /// for the first confirmed beat. With `metrics`, each confirmed beat + /// records the snapshot's age (`BackupSnapshotAge`), seconds since + /// `created_at`. + pub(crate) fn start( + claim: Arc, + period: Duration, + metrics: Option>, + created_at: f64, + claimed_at: Instant, + ) -> Self { + let (tx, last_confirmed) = watch::channel(claimed_at); + let lost = CancellationToken::new(); + let reason = Arc::new(Mutex::new(None)); + let timer = tokio::spawn({ + let lost = lost.clone(); + let reason = Arc::clone(&reason); + async move { + let mut ticks = tokio::time::interval(period); + ticks.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); + ticks.tick().await; + loop { + ticks.tick().await; + let why = match tokio::time::timeout(period, claim.heartbeat()).await { + Ok(Ok(Heartbeat::Confirmed)) => { + let _ = tx.send(Instant::now()); + if let Some(metrics) = &metrics { + metrics.record_backup_snapshot_age(seconds_since(created_at)); + } + continue; + } + Ok(Ok(Heartbeat::Lost)) => { + "the claim was lost: the row is gone or carries another generation" + .to_owned() + } + Ok(Err(e)) => format!("the heartbeat failed: {e}"), + Err(_) => { + format!("the heartbeat did not return within {} s", period.as_secs()) + } + }; + *reason.lock().unwrap_or_else(PoisonError::into_inner) = Some(why); + lost.cancel(); + return; + } + } + }); + Self { + last_confirmed, + lost, + reason, + period, + timer, + } + } + + /// The signal that the claim is lost; select on `lost().cancelled()`. + pub(crate) fn lost(&self) -> &CancellationToken { + &self.lost + } + + /// Why the claim is lost, once it is. + pub(crate) fn reason(&self) -> String { + self.reason + .lock() + .unwrap_or_else(PoisonError::into_inner) + .clone() + .unwrap_or_else(|| "the claim is lost".to_owned()) + } + + /// The pre-put check (module doc): no loss raised and the last confirmed + /// beat at most two periods old. The second clause is what catches a + /// timer task that stopped running without raising the loss. + pub(crate) fn confirms_the_claim(&self) -> bool { + !self.lost.is_cancelled() && self.last_confirmed.borrow().elapsed() <= 2 * self.period + } + + /// End the timer without raising the loss: a timer task that stopped + /// running, for the test of the pre-put check's second clause. + #[cfg(test)] + pub(crate) fn stop_timer(&self) { + self.timer.abort(); + } +} + +impl Drop for HeartbeatHandle { + fn drop(&mut self) { + self.timer.abort(); + } +} + +/// Seconds between `epoch_secs` and now, on the wall clock. +pub(crate) fn seconds_since(epoch_secs: f64) -> f64 { + let now = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|d| d.as_secs_f64()) + .unwrap_or_default(); + (now - epoch_secs).max(0.0) +} diff --git a/crates/engine/src/backup/job/writer.rs b/crates/engine/src/backup/job/writer.rs new file mode 100644 index 000000000..6f852d66c --- /dev/null +++ b/crates/engine/src/backup/job/writer.rs @@ -0,0 +1,633 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! The data-file pipeline of a backup job: items from the snapshot stream, +//! through the validator, into gzip data files, into the store. +//! +//! Three parts run at once. The feeder (a task of its own) reads the +//! snapshot stream, checks every item with +//! [`validate_restorable_item`] and sends the ones that pass over a bounded +//! item channel ([`ITEM_CHANNEL`] items). One encoder per data file runs on +//! the blocking pool: it takes items off that channel, writes them through +//! `DataFileEncoder` into a [`ChunkSink`] that hands [`CHUNK_BYTES`]-sized +//! chunks over a bounded chunk channel ([`CHUNK_CHANNEL`] chunks), and +//! closes the file once its uncompressed bytes reach +//! `data_file_target_bytes`, so a file exceeds the knob by less than one +//! line. The uploader (the job's own task) starts the store `put` of each +//! file when the file's first chunk arrives, with the chunk channel as the +//! body, so a file is never buffered whole; a file that gets no item (the +//! stream ended or stopped at a file boundary) is never started. The +//! memory the pipeline holds is the item channel's items, the chunk +//! channel's chunks and the encoder's own state, none of which grows with +//! the table or the file size; the feeder's measurement of an oversized +//! item's line (below) holds no line. +//! +//! The validator's rule: after the first refused item the job encodes and +//! uploads nothing more but keeps reading and checking the rest of the +//! stream, logs each refused item (key, attribute path, reason, backup ARN) +//! up to [`REFUSAL_LOG_CAP`] and counts the rest, and then fails with a +//! reason that gives the count and the first item, naming #618 only when +//! the first refusal is an invalid `N` value. An item whose data file line +//! would exceed the format's cap (`MAX_LINE_BYTES`) is refused the same way +//! (`RestorableItemReason::LineTooLong`): the feeder measures the line of +//! every item over the default item limit (`DEFAULT_MAX_ITEM_SIZE_BYTES`), +//! the largest item the cap is sized for, on the blocking pool and through +//! a writer that counts and stops at the cap, so the measurement holds no +//! line and an instance with a raised `max_item_size_bytes` names the item +//! instead of failing in the encoder with no item named, and an instance at +//! the default measures none. Such an item's validation runs on the +//! blocking pool with the measurement (its cost grows with the item: the +//! number rule over every member); an item within the default is validated +//! on the feeder's own task. +//! +//! Before every `put` the uploader checks the heartbeat +//! ([`HeartbeatHandle::confirms_the_claim`]) and starts none otherwise; a +//! lost claim or the shutdown token ends the pipeline at its next await. +//! When the uploader fails on its own (a put refused, an encoder error) it +//! cancels a token the feeder selects on, so the feeder stops reading at +//! once rather than at the stream's next item and the FAILED write does not +//! wait on a slow or dead snapshot stream; the feeder's own refusal leaves +//! that token alone, since the rest of the stream is still to be checked. +//! When the pipeline returns, no `put` is in flight: a stop lets the +//! current put end (its body ends when the encoder drops the sink) or drops +//! it, and the store's own rule for a dropped put applies (the filesystem +//! store removes its temporary, the S3 store aborts the upload, and at +//! shutdown the failure path's `delete_prefix` sweeps what is left). + +use std::io::Write; +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, Ordering}; + +use bytes::{Bytes, BytesMut}; +use extenddb_backup_store::{BackupStore, ByteStream}; +use extenddb_core::backup_format::{ + DataFileChecksums, DataFileEncoder, DataFileEntry, FormatError, KeyRange, KeyRangeTracker, + MAX_LINE_BYTES, data_file_key, encode_item_line_len_bounded, +}; +use extenddb_core::limits::DEFAULT_MAX_ITEM_SIZE_BYTES; +use extenddb_core::metrics::MetricsCollector; +use extenddb_core::types::{Item, item_size_bytes}; +use extenddb_core::validation::{ + RestorableItemError, RestorableItemReason, RestorableKeySchema, RestorableLimits, + validate_restorable_item, +}; +use extenddb_storage::CancellationToken; +use extenddb_storage::error::StorageError; +use futures::StreamExt; +use futures::stream::BoxStream; +use tokio::sync::mpsc; + +use super::heartbeat::HeartbeatHandle; +use super::{KillPoint, kill_point}; + +/// Bytes per chunk handed from the encoder to the upload. +pub(crate) const CHUNK_BYTES: usize = 1024 * 1024; +/// Chunks the chunk channel holds before the encoder waits. +pub(crate) const CHUNK_CHANNEL: usize = 4; +/// Items the item channel holds before the feeder waits. Four, not one: a +/// one-item channel made a 1M-row backup on SQLite take 75% longer (12.4 s +/// to 21.6 s, measured in both orders under one host state) for about +/// 50 MiB less at the peak of the worst-shape run (design section 7). +pub(crate) const ITEM_CHANNEL: usize = 4; +/// Refused items logged one by one; the rest are counted. +pub(crate) const REFUSAL_LOG_CAP: u64 = 1000; + +/// Why the job stops before the manifest. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) enum Stop { + /// The row is this job's and the failure path runs with this reason. + Fail(String), + /// The shutdown token fired: the failure path runs with reason + /// `shutdown`. + Cancelled, + /// The claim is lost or its heartbeat did not return: no further store + /// or catalog access. + ClaimLost(String), +} + +/// What the pipeline wrote. +#[derive(Debug, Default)] +pub(crate) struct DataFiles { + /// Each file's manifest entry with the etag the store reported. + pub entries: Vec<(DataFileEntry, Option)>, + /// Items in every file. + pub item_count: u64, + /// Bytes the store reported stored, over every file. + pub bytes_uploaded: u64, +} + +/// What the uploader needs from the job. +pub(crate) struct Pipeline<'a> { + pub store: &'a dyn BackupStore, + /// The backup's store prefix (`BackupArn::store_prefix`). + pub prefix: &'a str, + /// The ARN, for the log lines. + pub arn: &'a str, + pub heartbeat: &'a HeartbeatHandle, + pub shutdown: &'a CancellationToken, + pub metrics: Option<&'a MetricsCollector>, + /// `data_file_target_bytes`. + pub target_bytes: u64, + /// The heartbeat period, the bound on the claim re-check before the + /// manifest. + pub heartbeat_period: std::time::Duration, + /// The store's configured key prefix and the summary's bucket name + /// (`BackupDocuments::build`). + pub key_prefix: &'a str, + pub bucket: &'a str, +} + +/// The reason the FAILED row carries for refused items (module doc). +fn refusal_reason(count: u64, first: &RestorableItemError) -> String { + let issue = if matches!(first.reason, RestorableItemReason::InvalidNumber(_)) { + " (#618)" + } else { + "" + }; + format!("{count} item(s) cannot be restored; first: {first}{issue}") +} + +/// What the feeder found by the end of the stream. +#[derive(Default)] +struct FeedEnd { + refusals: Option<(u64, RestorableItemError)>, + stream_error: Option, + interrupted: Option, +} + +/// The feeder (module doc). `stop` is shared with the encoders: the feeder +/// sets it at the first refusal, and stops sending when another part set +/// it. `failed` is the uploader's: cancelled when the upload failed, so the +/// feeder returns at once. +struct Feeder { + tx: mpsc::Sender, + key: RestorableKeySchema, + limits: RestorableLimits, + stop: Arc, + failed: CancellationToken, + lost: CancellationToken, + shutdown: CancellationToken, + arn: String, +} + +impl Feeder { + async fn run(self, mut items: BoxStream<'static, Result>) -> FeedEnd { + let Self { + tx, + mut key, + limits, + stop, + failed, + lost, + shutdown, + arn, + } = self; + let mut tx = Some(tx); + let mut end = FeedEnd::default(); + loop { + let next = tokio::select! { + biased; + () = shutdown.cancelled() => { + end.interrupted = Some(Stop::Cancelled); + break; + } + () = lost.cancelled() => { + end.interrupted = Some(Stop::ClaimLost(String::new())); + break; + } + () = failed.cancelled() => break, + next = items.next() => next, + }; + let mut item = match next { + None => break, + Some(Ok(item)) => item, + Some(Err(e)) => { + end.stream_error = Some(e); + break; + } + }; + let refusal = if item_size_bytes(&item) > DEFAULT_MAX_ITEM_SIZE_BYTES { + // An item over the default item limit: the validator's walk + // (the number rule over every member, CPU work proportional + // to the item's size) and then the line's measurement (up + // to the cap's bytes) run on the blocking pool as the + // encoder does; the item and the key come back with the + // result. + let measured = tokio::task::spawn_blocking(move || { + let refusal = validate_restorable_item(&item, &key, &limits) + .err() + .or_else(|| line_too_long(&item, &key)); + (item, key, refusal) + }) + .await; + match measured { + Ok((back, key_back, refusal)) => { + item = back; + key = key_back; + refusal + } + Err(e) => { + end.stream_error = Some(StorageError::Internal(format!( + "checking an item over the default item limit: {e}" + ))); + break; + } + } + } else { + validate_restorable_item(&item, &key, &limits).err() + }; + if let Some(e) = refusal { + match &mut end.refusals { + None => { + tracing::error!("backup {arn}: item cannot be restored: {e}"); + end.refusals = Some((1, e)); + stop.store(true, Ordering::Release); + tx = None; + } + Some((count, _)) => { + *count += 1; + if *count <= REFUSAL_LOG_CAP { + tracing::error!("backup {arn}: item cannot be restored: {e}"); + } + } + } + continue; + } + if end.refusals.is_some() { + continue; + } + if stop.load(Ordering::Acquire) { + break; + } + let Some(sender) = &tx else { break }; + if sender.send(item).await.is_err() { + break; + } + } + if let Some((count, _)) = &end.refusals + && *count > REFUSAL_LOG_CAP + { + tracing::error!( + "backup {arn}: {} more item(s) cannot be restored (not listed)", + count - REFUSAL_LOG_CAP + ); + } + drop(items); + end + } +} + +/// The line-cap rule (module doc) for an item over the default item limit: +/// its line is measured against `MAX_LINE_BYTES` through a counting writer +/// that holds no line ([`encode_item_line_len_bounded`]), and one whose +/// line would exceed the cap is refused with `LineTooLong`. The feeder +/// measures no item within the default limit: the cap is sized to hold +/// every line such an item can expand to (`MAX_LINE_BYTES`'s doc, pinned by +/// the format's own test). An item the serializer refuses is left to the +/// encoder, which reports it. +fn line_too_long(item: &Item, key: &RestorableKeySchema) -> Option { + match encode_item_line_len_bounded(item, MAX_LINE_BYTES) { + Err(FormatError::LineTooLong { cap }) => Some(RestorableItemError { + key: key.shown_key(item), + path: String::new(), + reason: RestorableItemReason::LineTooLong { cap }, + }), + Ok(_) | Err(_) => None, + } +} + +const STOPPED: &str = "the pipeline stopped"; + +/// The encoder's sink: chunks of [`CHUNK_BYTES`] over the chunk channel, the +/// remainder on `flush`. A write after the pipeline stopped, or after the +/// upload ended, fails and poisons the encoder. Both failures are +/// `BrokenPipe`, never `Interrupted`: the encoder retries `Interrupted` on +/// every path (it is std's "try again" kind), and a stopped pipeline does +/// not come back, so an `Interrupted` here would spin the compressor's +/// trailer write for as long as the drain lasts. +struct ChunkSink { + tx: mpsc::Sender, + buf: BytesMut, + stop: Arc, +} + +impl ChunkSink { + fn send(&self, chunk: Bytes) -> std::io::Result<()> { + self.tx + .blocking_send(chunk) + .map_err(|_| std::io::Error::new(std::io::ErrorKind::BrokenPipe, "the upload ended")) + } +} + +impl Write for ChunkSink { + fn write(&mut self, data: &[u8]) -> std::io::Result { + if self.stop.load(Ordering::Acquire) { + return Err(std::io::Error::new(std::io::ErrorKind::BrokenPipe, STOPPED)); + } + self.buf.extend_from_slice(data); + while self.buf.len() >= CHUNK_BYTES { + let chunk = self.buf.split_to(CHUNK_BYTES).freeze(); + self.send(chunk)?; + } + Ok(data.len()) + } + + fn flush(&mut self) -> std::io::Result<()> { + if !self.buf.is_empty() { + let chunk = self.buf.split().freeze(); + self.send(chunk)?; + } + Ok(()) + } +} + +/// How one encoder ended. +enum FileEnd { + /// No item reached the encoder: nothing was written. + Empty, + /// The file reached `data_file_target_bytes` and closed; the stream goes + /// on. + Closed { + checksums: DataFileChecksums, + range: Option, + }, + /// The stream ended inside this file, which closed. + Ended { + checksums: DataFileChecksums, + range: Option, + }, +} + +/// What an encoder hands back: the item channel and the tracker for the +/// next file, and how it ended. +type Encoded = ( + mpsc::Receiver, + Option, + Result, +); + +/// One data file's encoder (module doc), on the blocking pool. +fn encode_file( + mut items: mpsc::Receiver, + sink: ChunkSink, + mut tracker: Option, + target_bytes: u64, +) -> Encoded { + let stop = Arc::clone(&sink.stop); + let mut encoder = DataFileEncoder::new(sink); + let result = loop { + if stop.load(Ordering::Acquire) { + break Err(FormatError::Encode(STOPPED.to_owned())); + } + let Some(item) = items.blocking_recv() else { + if encoder.item_count() == 0 { + break Ok(FileEnd::Empty); + } + break close(&mut encoder, &mut tracker) + .map(|(checksums, range)| FileEnd::Ended { checksums, range }); + }; + if let Some(tracker) = &mut tracker + && let Err(e) = tracker.push(&item) + { + break Err(e); + } + if let Err(e) = encoder.write_item(&item) { + break Err(e); + } + if encoder.uncompressed_bytes() >= target_bytes { + break close(&mut encoder, &mut tracker) + .map(|(checksums, range)| FileEnd::Closed { checksums, range }); + } + }; + // A finished encoder's drop writes nothing; an open one's drop takes + // the sink out first, so no trailer reaches the upload. + drop(encoder); + (items, tracker, result) +} + +/// Close the file: the trailer and the remainder go to the sink, the +/// tracker starts the next file. +fn close( + encoder: &mut DataFileEncoder, + tracker: &mut Option, +) -> Result<(DataFileChecksums, Option), FormatError> { + let checksums = encoder.try_finish()?; + Ok(( + checksums, + tracker.as_mut().and_then(KeyRangeTracker::next_file), + )) +} + +/// The pipeline (module doc). `tracker` is `Some` for a key-ordered backup. +pub(crate) async fn write_data_files( + p: &Pipeline<'_>, + items: BoxStream<'static, Result>, + key: RestorableKeySchema, + limits: RestorableLimits, + mut tracker: Option, +) -> Result { + let stop = Arc::new(AtomicBool::new(false)); + let failed = CancellationToken::new(); + let (item_tx, item_rx) = mpsc::channel(ITEM_CHANNEL); + let feeder = tokio::spawn( + Feeder { + tx: item_tx, + key, + limits, + stop: Arc::clone(&stop), + failed: failed.clone(), + lost: p.heartbeat.lost().clone(), + shutdown: p.shutdown.clone(), + arn: p.arn.to_owned(), + } + .run(items), + ); + let mut item_rx = Some(item_rx); + let mut files = DataFiles::default(); + let mut index: u32 = 1; + let uploaded: Result<(), Stop> = loop { + let (chunk_tx, mut chunk_rx) = mpsc::channel::(CHUNK_CHANNEL); + let sink = ChunkSink { + tx: chunk_tx, + buf: BytesMut::with_capacity(CHUNK_BYTES), + stop: Arc::clone(&stop), + }; + let rx = item_rx.take().expect("the previous encoder returned it"); + let file_tracker = tracker.take(); + let target = p.target_bytes; + let encoder = + tokio::task::spawn_blocking(move || encode_file(rx, sink, file_tracker, target)); + + // The file's first chunk, or the channel closing with none. + let first = tokio::select! { + biased; + () = p.shutdown.cancelled() => Err(Stop::Cancelled), + () = p.heartbeat.lost().cancelled() => Err(Stop::ClaimLost(p.heartbeat.reason())), + chunk = chunk_rx.recv() => Ok(chunk), + }; + let first = match first { + Ok(Some(first)) => first, + Ok(None) => { + let (_, _, end) = match encoder.await { + Ok(encoded) => encoded, + Err(e) => break Err(Stop::Fail(format!("the encoder task failed: {e}"))), + }; + break match end { + Ok(FileEnd::Empty) => Ok(()), + Ok(_) => Err(Stop::Fail(format!( + "data file {index} closed without a chunk" + ))), + Err(_) if stop.load(Ordering::Acquire) => Ok(()), + Err(e) => Err(Stop::Fail(format!("encoding data file {index}: {e}"))), + }; + } + Err(why) => { + stop.store(true, Ordering::Release); + drop(chunk_rx); + let _ = encoder.await; + break Err(why); + } + }; + + if !p.heartbeat.confirms_the_claim() { + stop.store(true, Ordering::Release); + drop(chunk_rx); + let _ = encoder.await; + break Err(Stop::ClaimLost(format!( + "no heartbeat confirmed the claim recently enough for data file {index}" + ))); + } + let name = data_file_key(index); + let key = format!("{}{name}", p.prefix); + let body: ByteStream = Box::pin(futures::stream::once(async move { Ok(first) }).chain( + futures::stream::unfold(chunk_rx, |mut rx| async move { + rx.recv().await.map(|chunk| (Ok(chunk), rx)) + }), + )); + let put = tokio::select! { + biased; + () = p.shutdown.cancelled() => Err(Stop::Cancelled), + () = p.heartbeat.lost().cancelled() => Err(Stop::ClaimLost(p.heartbeat.reason())), + result = p.store.put(&key, body) => result.map_err(|e| { + if let Some(metrics) = p.metrics { + metrics.record_backup_store_error("put"); + } + Stop::Fail(format!("writing {name}: {e}")) + }), + }; + let meta = match put { + Ok(meta) => meta, + Err(why) => { + stop.store(true, Ordering::Release); + let _ = encoder.await; + break Err(why); + } + }; + // The put ended because the encoder dropped the sink, so the encoder + // has returned or is about to. + let (rx, file_tracker, end) = match encoder.await { + Ok(encoded) => encoded, + Err(e) => break Err(Stop::Fail(format!("the encoder task failed: {e}"))), + }; + item_rx = Some(rx); + tracker = file_tracker; + let (checksums, range, last) = match end { + Ok(FileEnd::Closed { checksums, range }) => (checksums, range, false), + Ok(FileEnd::Ended { checksums, range }) => (checksums, range, true), + Ok(FileEnd::Empty) => { + break Err(Stop::Fail(format!( + "data file {index} sent a chunk and then reported no item" + ))); + } + Err(_) if stop.load(Ordering::Acquire) => break Ok(()), + Err(e) => break Err(Stop::Fail(format!("encoding data file {index}: {e}"))), + }; + files.item_count += checksums.item_count; + files.bytes_uploaded += meta.size; + files.entries.push(( + DataFileEntry::from_checksums(name, &checksums, range), + meta.etag, + )); + if index == 1 { + kill_point(KillPoint::AfterDataFile, p.shutdown).await; + } + if last { + break Ok(()); + } + index += 1; + }; + stop.store(true, Ordering::Release); + if uploaded.is_err() { + // The uploader's own failure: the feeder has nothing left to check. + failed.cancel(); + } + drop(item_rx); + let end = match feeder.await { + Ok(end) => end, + Err(e) => return Err(Stop::Fail(format!("the snapshot reader task failed: {e}"))), + }; + if let Err(why @ (Stop::Cancelled | Stop::ClaimLost(_))) = uploaded { + return Err(why); + } + match end.interrupted { + Some(Stop::Cancelled) => return Err(Stop::Cancelled), + Some(Stop::ClaimLost(_)) => return Err(Stop::ClaimLost(p.heartbeat.reason())), + Some(Stop::Fail(_)) | None => {} + } + if let Some(e) = end.stream_error { + return Err(Stop::Fail(format!("reading the snapshot: {e}"))); + } + if let Some((count, first)) = end.refusals { + return Err(Stop::Fail(refusal_reason(count, &first))); + } + uploaded.map(|()| files) +} + +#[cfg(test)] +mod tests { + use std::time::Duration; + + use extenddb_core::types::AttributeValue; + + use super::*; + + fn stopped_sink() -> ChunkSink { + let (tx, _rx) = mpsc::channel(CHUNK_CHANNEL); + ChunkSink { + tx, + buf: BytesMut::with_capacity(CHUNK_BYTES), + stop: Arc::new(AtomicBool::new(true)), + } + } + + /// A write after the stop is `BrokenPipe`, the kind nothing retries. + #[test] + fn a_write_after_the_stop_is_a_broken_pipe() { + let e = stopped_sink().write(b"x").unwrap_err(); + assert_eq!(e.kind(), std::io::ErrorKind::BrokenPipe); + assert_eq!(e.to_string(), STOPPED); + } + + /// An encoder holding items when the pipeline stops fails its close at + /// once instead of retrying the trailer write for the whole drain: the + /// shutdown of a job in the middle of a large table ends within the + /// server's drain window. + #[test] + fn closing_an_encoder_after_the_stop_fails_at_once() { + let (done_tx, done_rx) = std::sync::mpsc::channel(); + std::thread::spawn(move || { + let mut encoder = DataFileEncoder::new(stopped_sink()); + let item = Item::from([("pk".to_owned(), AttributeValue::S("a".to_owned()))]); + // The gzip header is the first write to reach the sink. + let written = encoder.write_item(&item); + let closed = encoder.try_finish().map(|_| ()); + drop(encoder); + let _ = done_tx.send((written, closed)); + }); + let (written, closed) = done_rx + .recv_timeout(Duration::from_secs(5)) + .expect("the close returned within the window"); + let e = written.and(closed).unwrap_err(); + assert!(e.to_string().contains(STOPPED), "{e}"); + } +} diff --git a/crates/engine/src/backup/reconciler.rs b/crates/engine/src/backup/reconciler.rs index 14d48c3bc..c12a5bf53 100644 --- a/crates/engine/src/backup/reconciler.rs +++ b/crates/engine/src/backup/reconciler.rs @@ -2,30 +2,31 @@ // SPDX-License-Identifier: Apache-2.0 //! The reconciler: the periodic worker that resolves catalog rows whose -//! owner is dead, and keeps the configured store's health current. +//! owner is dead, deletes what FAILED and DELETED rows leave in the store, +//! removes tombstones past their retention, and keeps the configured +//! store's health current. //! //! One cycle runs every `interval`. It first checks the catalog-level //! maintenance claim; while a maintenance command holds it the reconciler -//! runs only the duties below, which resolve the states the command waits -//! on or finishes itself (the dead-owner duties, the store's check and the -//! DELETING rows); the duties a later release adds that delete or move a -//! row the command does not wait on (a FAILED row's prefix, tombstone -//! retention) go in the branch the flag skips. The duties, over the records -//! `list_dead_restore_jobs` returns: +//! runs only the duties the command waits on or finishes itself (the +//! dead-owner duties, the store's check and the DELETING rows) and skips +//! the deleting duties it must not race (a FAILED row's prefix, a DELETED +//! row's objects, tombstone retention, the registrations sweep). The +//! duties, in the order a cycle runs them: //! -//! - A RUNNING restore job record whose `heartbeat_at` is older than -//! `stale_job_after` (on the catalog's clock) has a dead owner. One of kind -//! `legacy` is marked FAILED and its target removed through -//! `remove_restore_target`, never resumed: an in-catalog copy has no data -//! files or progress to resume from. One of kind `manifest` is left for the -//! duty that resumes it, and logged once per record for as long as this -//! process runs. -//! - A FAILED restore job record whose table is still a CREATING restore -//! target, the FAILED-target duty: the FAILED write landed (the job's own, -//! or an earlier cycle's) and the target's removal did not (an error -//! between the two statements, or the process stopped between them). The -//! backend lists it once the record is stale, and the target is removed -//! through `remove_restore_target` with no second FAILED write. +//! - Over the records `list_dead_restore_jobs` returns: a RUNNING restore +//! job record whose `heartbeat_at` is older than `stale_job_after` (on +//! the catalog's clock) has a dead owner. One of kind `legacy` is marked +//! FAILED and its target removed through `remove_restore_target`, never +//! resumed: an in-catalog copy has no data files or progress to resume +//! from. One of kind `manifest` is left for PR 9a's reconciler, the +//! release with the restore job, which resumes it, and logged once per +//! record for as long as this process runs. A FAILED restore job record +//! whose table is still a CREATING restore target, the FAILED-target +//! duty: the FAILED write landed (the job's own, or an earlier cycle's) +//! and the target's removal did not; the backend lists it once the record +//! is stale, and the target is removed through `remove_restore_target` +//! with no second FAILED write. //! - An in-catalog CREATING backup row whose create is dead (reported by the //! backend's `abandoned_creating_backups`) is removed through the reader's //! `delete`, which takes its items with it. @@ -34,43 +35,84 @@ //! then the identity check and manifest probe) and sets the store's //! health, so a store whose checks run after the listener is up, or that //! failed at startup or later, is healthy once a check passes, and one -//! that fails is refused from the next cycle on. The check runs after the -//! duties above and under the store's bound -//! ([`super::ConfiguredStore::check_timeout`]: 5 s for a filesystem root, -//! 30 s for a bucket), so a store that has stopped answering costs each -//! cycle that bound and nothing more: the dead-owner duties run every -//! `interval` plus the bound, the check is recorded as a transport -//! failure, and a filesystem check leaves one blocking thread parked on -//! the mount until it answers, after which no check starts until that -//! call returns (the runtime module's in-flight rule), so a mount that -//! stays stalled parks one thread and not one per cycle. The check is -//! also selected against the shutdown token, so a cycle that is checking -//! when the server stops returns at once and the drain finishes. -//! While the store is healthy, each DELETING row of the configured store -//! (a DeleteBackup whose deletion did not finish) is finished -//! (`finish_backup_delete` in the engine's backup module): the manifest, then the prefix, then -//! the row to DELETED. While it is unhealthy the reconciler acts on no row -//! of the store. +//! that fails is refused from the next cycle on. The check runs under the +//! store's bound ([`super::ConfiguredStore::check_timeout`]: 5 s for a +//! filesystem root, 30 s for a bucket), so a store that has stopped +//! answering costs each cycle that bound and nothing more: the duties +//! above run every `interval` plus the bound, the check is recorded as a +//! transport failure, and a filesystem check leaves one blocking thread +//! parked on the mount until it answers, after which no check starts +//! until that call returns (the runtime module's in-flight rule), so a +//! mount that stays stalled parks one thread and not one per cycle. The +//! check is also selected against the shutdown token, so a cycle that is +//! checking when the server stops returns at once and the drain finishes. +//! While the store is unhealthy the reconciler acts on no row of the +//! store. While it is healthy, on the rows of the configured store: +//! - Each DELETING row (a DeleteBackup whose deletion did not finish) is +//! finished (`finish_backup_delete` in the engine's backup module): the +//! manifest, then the prefix, then the row to DELETED. +//! - Each CREATING row whose owner is dead is resolved. The backend's +//! `claim_dead_backup` is the test and the takeover (a fresh generation +//! fences the dead job's writes out; `heartbeat_at` is left as it was); +//! a row it does not hand over has a live owner or is another's. Under +//! the claim the store is probed for a complete manifest: +//! `extenddb-manifest.json` and its sidecar present and agreeing, the +//! manifest parsing with the row's ARN, and every data file it lists +//! present under the prefix with the recorded size. A complete manifest +//! makes the row AVAILABLE with the manifest's size, item count and +//! `source_table`; anything else makes it FAILED with reason +//! [`OWNER_DEAD_NO_MANIFEST`], and nothing is deleted here: the +//! FAILED-prefix duty, which runs later in this cycle, deletes it once +//! the row's heartbeat is older than `stale_job_after` on the database's +//! clock: in this cycle on PostgreSQL, MongoDB and Cassandra, where a +//! heartbeat that old is what made the owner dead (on PostgreSQL +//! together with the owner's claim session gone, its advisory lock +//! free); on SQLite, where the slot did, once the dead owner's last +//! beat is that old. A store failure during the probe (not an absent +//! object) leaves the row for the next cycle. The two writes are +//! conditioned on the row being CREATING under the new generation +//! (`job::failure::write_status`); one that matches no row (another +//! reconciler took the row over in the gap, item 29) leaves the row to +//! that owner. +//! - Not while the maintenance claim is held: a FAILED row whose +//! `heartbeat_at` is older than `stale_job_after` (the backend's +//! `list_backup_rows_older_than` on the heartbeat clock) and whose +//! prefix lists objects has its prefix deleted, retried each cycle while +//! the listing shows objects; a DELETED row whose prefix lists objects +//! has its prefix deleted; a FAILED or DELETED row whose prefix lists +//! nothing and whose `completed_at` is older than `failed_row_retention` +//! (the completed clock) is removed. Every age is the database's +//! comparison; the engine holds no clock. +//! - Not while the maintenance claim is held, on every backend: the +//! backend's `sweep_restore_registrations` (a no-op but on Cassandra). //! -//! A cycle that fails logs and waits for the next; each cycle records the -//! worker metrics the other workers record (`WorkerLastSuccess` and -//! `WorkerCycleLatency` on success, `WorkerErrorCount` on failure, under -//! `QuerySource::BackupReconciler`); the worker stops at the shutdown token. +//! Every duty acts only on rows of the configured store (kind, root and +//! id), the legacy duty excepted. A cycle that fails logs and waits for the +//! next; each cycle records the worker metrics the other workers record +//! (`WorkerLastSuccess` and `WorkerCycleLatency` on success, +//! `WorkerErrorCount` on failure, under `QuerySource::BackupReconciler`); +//! the worker stops at the shutdown token. use std::collections::HashSet; use std::sync::Arc; use std::time::Duration; -use extenddb_core::backup_format::BackupArn; +use extenddb_backup_store::{BackupStore, StoreError}; +use extenddb_core::backup_format::{ + BackupArn, EXTENDDB_MANIFEST_FILE, ExtenddbManifest, MANIFEST_CHECKSUM_FILE, + verify_manifest_checksum, +}; use extenddb_core::metrics::{MetricsCollector, QuerySource}; use extenddb_storage::backup_catalog::{ - BackupStatus, JobStatus, RestoreJobRow, RestoreKind, RestoreTargetKey, + BackupRow, BackupStatus, JobStatus, RestoreJobRow, RestoreKind, RestoreTargetKey, StaleClock, + StatusFields, StoreIdentity, }; use extenddb_storage::error::StorageError; use extenddb_storage::{BackupCatalog, CancellationToken, LegacyBackupReader, sleep_or_shutdown}; use futures::TryStreamExt; use super::BackupRuntime; +use super::job::failure::{Written, write_status}; /// The two durations a reconciler reads from `[backup]`. #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -84,6 +126,11 @@ pub struct ReconcilerSettings { /// The reason written on a restore job record whose owner died. pub const OWNER_DEAD: &str = "owner dead: no heartbeat within stale_job_after_secs"; +/// The reason written on a CREATING backup row whose owner died without a +/// complete manifest in the store. +pub const OWNER_DEAD_NO_MANIFEST: &str = + "owner dead: no heartbeat within stale_job_after_secs; no complete manifest"; + /// What one cycle did. #[derive(Debug, Default, Clone, PartialEq, Eq)] pub struct CycleReport { @@ -110,6 +157,38 @@ pub struct CycleReport { pub store_healthy: Option, /// DELETING rows of the configured store moved to DELETED, by ARN. pub deletions_finished: Vec, + /// CREATING rows whose owner was dead, resolved to the status given, by + /// ARN. + pub creating_resolved: Vec<(String, BackupStatus)>, + /// CREATING rows this cycle took over whose status write matched no row + /// (another owner took them over first), left to that owner, by ARN. + pub creating_left_to_owner: Vec, + /// FAILED rows past the stale window whose prefix was deleted, by ARN. + pub failed_prefixes_deleted: Vec, + /// DELETED rows whose prefix still listed objects and was deleted, by + /// ARN. + pub deleted_prefixes_swept: Vec, + /// FAILED and DELETED rows past their retention with an empty prefix, + /// removed, by ARN. + pub tombstones_removed: Vec, + /// Backups whose restore registrations the backend swept, by ARN. + pub registrations_swept: Vec, +} + +impl CycleReport { + /// Whether the cycle changed a row or an object. + #[must_use] + pub fn acted(&self) -> bool { + !(self.legacy_restores_removed.is_empty() + && self.failed_targets_removed.is_empty() + && self.legacy_backups_removed.is_empty() + && self.deletions_finished.is_empty() + && self.creating_resolved.is_empty() + && self.failed_prefixes_deleted.is_empty() + && self.deleted_prefixes_swept.is_empty() + && self.tombstones_removed.is_empty() + && self.registrations_swept.is_empty()) + } } /// What a reconciler carries from one cycle to the next: the manifest @@ -165,17 +244,22 @@ pub async fn run( #[allow(clippy::cast_precision_loss)] let cycle_us = started.elapsed().as_micros() as f64; metrics.record_worker_success(QuerySource::BackupReconciler, cycle_us); - if !report.legacy_restores_removed.is_empty() - || !report.failed_targets_removed.is_empty() - || !report.legacy_backups_removed.is_empty() - || !report.deletions_finished.is_empty() - { + if report.acted() { tracing::info!( - "backup reconciler: removed {} abandoned restore target(s) and {} \ - abandoned in-catalog backup(s), finished {} deletion(s)", - report.legacy_restores_removed.len() + report.failed_targets_removed.len(), + "backup reconciler: failed and removed {} dead legacy restore(s), removed \ + the target(s) of {} FAILED restore record(s) and {} abandoned in-catalog \ + backup(s), finished {} deletion(s), resolved {} dead CREATING row(s), \ + deleted {} FAILED prefix(es), swept {} DELETED prefix(es), removed {} \ + tombstone(s), swept the registrations of {} backup(s)", + report.legacy_restores_removed.len(), + report.failed_targets_removed.len(), report.legacy_backups_removed.len(), - report.deletions_finished.len() + report.deletions_finished.len(), + report.creating_resolved.len(), + report.failed_prefixes_deleted.len(), + report.deleted_prefixes_swept.len(), + report.tombstones_removed.len(), + report.registrations_swept.len() ); } } @@ -312,17 +396,391 @@ where report.store_healthy = Some(store.is_healthy()); if let Some(identity) = store.identity().filter(|_| store.is_healthy()) { report.deletions_finished = finish_deleting_rows(storage, store, &identity).await?; + (report.creating_resolved, report.creating_left_to_owner) = + resolve_dead_creating_rows(storage, &*store.store, &identity, stale_job_after) + .await?; + if !report.maintenance_held { + let retention = runtime.job_settings().failed_row_retention; + report.failed_prefixes_deleted = delete_stale_failed_prefixes( + storage, + &*store.store, + &identity, + stale_job_after, + ) + .await?; + report.deleted_prefixes_swept = + sweep_deleted_prefixes(storage, &*store.store, &identity).await?; + report.tombstones_removed = + remove_tombstones(storage, &*store.store, &identity, retention).await?; + } } } if !report.maintenance_held { - // The duties a maintenance command does not wait on and must not - // race: a later release's deletion of a FAILED row's prefix and its - // removal of tombstones past their retention run here. + report.registrations_swept = storage.sweep_restore_registrations().await?; } Ok(report) } +/// The duty for CREATING rows of the configured store whose owner is dead +/// (module doc). Returns the rows resolved with their new status and the +/// rows whose write matched no row, left to their new owner; a row the +/// backend does not hand over is skipped, a row whose probe or write fails +/// is logged and left for the next cycle. +async fn resolve_dead_creating_rows( + storage: &S, + store: &dyn BackupStore, + identity: &StoreIdentity, + stale_job_after: Duration, +) -> Result<(Vec<(String, BackupStatus)>, Vec), StorageError> { + let rows: Vec = storage + .list_backup_rows(Some(BackupStatus::Creating), identity) + .try_collect() + .await?; + let mut resolved = Vec::new(); + let mut left_to_owner = Vec::new(); + for row in rows { + let claim = match storage.claim_dead_backup(&row.key(), stale_job_after).await { + Ok(Some(claim)) => claim, + Ok(None) => continue, + Err(e) => { + tracing::error!( + "could not test whether the owner of CREATING backup {} (account {}) is \ + dead; the next cycle retries: {e}", + row.backup_arn, + row.account_id + ); + continue; + } + }; + let arn = match BackupArn::parse(&row.backup_arn) { + Ok(arn) => arn, + Err(e) => { + tracing::error!( + "CREATING row {} carries an ARN this build cannot parse; left as it is: {e}", + row.backup_arn + ); + continue; + } + }; + let (to, fields) = match probe_manifest(store, &arn).await { + Ok(Probe::Complete(fields)) => (JobStatus::Available, fields), + Ok(Probe::Incomplete(why)) => { + tracing::warn!( + "backup {} (account {}): its owner is dead and the store holds no complete \ + manifest ({why}); the row is FAILED and its prefix is deleted once the row's \ + heartbeat is stale", + row.backup_arn, + row.account_id + ); + ( + JobStatus::Failed, + StatusFields { + failure_reason: Some(OWNER_DEAD_NO_MANIFEST.to_owned()), + completed: true, + ..StatusFields::default() + }, + ) + } + Err(e) => { + tracing::error!( + "backup {} (account {}): its owner is dead and the store could not be \ + probed for its manifest; the next cycle retries: {e}", + row.backup_arn, + row.account_id + ); + continue; + } + }; + match write_status(&*claim, to, &fields).await { + Written::Matched => { + tracing::info!( + "backup {} (account {}): its owner was dead; the row is now {to}", + row.backup_arn, + row.account_id + ); + let status = match to { + JobStatus::Available => BackupStatus::Available, + _ => BackupStatus::Failed, + }; + resolved.push((row.backup_arn, status)); + } + Written::NotMatched(why) => { + tracing::info!( + "backup {} was no longer this cycle's to resolve ({why}); left to its owner", + row.backup_arn + ); + left_to_owner.push(row.backup_arn); + } + Written::Error(e) => tracing::error!( + "backup {} (account {}): the {to} write failed; the next cycle retries: {e}", + row.backup_arn, + row.account_id + ), + } + drop(claim); + } + Ok((resolved, left_to_owner)) +} + +/// What the manifest probe found under a dead owner's prefix. +enum Probe { + /// A complete, verified manifest: the AVAILABLE write's fields. + Complete(StatusFields), + /// No manifest, or one the reader refuses, or a data file missing or of + /// another size; the text says which. + Incomplete(String), +} + +/// The manifest probe of the dead-owner duty (module doc). A store error +/// other than an absent object is the `Err`. +async fn probe_manifest(store: &dyn BackupStore, arn: &BackupArn) -> Result { + let prefix = arn.store_prefix(); + let manifest = match read_object(store, &format!("{prefix}{EXTENDDB_MANIFEST_FILE}")).await? { + Some(bytes) => bytes, + None => return Ok(Probe::Incomplete("no manifest".to_owned())), + }; + let sidecar = match read_object(store, &format!("{prefix}{MANIFEST_CHECKSUM_FILE}")).await? { + Some(bytes) => bytes, + None => { + return Ok(Probe::Incomplete( + "a manifest without its sidecar".to_owned(), + )); + } + }; + if let Err(e) = verify_manifest_checksum(&manifest, &sidecar) { + return Ok(Probe::Incomplete(format!( + "the sidecar does not match: {e}" + ))); + } + let manifest = match ExtenddbManifest::from_json_bytes(&manifest) { + Ok(manifest) => manifest, + Err(e) => return Ok(Probe::Incomplete(format!("the manifest is refused: {e}"))), + }; + match manifest.backup_arn() { + Ok(found) if found == *arn => {} + Ok(found) => { + return Ok(Probe::Incomplete(format!( + "the manifest names {}, not the row's backup", + found.to_arn_string() + ))); + } + Err(e) => { + return Ok(Probe::Incomplete(format!( + "the manifest's ARN is refused: {e}" + ))); + } + } + let listed: std::collections::HashMap = store + .list(&prefix) + .map_ok(|meta| (meta.key, meta.size)) + .try_collect() + .await?; + for file in &manifest.data_files { + match listed.get(&format!("{prefix}{}", file.key)) { + Some(size) if *size == file.bytes => {} + Some(size) => { + return Ok(Probe::Incomplete(format!( + "data file {} has {size} bytes in the store and {} in the manifest", + file.key, file.bytes + ))); + } + None => { + return Ok(Probe::Incomplete(format!( + "data file {} is not in the store", + file.key + ))); + } + } + } + let item_count = manifest + .data_files + .iter() + .map(|f| f.item_count) + .sum::(); + Ok(Probe::Complete(StatusFields { + size_bytes: Some(manifest.backup_size_bytes), + item_count: Some(i64::try_from(item_count).unwrap_or(i64::MAX)), + failure_reason: None, + source_table_details_json: serde_json::to_value(&manifest.source_table).ok(), + completed: true, + })) +} + +/// One object's bytes, `None` when absent. +async fn read_object(store: &dyn BackupStore, key: &str) -> Result>, StoreError> { + let body = match store.get(key).await { + Ok(body) => body, + Err(StoreError::NotFound) => return Ok(None), + Err(e) => return Err(e), + }; + let bytes = body + .try_fold(Vec::new(), |mut acc, chunk| async move { + acc.extend_from_slice(&chunk); + Ok(acc) + }) + .await?; + Ok(Some(bytes)) +} + +/// Whether the prefix lists at least one object. +async fn prefix_has_objects(store: &dyn BackupStore, prefix: &str) -> Result { + Ok(store.list(prefix).try_next().await?.is_some()) +} + +/// The FAILED-prefix duty (module doc): FAILED rows whose heartbeat is older +/// than `stale_job_after`, their prefix deleted while it lists objects. +async fn delete_stale_failed_prefixes( + storage: &S, + store: &dyn BackupStore, + identity: &StoreIdentity, + stale_job_after: Duration, +) -> Result, StorageError> { + let rows: Vec = storage + .list_backup_rows_older_than( + BackupStatus::Failed, + identity, + StaleClock::Heartbeat, + stale_job_after, + ) + .try_collect() + .await?; + let mut deleted = Vec::new(); + for row in rows { + if delete_prefix_if_listed(store, &row, "FAILED").await { + deleted.push(row.backup_arn); + } + } + Ok(deleted) +} + +/// The DELETED-prefix duty (module doc): DELETED rows whose prefix still +/// lists objects, the prefix deleted. +async fn sweep_deleted_prefixes( + storage: &S, + store: &dyn BackupStore, + identity: &StoreIdentity, +) -> Result, StorageError> { + let rows: Vec = storage + .list_backup_rows(Some(BackupStatus::Deleted), identity) + .try_collect() + .await?; + let mut swept = Vec::new(); + for row in rows { + if delete_prefix_if_listed(store, &row, "DELETED").await { + swept.push(row.backup_arn); + } + } + Ok(swept) +} + +/// Delete `row`'s prefix when it lists objects; returns whether a deletion +/// ran. A store failure is logged and left for the next cycle. +async fn delete_prefix_if_listed(store: &dyn BackupStore, row: &BackupRow, status: &str) -> bool { + let prefix = match BackupArn::parse(&row.backup_arn) { + Ok(arn) => arn.store_prefix(), + Err(e) => { + tracing::error!( + "{status} row {} carries an ARN this build cannot parse; left as it is: {e}", + row.backup_arn + ); + return false; + } + }; + match prefix_has_objects(store, &prefix).await { + Ok(true) => {} + Ok(false) => return false, + Err(e) => { + tracing::error!( + "could not list the prefix of {status} backup {}; the next cycle retries: {e}", + row.backup_arn + ); + return false; + } + } + match store.delete_prefix(&prefix).await { + Ok(n) => { + tracing::info!( + "deleted {n} object(s) left under the prefix of {status} backup {} (account {})", + row.backup_arn, + row.account_id + ); + true + } + Err(e) => { + tracing::error!( + "could not delete the prefix of {status} backup {} (account {}); the next cycle \ + retries: {e}", + row.backup_arn, + row.account_id + ); + false + } + } +} + +/// The tombstone duty (module doc): FAILED and DELETED rows whose +/// `completed_at` is older than `retention` and whose prefix lists nothing +/// are removed. +async fn remove_tombstones( + storage: &S, + store: &dyn BackupStore, + identity: &StoreIdentity, + retention: Duration, +) -> Result, StorageError> { + let mut removed = Vec::new(); + for status in [BackupStatus::Failed, BackupStatus::Deleted] { + let rows: Vec = storage + .list_backup_rows_older_than(status, identity, StaleClock::Completed, retention) + .try_collect() + .await?; + for row in rows { + let prefix = match BackupArn::parse(&row.backup_arn) { + Ok(arn) => arn.store_prefix(), + Err(e) => { + tracing::error!( + "{status} row {} carries an ARN this build cannot parse; left as it \ + is: {e}", + row.backup_arn + ); + continue; + } + }; + match prefix_has_objects(store, &prefix).await { + Ok(false) => {} + Ok(true) => continue, + Err(e) => { + tracing::error!( + "could not list the prefix of {status} backup {} before removing its \ + row; the next cycle retries: {e}", + row.backup_arn + ); + continue; + } + } + match storage.remove_backup_row(&row.key()).await { + Ok(true) => { + tracing::info!( + "removed the {status} row of backup {} (account {}): its prefix is empty \ + and its retention has passed", + row.backup_arn, + row.account_id + ); + removed.push(row.backup_arn); + } + Ok(false) => {} + Err(e) => tracing::error!( + "could not remove the {status} row of backup {} (account {}); the next cycle \ + retries: {e}", + row.backup_arn, + row.account_id + ), + } + } + } + Ok(removed) +} + /// The duty for DELETING rows of the configured store: finish each deletion. /// Returns the ARNs moved to DELETED; a row whose deletion fails is logged /// and left for the next cycle. diff --git a/crates/engine/src/backup/runtime.rs b/crates/engine/src/backup/runtime.rs index 0301caa07..a5c998a90 100644 --- a/crates/engine/src/backup/runtime.rs +++ b/crates/engine/src/backup/runtime.rs @@ -35,17 +35,20 @@ use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::{Arc, RwLock}; +use std::time::Duration; use extenddb_backup_store::{BackupStore, ByteStream, ObjectMeta, StoreError}; use extenddb_core::error::DynamoDbError; +use extenddb_core::metrics::MetricsCollector; use extenddb_storage::backup_catalog::{ - JobBound, StoreIdentity, StoreKind, limit_exceeded_message, + JOB_HEARTBEAT_INTERVAL, JobBound, StoreIdentity, StoreKind, limit_exceeded_message, }; -use extenddb_storage::{BackupCatalog, JobPermit}; +use extenddb_storage::{BackupCatalog, CancellationToken, JobPermit}; use futures::StreamExt; use futures::future::BoxFuture; use futures::stream::BoxStream; use tokio::sync::Semaphore; +use tokio_util::task::TaskTracker; use super::store_identity::{self, StoreProblem}; @@ -53,11 +56,43 @@ use super::store_identity::{self, StoreProblem}; /// refusal names it. const MAX_CONCURRENT_JOBS_KEY: &str = "[backup] max_concurrent_jobs"; +/// The `[backup]` durations and sizes the store-backed backup job and the +/// reconciler read, as the server resolved them from the configuration. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct JobSettings { + /// `data_file_target_bytes`: a data file closes once its uncompressed + /// bytes reach this. + pub data_file_target_bytes: u64, + /// The job's heartbeat period: `max(heartbeat_interval_secs, + /// JOB_HEARTBEAT_INTERVAL)`, which the server computes; a heartbeat that + /// has not confirmed the claim within one period aborts the job. + pub heartbeat_interval: Duration, + /// `stale_job_after_secs`: the window the maintenance-claim check reads + /// with, and the heartbeat age after which a job's owner is dead. + pub stale_job_after: Duration, + /// `failed_row_retention_secs`: how long a FAILED or DELETED row whose + /// prefix is empty is kept. + pub failed_row_retention: Duration, +} + +impl Default for JobSettings { + /// The configuration's defaults: 64 MiB files, the 15 s heartbeat, + /// 300 s staleness, 7 days of retention. + fn default() -> Self { + Self { + data_file_target_bytes: 64 * 1024 * 1024, + heartbeat_interval: JOB_HEARTBEAT_INTERVAL, + stale_job_after: Duration::from_secs(300), + failed_row_retention: Duration::from_secs(7 * 24 * 3600), + } + } +} + /// How long a filesystem store's check may run, at startup and in a /// reconciler cycle, before the store is treated as not answering: a local /// disk answers in milliseconds, and a network mount that takes longer is /// one that has stopped answering. -pub const LOCAL_CHECK_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(5); +pub const LOCAL_CHECK_TIMEOUT: Duration = Duration::from_secs(5); /// How long an S3 store's check may run in a reconciler cycle. The check /// sends a few small requests (`validate`'s listing, the identity file's @@ -67,14 +102,18 @@ pub const LOCAL_CHECK_TIMEOUT: std::time::Duration = std::time::Duration::from_s /// so a timed-out check is dropped before the SDK starts a retry of its /// stalled request, and no retry outlives the check. A stalled endpoint thus /// costs a cycle this bound, not the SDK's whole retry budget. -pub const S3_CHECK_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(30); +pub const S3_CHECK_TIMEOUT: Duration = Duration::from_secs(30); const _: () = assert!( S3_CHECK_TIMEOUT.as_secs() < extenddb_backup_store::S3_ATTEMPT_TIMEOUT.as_secs(), "the S3 check bound must fire before the SDK's attempt timeout" ); /// One per server process, shared by every request through the operation -/// context. +/// context. Besides the slots and the store it carries what a store-backed +/// backup job needs and no request hands it: the job settings, the metrics +/// collector, the shutdown token the jobs are cancelled on, the tracker the +/// server drains them through ([`BackupRuntime::drain_jobs`]), and the +/// backend and version the manifest names as its source. pub struct BackupRuntime { slots: Arc, max_concurrent_jobs: usize, @@ -86,9 +125,19 @@ pub struct BackupRuntime { /// configuration (a backend that cannot pin a table snapshot); such a /// create is not refused by an unhealthy store. legacy_create_only: bool, - /// `[backup] stale_job_after_secs`: the window a job's claim check reads - /// the maintenance claim with. - stale_job_after: std::time::Duration, + job_settings: JobSettings, + /// The collector the jobs record into; `None` in commands and tests + /// without one. + metrics: Option>, + /// Cancelled by the server's shutdown: a running job then runs its + /// failure path with reason `shutdown`. + shutdown: CancellationToken, + /// Every job task spawned through [`BackupRuntime::spawn_job`]. + jobs: TaskTracker, + /// The storage backend's name, the manifest's `source_backend`. + source_backend: String, + /// The build's version, the manifest's `source_extenddb_version`. + source_extenddb_version: String, } impl std::fmt::Debug for BackupRuntime { @@ -103,13 +152,15 @@ impl std::fmt::Debug for BackupRuntime { .map(ConfiguredStore::identity), ) .field("legacy_create_only", &self.legacy_create_only) + .field("job_settings", &self.job_settings) .finish_non_exhaustive() } } /// The store this instance is configured with: its kind and root (the /// configuration's), its id once the identity check has read it, the handle, -/// the spool for restores from it, and its health. +/// the spool for restores from it, the names the DynamoDB-shaped manifests +/// carry for it, and its health. /// /// The id is `None` until a check passes: a store whose startup check failed /// (a transport failure, say) has an unknown id, and a row of its kind and @@ -125,6 +176,15 @@ pub struct ConfiguredStore { /// The restore spool for an S3 store: the directory and the /// `:` label a restore job record names. pub spool: Option, + /// `manifest-summary.json`'s `s3Bucket`: the bucket name on S3, the root + /// path on a filesystem store ([`Self::with_manifest_names`]; the root + /// label until set). + pub bucket: String, + /// The key prefix the store was opened with (`[backup.store.s3] prefix`; + /// a filesystem store has none), which every key of the DynamoDB-shaped + /// manifests starts with ([`Self::with_manifest_names`]; empty until + /// set). + pub key_prefix: String, store_id: RwLock>, health: RwLock, /// Whether the store's `validate` has passed; the checks run it first @@ -203,6 +263,8 @@ impl ConfiguredStore { }; Self { kind, + bucket: root.clone(), + key_prefix: String::new(), root, store, spool, @@ -226,6 +288,8 @@ impl ConfiguredStore { ) -> Self { Self { kind, + bucket: root.clone(), + key_prefix: String::new(), root, store, spool, @@ -236,6 +300,17 @@ impl ConfiguredStore { } } + /// The same store with the names the DynamoDB-shaped manifests carry + /// for it: `bucket` is `manifest-summary.json`'s `s3Bucket` (the bucket + /// on S3, the root path on a filesystem store) and `key_prefix` the + /// store's configured key prefix (empty when it has none). + #[must_use] + pub fn with_manifest_names(mut self, bucket: String, key_prefix: String) -> Self { + self.bucket = bucket; + self.key_prefix = key_prefix; + self + } + /// The store's identity, once a check has read its id. #[must_use] pub fn identity(&self) -> Option { @@ -354,7 +429,7 @@ impl ConfiguredStore { /// ([`LOCAL_CHECK_TIMEOUT`] for a filesystem root, [`S3_CHECK_TIMEOUT`] /// for a bucket). #[must_use] - pub fn check_timeout(&self) -> std::time::Duration { + pub fn check_timeout(&self) -> Duration { match self.kind { StoreKind::Filesystem => LOCAL_CHECK_TIMEOUT, StoreKind::S3 => S3_CHECK_TIMEOUT, @@ -370,7 +445,7 @@ impl ConfiguredStore { /// The problem a check, or a command's identity read, that did not finish /// within `bound` is recorded as: a transport failure naming the bound. -fn did_not_finish(what: &str, bound: std::time::Duration) -> StoreProblem { +fn did_not_finish(what: &str, bound: Duration) -> StoreProblem { StoreProblem::Transport(format!( "the {what} did not finish within {} s", bound.as_secs() @@ -605,7 +680,9 @@ impl BackupStore for CheckCalls<'_> { } impl BackupRuntime { - /// `max_concurrent_jobs` slots and no configured store. + /// `max_concurrent_jobs` slots, no configured store, the default job + /// settings, no metrics, a shutdown token nothing cancels, and `test` as + /// the source backend with this crate's version. #[must_use] pub fn new(max_concurrent_jobs: usize) -> Self { Self { @@ -613,7 +690,12 @@ impl BackupRuntime { max_concurrent_jobs, configured_store: None, legacy_create_only: false, - stale_job_after: std::time::Duration::from_secs(300), + job_settings: JobSettings::default(), + metrics: None, + shutdown: CancellationToken::new(), + jobs: TaskTracker::new(), + source_backend: "test".to_owned(), + source_extenddb_version: env!("CARGO_PKG_VERSION").to_owned(), } } @@ -621,15 +703,118 @@ impl BackupRuntime { /// the jobs' maintenance-claim check reads with (the default is the /// configuration's default, 300 s). #[must_use] - pub fn with_stale_job_after(mut self, stale_job_after: std::time::Duration) -> Self { - self.stale_job_after = stale_job_after; + pub fn with_stale_job_after(mut self, stale_job_after: Duration) -> Self { + self.job_settings.stale_job_after = stale_job_after; self } /// `[backup] stale_job_after_secs`. #[must_use] - pub fn stale_job_after(&self) -> std::time::Duration { - self.stale_job_after + pub fn stale_job_after(&self) -> Duration { + self.job_settings.stale_job_after + } + + /// The same runtime with every job setting set. + #[must_use] + pub fn with_job_settings(mut self, settings: JobSettings) -> Self { + self.job_settings = settings; + self + } + + /// The settings the jobs and the reconciler read. + #[must_use] + pub fn job_settings(&self) -> JobSettings { + self.job_settings + } + + /// The same runtime with the collector the jobs record into. + #[must_use] + pub fn with_metrics(mut self, metrics: Arc) -> Self { + self.metrics = Some(metrics); + self + } + + /// The collector the jobs record into, if any. + #[must_use] + pub fn metrics(&self) -> Option<&Arc> { + self.metrics.as_ref() + } + + /// The same runtime with the token the server's shutdown cancels the + /// jobs on ([`Self::drain_jobs`] cancels it too). + #[must_use] + pub fn with_shutdown(mut self, shutdown: CancellationToken) -> Self { + self.shutdown = shutdown; + self + } + + /// The token the jobs are cancelled on. + #[must_use] + pub fn shutdown(&self) -> &CancellationToken { + &self.shutdown + } + + /// The same runtime with the backend name and build version the + /// manifests name as their source. + #[must_use] + pub fn with_source(mut self, backend: &str, version: &str) -> Self { + self.source_backend = backend.to_owned(); + self.source_extenddb_version = version.to_owned(); + self + } + + /// The manifest's `source_backend` and `source_extenddb_version`. + #[must_use] + pub fn source(&self) -> (&str, &str) { + (&self.source_backend, &self.source_extenddb_version) + } + + /// Spawn a job task under the tracker [`Self::drain_jobs`] waits on. + pub(crate) fn spawn_job(&self, job: F) -> tokio::task::JoinHandle + where + F: std::future::Future + Send + 'static, + F::Output: Send + 'static, + { + self.jobs.spawn(job) + } + + /// Jobs spawned and not yet ended. + #[must_use] + pub fn running_jobs(&self) -> usize { + self.jobs.len() + } + + /// Cancel the jobs' shutdown token and wait up to `bound` for every job + /// to end: a cancelled job runs its failure path (the FAILED write and + /// the deletion of its prefix), which is what sweeps an upload still in + /// flight on S3, where the runtime's own shutdown sends no abort. Jobs + /// still running at the bound are logged and left to the runtime's + /// shutdown; their rows are CREATING (a job still in its pipeline) or + /// FAILED (one still in its `delete_prefix`), and the reconciler resolves + /// or sweeps them once their heartbeat is stale. Returns the number + /// still running, for the caller's information: the server reads the + /// warn line and nothing else. + pub async fn drain_jobs(&self, bound: Duration) -> usize { + self.shutdown.cancel(); + self.jobs.close(); + let running = self.jobs.len(); + if running == 0 { + return 0; + } + tracing::info!( + "waiting up to {} s for {running} backup job(s) to end", + bound.as_secs() + ); + if tokio::time::timeout(bound, self.jobs.wait()).await.is_err() { + let left = self.jobs.len(); + tracing::warn!( + "{left} backup job(s) did not end within {} s; their rows are CREATING or FAILED \ + and the reconciler resolves or sweeps them once their heartbeat is stale", + bound.as_secs() + ); + return left; + } + 0 } /// The same runtime with `store` configured. diff --git a/crates/engine/src/backup/tests.rs b/crates/engine/src/backup/tests.rs index e77f2f80e..6d47cd5dc 100644 --- a/crates/engine/src/backup/tests.rs +++ b/crates/engine/src/backup/tests.rs @@ -14,21 +14,22 @@ use extenddb_core::limits::LimitsConfig; use extenddb_core::types::{ AttributeDefinition, BackupDescription, BackupDetails, BackupSummary, BillingMode, BillingModeSummary, ContinuousBackupsDescription, CreateTableInput, DeleteTableInput, - DescribeTableInput, GsiDescription, GsiInput, IndexInfo, KeySchemaElement, KeyType, + DescribeTableInput, GsiDescription, GsiInput, IndexInfo, Item, KeySchemaElement, KeyType, ListTablesInput, ListTablesOutput, LsiInput, OnDemandThroughput, Projection, ProjectionType, ProvisionedThroughput, ProvisionedThroughputDescription, RestoreSummary, ScalarAttributeType, SourceTableDetails, StreamSpecification, StreamViewType, TableDescription, TableKeyInfo, TableStatus, TimeToLiveDescription, TimeToLiveStatus, UpdateTableInput, }; use extenddb_storage::backup_catalog::{ - AbandonedLegacyBackup, BackupKey, BackupRow, BackupStatus, DeleteStart, JobClaim, - RestoreJobRow, RestoreTargetKey, StoreIdentity, StoreKind, TimeBounds, + AbandonedLegacyBackup, BackupKey, BackupRow, BackupStatus, DeleteStart, Heartbeat, JobClaim, + JobRow, JobStatus, RestoreJobRow, RestoreTargetKey, StaleClock, StatusFields, StoreIdentity, + StoreKind, TimeBounds, WriteOutcome, new_generation, }; use extenddb_storage::backup_definition::{BACKUP_DEFINITION_VERSION, BackupTableDefinition}; use extenddb_storage::error::StorageError; use extenddb_storage::{ - BackupCatalog, BackupRestoreSource, BoxedFuture, JobPermit, LegacyBackupReader, RestoreRequest, - TableEngine, + BackupCatalog, BackupRestoreSource, BoxedFuture, JobPermit, LegacyBackupReader, MetadataEngine, + RestoreRequest, SnapshotEngine, TableEngine, TableSnapshot, }; use futures::TryStreamExt; use futures::stream::BoxStream; @@ -67,8 +68,8 @@ struct Fake { listed_for: Mutex>>, /// When set, `restore_source` waits for a permit before answering. source_gate: Option>, - /// The catalog index: rows by ARN. - rows: Mutex>, + /// The catalog index: rows by ARN, shared with the claims handed out. + rows: Arc>>, /// `backup_in_use` answers: the CREATING target's name, by backup ARN. in_use: Mutex>, /// What `start_backup_delete` answers after the row check, by ARN. @@ -93,6 +94,9 @@ struct Fake { maintenance_checks: Mutex>, /// When set, `check_maintenance_claim` fails with it (every call). maintenance_error: Mutex>, + /// When set, `check_maintenance_claim` takes this long to answer (a + /// catalog query held behind the pool). + maintenance_delay: Mutex>, /// The `TimeBounds` `list_account_backup_rows` received. bounds_seen: Mutex>, /// The account's tables, by name, as `describe_table` reports them; a @@ -108,8 +112,266 @@ struct Fake { set_deleted: Mutex>, /// Rows `remove_backup_row` removed, by ARN. removed_rows: Mutex>, + /// Every `remove_backup_row` call, by ARN, refused ones included. + remove_attempts: Mutex>, /// When set, `list_backup_rows` fails with it. list_error: Mutex>, + /// The table snapshots `open_table_snapshot` serves, by table name; a + /// name absent is `TableNotFound`. + snapshots: Mutex>, + /// When set, `open_table_snapshot` fails with it (once). + snapshot_error: Mutex>, + /// Snapshots opened, by table name. + snapshots_opened: Mutex>, + /// What `describe_ttl` answers; disabled when unset. + ttl: Mutex>, + /// The rows `claim_backup` received, as given. + claims: Mutex>, + /// The ARNs `claim_dead_backup` hands over (the backend's liveness test + /// modelled as a set): a CREATING row named here has a dead owner. + dead: Mutex>, + /// The ARNs `claim_dead_backup` took over. + takeovers: Mutex>, + /// Every `set_status` a claim received, in order. + status_writes: Arc>>, + /// What a claim's `heartbeat` does. + heartbeat_answer: Arc>, + /// Heartbeats the claims received. + heartbeats: Arc, + /// When set, the next `set_status` reports `Unknown` (applied or not as + /// the flag says) instead of its outcome; taken by that write unless + /// `unknown_write_sticky` is set (a connection that fails on every + /// write). + unknown_write: Arc>>, + unknown_write_sticky: Arc, + /// When set, moved into `unknown_write` once a (non-sticky) unknown + /// write has been taken: the write sent after an unknown outcome is + /// itself unknown, in this way. + unknown_write_then: Arc>>, + /// When set, every `set_status` fails as a claim connection that was + /// reset. + status_write_error: Arc, + /// When set, `claim_dead_backup` hands the claim over and then gives + /// the row a newer generation: another reconciler took the row over in + /// the gap before this one's write (item 29's two-reconciler race). + takeover_lost_to_another: std::sync::atomic::AtomicBool, + /// What `sweep_restore_registrations` answers. + swept_registrations: Mutex>, +} + +/// A table snapshot the `Fake` serves. +struct SnapshotFixture { + description: TableDescription, + key_ordered: bool, + items: Vec>, + /// When set, the stream takes one permit per item before yielding it, + /// so a test can hold the job inside the stream. + gate: Option>, +} + +/// One `set_status` a claim received. +#[derive(Debug, Clone)] +struct StatusWrite { + arn: String, + generation: u64, + from: JobStatus, + to: JobStatus, + fields: StatusFields, +} + +/// What a `Fake` claim's heartbeat does. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +enum HeartbeatAnswer { + /// Confirm while the row carries the claim's generation, whatever its + /// status (the backends' `UPDATE ... WHERE owner_generation = ?` has no + /// status clause, so a job's beats during its own `delete_prefix` + /// refresh the FAILED row), refreshing `heartbeat_at`; `Lost` otherwise. + #[default] + Confirm, + /// Confirm whatever the row carries (a connection whose session still + /// holds its lock while the row was rewritten). + AlwaysConfirm, + /// `Lost`, whatever the row. + Lost, + /// Never return. + Stall, + /// An error. + Error, +} + +/// How the next `set_status` reports `Unknown`. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum UnknownWrite { + /// The write landed and the connection failed after it. + Applied, + /// The write was lost with the connection. + Lost, +} + +/// A `Fake` claim on one row. +struct FakeClaim { + rows: Arc>>, + arn: String, + generation: u64, + status_writes: Arc>>, + heartbeat_answer: Arc>, + heartbeats: Arc, + unknown_write: Arc>>, + unknown_write_sticky: Arc, + unknown_write_then: Arc>>, + status_write_error: Arc, +} + +impl Fake { + fn claim_on(&self, arn: &str, generation: u64) -> Box { + Box::new(FakeClaim { + rows: Arc::clone(&self.rows), + arn: arn.to_owned(), + generation, + status_writes: Arc::clone(&self.status_writes), + heartbeat_answer: Arc::clone(&self.heartbeat_answer), + heartbeats: Arc::clone(&self.heartbeats), + unknown_write: Arc::clone(&self.unknown_write), + unknown_write_sticky: Arc::clone(&self.unknown_write_sticky), + unknown_write_then: Arc::clone(&self.unknown_write_then), + status_write_error: Arc::clone(&self.status_write_error), + }) + } +} + +/// `JobStatus` as a backup row carries it. +fn row_status(status: JobStatus) -> Option { + match status { + JobStatus::Creating => Some(BackupStatus::Creating), + JobStatus::Available => Some(BackupStatus::Available), + JobStatus::Failed => Some(BackupStatus::Failed), + JobStatus::Running | JobStatus::Done => None, + } +} + +/// `BackupStatus` as a job record reports it. +fn job_status(status: BackupStatus) -> JobStatus { + match status { + BackupStatus::Creating => JobStatus::Creating, + BackupStatus::Available => JobStatus::Available, + BackupStatus::Failed => JobStatus::Failed, + BackupStatus::Deleting | BackupStatus::Deleted => JobStatus::Done, + } +} + +impl JobClaim for FakeClaim { + fn generation(&self) -> u64 { + self.generation + } + + fn heartbeat(&self) -> BoxedFuture<'_, Result> { + self.heartbeats + .fetch_add(1, std::sync::atomic::Ordering::AcqRel); + let answer = *self.heartbeat_answer.lock().unwrap(); + let confirmed = match self.rows.lock().unwrap().get_mut(&self.arn) { + Some(row) + if answer == HeartbeatAnswer::Confirm + && row.owner_generation == self.generation => + { + row.heartbeat_at = seconds_since_epoch(); + true + } + _ => false, + }; + Box::pin(async move { + match answer { + HeartbeatAnswer::Confirm if confirmed => Ok(Heartbeat::Confirmed), + HeartbeatAnswer::AlwaysConfirm => Ok(Heartbeat::Confirmed), + HeartbeatAnswer::Confirm | HeartbeatAnswer::Lost => Ok(Heartbeat::Lost), + HeartbeatAnswer::Stall => futures::future::pending().await, + HeartbeatAnswer::Error => Err(StorageError::Internal( + "the claim connection failed".to_owned(), + )), + } + }) + } + + /// The backends' write: conditioned on the claim's generation and on + /// the row being in `from`; `completed` stamps `completed_at`. + fn set_status<'a>( + &'a self, + from: JobStatus, + to: JobStatus, + fields: &'a StatusFields, + ) -> BoxedFuture<'a, Result> { + self.status_writes.lock().unwrap().push(StatusWrite { + arn: self.arn.clone(), + generation: self.generation, + from, + to, + fields: fields.clone(), + }); + if self + .status_write_error + .load(std::sync::atomic::Ordering::Acquire) + { + return Box::pin(async move { + Err(StorageError::Internal( + "claim session: connection reset".to_owned(), + )) + }); + } + let unknown = if self + .unknown_write_sticky + .load(std::sync::atomic::Ordering::Acquire) + { + *self.unknown_write.lock().unwrap() + } else { + let taken = self.unknown_write.lock().unwrap().take(); + if taken.is_some() { + *self.unknown_write.lock().unwrap() = + self.unknown_write_then.lock().unwrap().take(); + } + taken + }; + let apply = unknown != Some(UnknownWrite::Lost); + let mut outcome = WriteOutcome::NotMatched; + if apply + && let Some(row) = self.rows.lock().unwrap().get_mut(&self.arn) + && row.owner_generation == self.generation + && Some(row.status) == row_status(from) + && let Some(to_status) = row_status(to) + { + row.status = to_status; + if let Some(size) = fields.size_bytes { + row.size_bytes = size; + } + if let Some(count) = fields.item_count { + row.item_count = count; + } + if let Some(reason) = &fields.failure_reason { + row.failure_reason = Some(reason.clone()); + } + if let Some(details) = &fields.source_table_details_json { + row.source_table_details_json = details.clone(); + } + if fields.completed { + row.completed_at = Some(seconds_since_epoch()); + } + outcome = WriteOutcome::Matched; + } + if unknown.is_some() { + outcome = WriteOutcome::Unknown; + } + Box::pin(async move { Ok(outcome) }) + } + + fn read(&self) -> BoxedFuture<'_, Result> { + let found = self.rows.lock().unwrap().get(&self.arn).map(|row| JobRow { + status: job_status(row.status), + owner_generation: row.owner_generation, + heartbeat_at: row.heartbeat_at, + failure_reason: row.failure_reason.clone(), + }); + Box::pin(async move { + found.ok_or_else(|| StorageError::Internal("the row is gone".to_owned())) + }) + } } impl Fake { @@ -164,6 +426,19 @@ struct MemStore { stalls: std::sync::atomic::AtomicUsize, deletes: Mutex>, puts: Mutex>, + /// When set, `put` fails as a transport error after reading its body; + /// every other call answers. + fail_puts: std::sync::atomic::AtomicBool, + /// When set, `get` of a key under this prefix fails as a transport + /// error; every other call answers (the identity file's read among + /// them, so the store stays healthy). + fail_gets_under: Mutex>, + /// Called with the key of every `put` that stored its object, so a test + /// can act at a point of the job (after the sidecar's put, before the + /// claim re-check). + on_put: Mutex>, + /// Bytes `put` stored, by key, in the order the puts ended. + put_sizes: Mutex>, /// When set, the bytes `put_if_absent` leaves under the key right after /// it stored its own: a server that ignores `If-None-Match` and a second /// writer that won the plain-put race. @@ -172,8 +447,14 @@ struct MemStore { /// and `head` find no object: something that is not an object (a FIFO, /// a socket) sits at the path. key_taken_by_a_non_object: std::sync::atomic::AtomicBool, + /// When set, `delete_prefix` takes this long before it answers (a slow + /// store's deletion). + delete_delay: Mutex>, } +/// What `MemStore::on_put` runs after a stored `put`, with the object's key. +type OnPut = Box; + /// A transport error with the SDK's shape: a top link naming only the class /// and the cause one link below it. #[derive(Debug)] @@ -266,9 +547,18 @@ impl extenddb_backup_store::BackupStore for MemStore { self.stall().await; self.check()?; let bytes = collect_body(body).await?; + if self.fail_puts.load(std::sync::atomic::Ordering::Acquire) { + return Err(extenddb_backup_store::StoreError::Transport { + source: Box::new(DispatchFailure(ConnectError("put refused".to_owned()))), + }); + } self.puts.lock().unwrap().push(key.to_owned()); let size = bytes.len() as u64; + self.put_sizes.lock().unwrap().push((key.to_owned(), size)); self.insert(key, &bytes); + if let Some(on_put) = self.on_put.lock().unwrap().as_ref() { + on_put(key); + } Ok(extenddb_backup_store::ObjectMeta { key: key.to_owned(), size, @@ -323,6 +613,17 @@ impl extenddb_backup_store::BackupStore for MemStore { Box::pin(async move { self.stall().await; self.check()?; + if self + .fail_gets_under + .lock() + .unwrap() + .as_deref() + .is_some_and(|prefix| key.starts_with(prefix)) + { + return Err(extenddb_backup_store::StoreError::Transport { + source: Box::new(DispatchFailure(ConnectError("get refused".to_owned()))), + }); + } let bytes = self .objects .lock() @@ -388,6 +689,10 @@ impl extenddb_backup_store::BackupStore for MemStore { Box::pin(async move { self.stall().await; self.check()?; + let delay = *self.delete_delay.lock().unwrap(); + if let Some(delay) = delay { + tokio::time::sleep(delay).await; + } self.deletes.lock().unwrap().push(prefix.to_owned()); let mut objects = self.objects.lock().unwrap(); let before = objects.len(); @@ -480,11 +785,92 @@ impl BackupCatalog for Fake { Box::pin(async move { Ok(id) }) } + /// The backends' insert: the row as given, CREATING under a fresh + /// generation with `heartbeat_at` now; `AlreadyExists` when the ARN has a + /// row. fn claim_backup<'a>( &'a self, - _row: &'a BackupRow, + row: &'a BackupRow, ) -> BoxedFuture<'a, Result, StorageError>> { - unused() + self.claims.lock().unwrap().push(row.clone()); + let mut rows = self.rows.lock().unwrap(); + if rows.contains_key(&row.backup_arn) { + let arn = row.backup_arn.clone(); + return Box::pin(async move { Err(StorageError::AlreadyExists(arn)) }); + } + let generation = new_generation(); + let mut inserted = row.clone(); + inserted.status = BackupStatus::Creating; + inserted.owner_generation = generation; + inserted.heartbeat_at = seconds_since_epoch(); + rows.insert(row.backup_arn.clone(), inserted); + drop(rows); + let claim = self.claim_on(&row.backup_arn, generation); + Box::pin(async move { Ok(claim) }) + } + + /// The takeover: a CREATING row whose ARN is in the `dead` set gets a + /// fresh generation, `heartbeat_at` untouched; anything else is `None`. + fn claim_dead_backup<'a>( + &'a self, + key: &'a BackupKey, + _stale_after: Duration, + ) -> BoxedFuture<'a, Result>, StorageError>> { + let dead = self.dead.lock().unwrap().contains(&key.backup_arn); + let mut rows = self.rows.lock().unwrap(); + let claim = match rows.get_mut(&key.backup_arn) { + Some(row) if dead && row.status == BackupStatus::Creating => { + row.owner_generation = new_generation(); + let generation = row.owner_generation; + if self + .takeover_lost_to_another + .load(std::sync::atomic::Ordering::Acquire) + { + row.owner_generation = new_generation(); + } + drop(rows); + self.takeovers.lock().unwrap().push(key.backup_arn.clone()); + Some(self.claim_on(&key.backup_arn, generation)) + } + _ => None, + }; + Box::pin(async move { Ok(claim) }) + } + + /// The backends' listing on the named clock, with this process's clock + /// standing for the catalog's: rows of the store with `status` whose + /// `heartbeat_at`, or `completed_at` when set, is older than `age`. + fn list_backup_rows_older_than<'a>( + &'a self, + status: BackupStatus, + store: &'a StoreIdentity, + clock: StaleClock, + age: Duration, + ) -> BoxStream<'a, Result> { + let cutoff = seconds_since_epoch() - age.as_secs_f64(); + let mut rows: Vec = self + .rows + .lock() + .unwrap() + .values() + .filter(|r| r.store == *store && r.status == status) + .filter(|r| match clock { + StaleClock::Heartbeat => r.heartbeat_at < cutoff, + StaleClock::Completed => r.completed_at.is_some_and(|c| c < cutoff), + }) + .cloned() + .collect(); + rows.sort_by(|a, b| { + a.created_at + .total_cmp(&b.created_at) + .then_with(|| a.backup_arn.cmp(&b.backup_arn)) + }); + Box::pin(futures::stream::iter(rows.into_iter().map(Ok))) + } + + fn sweep_restore_registrations(&self) -> BoxedFuture<'_, Result, StorageError>> { + let swept = self.swept_registrations.lock().unwrap().clone(); + Box::pin(async move { Ok(swept) }) } /// The backend's listing: rows of the store (kind, root and id), of the @@ -612,6 +998,10 @@ impl BackupCatalog for Fake { &'a self, key: &'a BackupKey, ) -> BoxedFuture<'a, Result> { + self.remove_attempts + .lock() + .unwrap() + .push(key.backup_arn.clone()); let mut rows = self.rows.lock().unwrap(); let removed = match rows.get(&key.backup_arn) { Some(row) if matches!(row.status, BackupStatus::Deleted | BackupStatus::Failed) => { @@ -725,7 +1115,11 @@ impl BackupCatalog for Fake { self.maintenance_checks.lock().unwrap().push(stale_after); let held = *self.maintenance_held.lock().unwrap(); let error = self.maintenance_error.lock().unwrap().clone(); + let delay = *self.maintenance_delay.lock().unwrap(); Box::pin(async move { + if let Some(delay) = delay { + tokio::time::sleep(delay).await; + } match error { Some(e) => Err(StorageError::Internal(e)), None => Ok(held), @@ -734,6 +1128,168 @@ impl BackupCatalog for Fake { } } +impl SnapshotEngine for Fake { + /// The fixture for the table, or `TableNotFound`; the description and + /// the items are the fixture's, in its order, through the gate when it + /// has one. + fn open_table_snapshot( + &self, + _account_id: &str, + table_name: &str, + ) -> BoxedFuture<'_, Result> { + self.snapshots_opened + .lock() + .unwrap() + .push(table_name.to_owned()); + if let Some(e) = self.snapshot_error.lock().unwrap().take() { + return Box::pin(async move { Err(e) }); + } + let mut snapshots = self.snapshots.lock().unwrap(); + let Some(fixture) = snapshots.get_mut(table_name) else { + let name = table_name.to_owned(); + return Box::pin(async move { + Err(StorageError::TableNotFound(format!( + "Table not found: {name}" + ))) + }); + }; + let items = std::mem::take(&mut fixture.items); + let gate = fixture.gate.clone(); + let snapshot = TableSnapshot { + description: fixture.description.clone(), + info: extenddb_core::backup_format::SnapshotDescriptor { + kind: "test".to_owned(), + marker: String::new(), + }, + key_ordered: fixture.key_ordered, + items: Box::pin(futures::stream::unfold( + (items.into_iter(), gate), + |(mut items, gate)| async move { + let next = items.next()?; + if let Some(gate) = &gate { + gate.acquire() + .await + .expect("the gate is never closed") + .forget(); + } + Some((next, (items, gate))) + }, + )), + }; + Box::pin(async move { Ok(snapshot) }) + } +} + +impl MetadataEngine for Fake { + fn describe_ttl( + &self, + _account_id: &str, + _table_name: &str, + ) -> BoxedFuture<'_, Result> { + let ttl = self + .ttl + .lock() + .unwrap() + .clone() + .unwrap_or(TimeToLiveDescription { + time_to_live_status: TimeToLiveStatus::Disabled, + attribute_name: None, + }); + Box::pin(async move { Ok(ttl) }) + } + + fn update_ttl( + &self, + _: &str, + _: &str, + _: &str, + _: bool, + ) -> BoxedFuture<'_, Result<(), StorageError>> { + unused() + } + + fn tag_resource( + &self, + _: &str, + _: &[extenddb_core::types::Tag], + ) -> BoxedFuture<'_, Result<(), StorageError>> { + unused() + } + + fn untag_resource(&self, _: &str, _: &[String]) -> BoxedFuture<'_, Result<(), StorageError>> { + unused() + } + + fn list_tags( + &self, + _: &str, + ) -> BoxedFuture<'_, Result, StorageError>> { + unused() + } + + fn tables_with_ttl( + &self, + _: &str, + ) -> BoxedFuture<'_, Result, StorageError>> { + unused() + } + + fn all_tables_with_ttl( + &self, + ) -> BoxedFuture<'_, Result, StorageError>> { + unused() + } + + fn all_tables_with_ttl_index_ready( + &self, + ) -> BoxedFuture<'_, Result, StorageError>> { + unused() + } + + fn create_ttl_index( + &self, + _: &str, + _: &str, + _: &str, + ) -> BoxedFuture<'_, Result<(), StorageError>> { + unused() + } + + fn drop_ttl_index( + &self, + _: &str, + _: &str, + _: &str, + ) -> BoxedFuture<'_, Result<(), StorageError>> { + unused() + } + + fn find_expired_items_indexed( + &self, + _: &str, + _: &str, + _: &str, + _: usize, + ) -> BoxedFuture<'_, Result, StorageError>> { + unused() + } + + fn refresh_table_size(&self, _: &str, _: &str) -> BoxedFuture<'_, Result<(), StorageError>> { + unused() + } + + fn list_active_table_names( + &self, + _: &str, + ) -> BoxedFuture<'_, Result, StorageError>> { + unused() + } + + fn all_active_tables(&self) -> BoxedFuture<'_, Result, StorageError>> { + unused() + } +} + impl TableEngine for Fake { fn create_table( &self, @@ -4941,13 +5497,14 @@ async fn create_backup_checks_the_table_then_takes_the_permit_last() { }; let full = BackupRuntime::new(1); let _held = full.take_permit().unwrap(); - let fake = Fake::default(); + let fake = Arc::new(Fake::default()); let err = create_backup( &fake, &full, ACCOUNT, REGION, + &LimitsConfig::default(), "Music", "nightly", Err(StorageError::TableNotFound("Music".to_owned())), @@ -4967,6 +5524,7 @@ async fn create_backup_checks_the_table_then_takes_the_permit_last() { &full, ACCOUNT, REGION, + &LimitsConfig::default(), "Music", "nightly", Ok(creating), @@ -5009,6 +5567,7 @@ async fn create_backup_checks_the_table_then_takes_the_permit_last() { &full, ACCOUNT, REGION, + &LimitsConfig::default(), "Music", "nightly", Ok(with_vectors), @@ -5023,6 +5582,7 @@ async fn create_backup_checks_the_table_then_takes_the_permit_last() { &full, ACCOUNT, REGION, + &LimitsConfig::default(), "Music", "nightly", Ok(active.clone()), @@ -5044,6 +5604,7 @@ async fn create_backup_checks_the_table_then_takes_the_permit_last() { &free, ACCOUNT, REGION, + &LimitsConfig::default(), "Music", "nightly", Ok(active), @@ -5066,7 +5627,7 @@ async fn create_backup_checks_the_table_then_takes_the_permit_last() { async fn create_backup_refuses_a_name_no_backup_arn_can_carry() { let runtime = BackupRuntime::new(1); let _held = runtime.take_permit().unwrap(); - let fake = Fake::default(); + let fake = Arc::new(Fake::default()); // 256 bytes, and 1024 bytes: a name the handler admits under a raised // `limits.max_table_name_length` that takes the ARN itself over its // 1024-byte bound, so the limit must be named before the ARN is formed. @@ -5078,9 +5639,18 @@ async fn create_backup_refuses_a_name_no_backup_arn_can_carry() { ..Default::default() }; let msg = validation_message( - create_backup(&fake, &runtime, ACCOUNT, REGION, &long, "nightly", Ok(desc)) - .await - .unwrap_err(), + create_backup( + &fake, + &runtime, + ACCOUNT, + REGION, + &LimitsConfig::default(), + &long, + "nightly", + Ok(desc), + ) + .await + .unwrap_err(), ); assert!( msg.contains(&format!("{len} bytes")) && msg.contains("255"), @@ -5095,9 +5665,18 @@ async fn create_backup_refuses_a_name_no_backup_arn_can_carry() { ..Default::default() }; let msg = validation_message( - create_backup(&fake, &runtime, ACCOUNT, REGION, dots, "nightly", Ok(desc)) - .await - .unwrap_err(), + create_backup( + &fake, + &runtime, + ACCOUNT, + REGION, + &LimitsConfig::default(), + dots, + "nightly", + Ok(desc), + ) + .await + .unwrap_err(), ); assert!( msg.contains("TableName") && msg.contains("only dots"), @@ -5119,7 +5698,7 @@ async fn create_backup_refuses_a_name_no_backup_arn_can_carry() { async fn create_backup_under_a_region_outside_the_arn_alphabet_names_the_setting() { let runtime = BackupRuntime::new(1); let _held = runtime.take_permit().unwrap(); - let fake = Fake::default(); + let fake = Arc::new(Fake::default()); let desc = TableDescription { table_name: "Music".to_owned(), table_status: TableStatus::Active, @@ -5131,6 +5710,7 @@ async fn create_backup_under_a_region_outside_the_arn_alphabet_names_the_setting &runtime, ACCOUNT, "LOCAL", + &LimitsConfig::default(), "Music", "nightly", Ok(desc), @@ -5157,7 +5737,7 @@ async fn create_backup_under_a_region_outside_the_arn_alphabet_names_the_setting .unwrap() .insert(stored.clone(), row(&stored, BackupStatus::Available)); let spellings = backup_arn_field(&json!({ "BackupArn": stored }), ACCOUNT, "LOCAL").unwrap(); - let desc = describe_backup(&fake, ACCOUNT, "LOCAL", &spellings) + let desc = describe_backup(&*fake, ACCOUNT, "LOCAL", &spellings) .await .unwrap(); assert_eq!(desc.backup_details.backup_arn, stored); @@ -5185,7 +5765,7 @@ async fn create_backup_under_a_region_outside_the_arn_alphabet_names_the_setting /// the claim is gone both run. #[tokio::test] async fn jobs_are_refused_while_a_maintenance_claim_is_held() { - let fake = Fake::with_source(source(Some(definition("PROVISIONED")))); + let fake = Arc::new(Fake::with_source(source(Some(definition("PROVISIONED"))))); *fake.maintenance_held.lock().unwrap() = true; let runtime = BackupRuntime::new(1).with_stale_job_after(Duration::from_secs(7)); let err = create_backup( @@ -5193,6 +5773,7 @@ async fn jobs_are_refused_while_a_maintenance_claim_is_held() { &runtime, ACCOUNT, REGION, + &LimitsConfig::default(), "Music", "nightly", Ok(active_table("Music")), @@ -5210,6 +5791,7 @@ async fn jobs_are_refused_while_a_maintenance_claim_is_held() { &runtime, ACCOUNT, REGION, + &LimitsConfig::default(), "Music", "nightly", Err(StorageError::TableNotFound("Music".to_owned())), @@ -5242,6 +5824,7 @@ async fn jobs_are_refused_while_a_maintenance_claim_is_held() { &runtime, ACCOUNT, REGION, + &LimitsConfig::default(), "Music", "nightly", Ok(active_table("Music")), @@ -5267,6 +5850,7 @@ async fn jobs_are_refused_while_a_maintenance_claim_is_held() { &full, ACCOUNT, REGION, + &LimitsConfig::default(), "Music", "nightly", Ok(active_table("Music")), @@ -6905,18 +7489,17 @@ async fn a_store_checked_after_startup_is_checking_until_the_first_check() { assert_eq!(runtime.store_health().unwrap().status, "healthy"); } -/// The refusals while the configured store is unhealthy: DeleteBackup of a -/// row of the store, and the restore of such a row, each a -/// ValidationException naming the store's kind and the problem's category -/// and never the root; the legacy reader is not reached, and the restore's -/// refusal comes before the permit (with no slot free it is still the -/// store's, not LimitExceeded). CreateBackup is not refused: the in-catalog -/// create, the one this release runs, touches no store, so it runs with an -/// unhealthy store as it does with none (a server that never configured -/// backups keeps CreateBackup through a failed check of the default root), -/// and on a backend bound to the in-catalog create. +/// An unhealthy store refuses the operations that touch it, before a slot is +/// taken and naming the store's kind and the problem's category and never +/// its root or the cause: CreateBackup (store-backed on this runtime), +/// DeleteBackup of a row of the store, and the restore of one (with no slot +/// free it is still the store's refusal, not LimitExceeded). A backend +/// bound to the in-catalog create is not refused: its create touches no +/// store. A row of another store is refused as another store's, whatever +/// the configured one's health. #[tokio::test] -async fn an_unhealthy_store_refuses_delete_and_restore_but_not_the_in_catalog_create() { +async fn an_unhealthy_store_refuses_create_delete_and_restore_unless_bound_to_the_in_catalog_create() + { let mem = Arc::new(MemStore::default()); let unhealthy = || { runtime_unhealthy( @@ -6934,62 +7517,56 @@ async fn an_unhealthy_store_refuses_delete_and_restore_but_not_the_in_catalog_cr assert!(!m.contains("connect"), "{m}"); }; - // CreateBackup runs the in-catalog create, taking its slot. - let fake = Fake::default(); - for runtime in [unhealthy(), unhealthy().with_legacy_create_only(true)] { - let before = fake.creates.lock().unwrap().len(); - create_backup( - &fake, - &runtime, - ACCOUNT, - REGION, - "Music", - "nightly", - Ok(active_table("Music")), - ) - .await - .unwrap(); - assert_eq!(fake.creates.lock().unwrap().len(), before + 1); - assert_eq!(runtime.free_slots(), 3, "the create holds its slot"); - fake.permits.lock().unwrap().clear(); - } - // With no slot free the create is LimitExceeded, the store's health - // notwithstanding. - let full = BackupRuntime::new(0).with_store(ConfiguredStore::new( - store().store_kind, - store().store_root, - Arc::clone(&mem) as Arc, - None, - Err(StoreProblem::Transport("refused".to_owned())), - )); + // The store-backed create is refused before the permit and the open. + let fake = Arc::new(Fake::default()); + let runtime = unhealthy(); let err = create_backup( &fake, - &full, + &runtime, ACCOUNT, REGION, + &LimitsConfig::default(), "Music", "nightly", Ok(active_table("Music")), ) .await .unwrap_err(); - assert!( - matches!(err, DynamoDbError::LimitExceededException(_)), - "{err:?}" - ); - - // DeleteBackup of a row of the store. - let fake = fake_with_row(BackupStatus::Available); - let err = delete_backup(&fake, &unhealthy(), ACCOUNT, REGION, &own(&arn(ACCOUNT))) - .await - .unwrap_err(); check(err); - assert!( - fake.delete_starts_seen.lock().unwrap().is_empty(), - "no DELETING write" - ); + assert_eq!(runtime.free_slots(), 4, "no slot was taken"); + assert!(fake.snapshots_opened.lock().unwrap().is_empty()); + assert!(fake.creates.lock().unwrap().is_empty()); - // The restore of a row of the store, with no slot free: the refusal is + // A backend bound to the in-catalog create runs it, taking its slot. + let runtime = unhealthy().with_legacy_create_only(true); + create_backup( + &fake, + &runtime, + ACCOUNT, + REGION, + &LimitsConfig::default(), + "Music", + "nightly", + Ok(active_table("Music")), + ) + .await + .unwrap(); + assert_eq!(fake.creates.lock().unwrap().len(), 1); + assert_eq!(runtime.free_slots(), 3, "the create holds its slot"); + fake.permits.lock().unwrap().clear(); + + // DeleteBackup of a row of the store. + let fake = Arc::new(fake_with_row(BackupStatus::Available)); + let err = delete_backup(&*fake, &unhealthy(), ACCOUNT, REGION, &own(&arn(ACCOUNT))) + .await + .unwrap_err(); + check(err); + assert!( + fake.delete_starts_seen.lock().unwrap().is_empty(), + "no DELETING write" + ); + + // The restore of a row of the store, with no slot free: the refusal is // the store's, so it came before the permit. let full = BackupRuntime::new(0).with_store(ConfiguredStore::new( store().store_kind, @@ -7010,9 +7587,9 @@ async fn an_unhealthy_store_refuses_delete_and_restore_but_not_the_in_catalog_cr // of the configured one. let mut other = row(&arn(ACCOUNT), BackupStatus::Available); other.store.store_root = "/elsewhere".to_owned(); - let fake = Fake::default(); + let fake = Arc::new(Fake::default()); fake.rows.lock().unwrap().insert(arn(ACCOUNT), other); - let err = delete_backup(&fake, &unhealthy(), ACCOUNT, REGION, &own(&arn(ACCOUNT))) + let err = delete_backup(&*fake, &unhealthy(), ACCOUNT, REGION, &own(&arn(ACCOUNT))) .await .unwrap_err(); assert!( @@ -7021,25 +7598,43 @@ async fn an_unhealthy_store_refuses_delete_and_restore_but_not_the_in_catalog_cr ); } -/// With a healthy store configured, CreateBackup takes the in-catalog path -/// (this release has no store-backed create) and logs that the store is -/// configured; the slot is taken and the reader's create runs. +/// With a healthy store configured, CreateBackup is store-backed: the +/// legacy reader is not asked, the table's snapshot is opened and the answer +/// is CREATING; a table with vector indexes is backed up on this path (the +/// refusal belongs to the in-catalog create alone). #[tokio::test] -async fn a_healthy_store_leaves_create_backup_on_the_in_catalog_path() { - let fake = Fake::default(); - let runtime = runtime_with_store(); - create_backup( +async fn a_healthy_store_makes_create_backup_store_backed() { + let mem = Arc::new(MemStore::default()); + let fake = Arc::new(Fake::default()); + let mut table = music_table(); + table.vector_indexes = Some(vec![vector_index()]); + snapshot_fixture(&fake, &table, Vec::new(), true, None); + let runtime = job_runtime(&mem); + let details = create_backup( &fake, &runtime, ACCOUNT, REGION, + &LimitsConfig::default(), "Music", "nightly", - Ok(active_table("Music")), + Ok(table), ) .await .unwrap(); - assert_eq!(fake.creates.lock().unwrap().len(), 1); + assert_eq!(details.backup_status, "CREATING"); + assert!( + fake.creates.lock().unwrap().is_empty(), + "the legacy reader was not asked" + ); + assert_eq!(fake.snapshots_opened.lock().unwrap().as_slice(), ["Music"]); + wait_for_jobs(&runtime).await; + let row = fake.rows.lock().unwrap()[&details.backup_arn].clone(); + assert_eq!(row.status, BackupStatus::Available); + assert_eq!( + row.source_table_details_json["vector_indexes"][0]["IndexName"], + "v" + ); } /// DeleteBackup of a row of the configured store deletes the manifest first @@ -7279,3 +7874,2303 @@ async fn a_prefix_that_still_lists_an_object_is_left_deleting() { BackupStatus::Deleting ); } + +// --- the store-backed create and its job ------------------------------------ + +/// The `Music` table as a snapshot describes it: composite key, one GSI, +/// provisioned. +fn music_table() -> TableDescription { + TableDescription { + table_name: "Music".to_owned(), + key_schema: vec![ + key("Artist", KeyType::Hash), + key("SongTitle", KeyType::Range), + ], + attribute_definitions: vec![attr("Artist"), attr("SongTitle"), attr("Genre")], + table_status: TableStatus::Active, + creation_date_time: 1_757_400_000.0, + table_size_bytes: 123_456, + item_count: 100_000, + table_arn: source_arn(), + table_id: "12345a12-abcd-123a-ab12-1234abc12345".to_owned(), + provisioned_throughput: ProvisionedThroughputDescription { + read_capacity_units: 5, + write_capacity_units: 5, + ..Default::default() + }, + billing_mode_summary: Some(BillingModeSummary { + billing_mode: BillingMode::Provisioned, + last_update_to_pay_per_request_date_time: None, + }), + global_secondary_indexes: Some(vec![GsiDescription { + index_name: "GenreIndex".to_owned(), + key_schema: vec![key("Genre", KeyType::Hash)], + projection: Projection { + projection_type: ProjectionType::All, + non_key_attributes: None, + }, + index_status: "ACTIVE".to_owned(), + provisioned_throughput: Some(ProvisionedThroughputDescription { + read_capacity_units: 1, + write_capacity_units: 1, + ..Default::default() + }), + index_size_bytes: 0, + item_count: 0, + index_arn: String::new(), + }]), + ..Default::default() + } +} + +/// A vector index description for a table with vector support. +fn vector_index() -> extenddb_core::types::VectorIndexDescription { + serde_json::from_value(json!({ + "IndexName": "v", + "VectorAttribute": { "AttributeName": "emb" }, + "Dimensions": 3, + "DistanceFunction": "COSINE", + "IndexStatus": "ACTIVE", + "IndexSizeBytes": 0, + "ItemCount": 0, + "IndexArn": "" + })) + .unwrap() +} + +/// An item of the `Music` table with a `Year` number. +fn song(artist: &str, title: &str) -> Item { + use extenddb_core::types::AttributeValue; + Item::from([ + ("Artist".to_owned(), AttributeValue::S(artist.to_owned())), + ("SongTitle".to_owned(), AttributeValue::S(title.to_owned())), + ("Year".to_owned(), AttributeValue::N("1999".to_owned())), + ]) +} + +/// `song` with a `Year` the wire refuses (#618's shape). +fn song_with_bad_number(artist: &str, title: &str, year: &str) -> Item { + use extenddb_core::types::AttributeValue; + let mut item = song(artist, title); + item.insert("Year".to_owned(), AttributeValue::N(year.to_owned())); + item +} + +/// `n` songs in key order. +fn songs(n: usize) -> Vec { + (0..n) + .map(|i| song(&format!("artist-{i:05}"), &format!("title-{i:05}"))) + .collect() +} + +/// Make `fake` serve a snapshot of `table` yielding `items` (through `gate` +/// when given). +fn snapshot_fixture( + fake: &Fake, + table: &TableDescription, + items: Vec, + key_ordered: bool, + gate: Option>, +) { + fake.snapshots.lock().unwrap().insert( + table.table_name.clone(), + SnapshotFixture { + description: table.clone(), + key_ordered, + items: items.into_iter().map(Ok).collect(), + gate, + }, + ); +} + +/// A gate with no permit: the stream yields nothing until the test adds +/// some. +fn closed_gate() -> Arc { + Arc::new(tokio::sync::Semaphore::new(0)) +} + +/// The settings the job tests run with: small files, a short heartbeat. +fn job_settings() -> JobSettings { + JobSettings { + data_file_target_bytes: 64 * 1024 * 1024, + heartbeat_interval: Duration::from_millis(50), + stale_job_after: Duration::from_secs(300), + failed_row_retention: Duration::from_secs(7 * 24 * 3600), + } +} + +/// A healthy runtime on `mem` with the job settings above and a collector. +fn job_runtime(mem: &Arc) -> BackupRuntime { + runtime_on(mem, store()) + .with_job_settings(job_settings()) + .with_metrics(Arc::new(extenddb_core::metrics::MetricsCollector::new())) +} + +/// CreateBackup of `Music` through `runtime`. +async fn create_music_backup( + fake: &Arc, + runtime: &BackupRuntime, +) -> Result { + create_backup( + fake, + runtime, + ACCOUNT, + REGION, + &LimitsConfig::default(), + "Music", + "nightly", + Ok(music_table()), + ) + .await +} + +/// Wait until no job task of `runtime` runs. +async fn wait_for_jobs(runtime: &BackupRuntime) { + let deadline = tokio::time::Instant::now() + Duration::from_secs(30); + while runtime.running_jobs() > 0 { + assert!(tokio::time::Instant::now() < deadline, "a job did not end"); + tokio::time::sleep(Duration::from_millis(5)).await; + } +} + +/// The objects under the backup's prefix, by key. +fn objects_under(mem: &MemStore, arn: &str) -> Vec { + let prefix = BackupArn::parse(arn).unwrap().store_prefix(); + mem.keys() + .into_iter() + .filter(|k| k.starts_with(&prefix)) + .collect() +} + +/// The stored manifest of `arn`, parsed. +fn stored_manifest(mem: &MemStore, arn: &str) -> extenddb_core::backup_format::ExtenddbManifest { + let bytes = mem.objects.lock().unwrap()[&manifest_key(arn)].clone(); + extenddb_core::backup_format::ExtenddbManifest::from_json_bytes(&bytes).unwrap() +} + +/// The status writes the fake's claims received, as `(to, failure_reason)`. +fn writes(fake: &Fake) -> Vec<(JobStatus, Option)> { + fake.status_writes + .lock() + .unwrap() + .iter() + .map(|w| (w.to, w.fields.failure_reason.clone())) + .collect() +} + +/// The `BackupJobs` outcomes the collector drained, with their count. +fn job_outcomes(runtime: &BackupRuntime) -> Vec<(String, f64)> { + use extenddb_core::metrics::MetricName; + let mut outcomes: Vec<(String, f64)> = runtime + .metrics() + .unwrap() + .drain(Duration::ZERO) + .into_iter() + .filter(|b| b.metric == MetricName::BackupJobs) + .map(|b| (b.operation, b.sum)) + .collect(); + outcomes.sort_by(|a, b| a.0.cmp(&b.0)); + outcomes +} + +/// The request's order and the job's happy path. The checks before the +/// permit: a missing table is TableNotFound with every slot held and the +/// store untouched. Then the detached task: the snapshot is opened, the row +/// is inserted CREATING under the claim with the manifest's `source_table` +/// (the TTL read beside the open), the maintenance claim is read, and the +/// request is answered CREATING with the id's instant. The job then writes +/// the data file, `manifest-files.json`, `manifest-summary.json`, the +/// sidecar and the manifest, in that order and nothing else, and moves the +/// row to AVAILABLE with the manifest's size, the stream's count and the +/// manifest's `source_table`; the ARN's id, the row's `created_at`, the +/// response's time and the manifest's `backup_creation_date_time` are one +/// instant; the data file decodes back to the items; the slot is freed; the +/// metrics name the outcome, and `BackupSnapshotAge` was recorded by the +/// beats the job's three periods inside the stream gave it. +#[tokio::test] +async fn a_store_backed_create_answers_creating_and_its_job_writes_the_backup() { + let mem = Arc::new(MemStore::default()); + let fake = Arc::new(Fake::default()); + let items = songs(5); + let gate = closed_gate(); + snapshot_fixture( + &fake, + &music_table(), + items.clone(), + true, + Some(Arc::clone(&gate)), + ); + *fake.ttl.lock().unwrap() = Some(TimeToLiveDescription { + time_to_live_status: TimeToLiveStatus::Enabled, + attribute_name: Some("expires_at".to_owned()), + }); + let full = job_runtime(&mem); + let _held: Vec<_> = (0..4).map(|_| full.take_permit().unwrap()).collect(); + let err = create_backup( + &fake, + &full, + ACCOUNT, + REGION, + &LimitsConfig::default(), + "Music", + "nightly", + Err(StorageError::TableNotFound("Music".to_owned())), + ) + .await + .unwrap_err(); + assert!( + matches!(err, DynamoDbError::TableNotFoundException(_)), + "{err:?}" + ); + let err = create_music_backup(&fake, &full).await.unwrap_err(); + assert!( + matches!(err, DynamoDbError::LimitExceededException(_)), + "{err:?}" + ); + assert!( + fake.snapshots_opened.lock().unwrap().is_empty(), + "nothing was opened" + ); + drop(_held); + + let runtime = job_runtime(&mem); + let details = create_music_backup(&fake, &runtime).await.unwrap(); + assert_eq!(details.backup_status, "CREATING"); + assert_eq!(details.backup_type, "USER"); + assert_eq!(details.backup_name, "nightly"); + assert_eq!(details.backup_size_bytes, 0); + let arn = BackupArn::parse(&details.backup_arn).unwrap(); + assert_eq!( + extenddb_storage::util::backup_creation_date_time(&details.backup_arn), + Some(details.backup_creation_date_time), + "the response's time is the id's instant" + ); + let claimed = fake.claims.lock().unwrap().clone(); + assert_eq!(claimed.len(), 1); + assert_eq!(claimed[0].status, BackupStatus::Creating); + assert_eq!(claimed[0].store, store()); + assert_eq!(claimed[0].created_at, details.backup_creation_date_time); + assert_eq!(claimed[0].item_count, 0); + assert_eq!(claimed[0].source_table_details_json["item_count"], 100_000); + assert_eq!( + claimed[0].source_table_details_json["time_to_live"]["AttributeName"], + "expires_at" + ); + assert_eq!( + fake.maintenance_checks.lock().unwrap().as_slice(), + [Duration::from_secs(300)], + "the maintenance claim is read once the row exists" + ); + + // Three heartbeat periods inside the stream, so the timer beats before + // the job ends. + tokio::time::sleep(3 * job_settings().heartbeat_interval).await; + gate.add_permits(items.len()); + wait_for_jobs(&runtime).await; + assert_eq!(runtime.free_slots(), 4, "the job freed its slot"); + let prefix = arn.store_prefix(); + assert_eq!( + mem.puts.lock().unwrap().as_slice(), + [ + format!("{prefix}data/000001.json.gz"), + format!("{prefix}manifest-files.json"), + format!("{prefix}manifest-summary.json"), + format!("{prefix}extenddb-manifest.sha256"), + format!("{prefix}extenddb-manifest.json"), + ], + "the write order" + ); + assert!(mem.deletes.lock().unwrap().is_empty()); + let manifest = stored_manifest(&mem, &details.backup_arn); + assert_eq!(manifest.backup_arn, details.backup_arn); + assert_eq!( + manifest.backup_creation_date_time, + details.backup_creation_date_time + ); + assert_eq!(manifest.source_table.item_count, 5, "the stream's count"); + assert_eq!(manifest.source_table.table_size_bytes, 123_456); + assert_eq!(manifest.source_backend, "test"); + assert_eq!( + manifest.item_order, + extenddb_core::backup_format::ItemOrder::KeyOrdered + ); + assert_eq!(manifest.data_files.len(), 1); + assert!(manifest.data_files[0].key_range.is_some()); + let file = mem.objects.lock().unwrap()[&format!("{prefix}data/000001.json.gz")].clone(); + assert_eq!(manifest.data_files[0].bytes, file.len() as u64); + let mut reader = manifest.reader(&manifest.data_files[0], &file[..]).unwrap(); + let mut decoded = Vec::new(); + while let Some(item) = reader.next_item().unwrap() { + decoded.push(item); + } + assert_eq!(decoded, items); + let manifest_bytes = + mem.objects.lock().unwrap()[&format!("{prefix}extenddb-manifest.json")].clone(); + let sidecar = mem.objects.lock().unwrap()[&format!("{prefix}extenddb-manifest.sha256")].clone(); + extenddb_core::backup_format::verify_manifest_checksum(&manifest_bytes, &sidecar).unwrap(); + + let row = fake.rows.lock().unwrap()[&details.backup_arn].clone(); + assert_eq!(row.status, BackupStatus::Available); + assert_eq!(row.size_bytes, manifest.backup_size_bytes); + assert_eq!(row.item_count, 5); + assert_eq!( + row.source_table_details_json, + serde_json::to_value(&manifest.source_table).unwrap() + ); + assert!(row.completed_at.is_some()); + assert_eq!(writes(&fake), [(JobStatus::Available, None)]); + let write = fake.status_writes.lock().unwrap()[0].clone(); + assert_eq!( + (write.arn.as_str(), write.from, write.generation), + ( + details.backup_arn.as_str(), + JobStatus::Creating, + row.owner_generation + ), + "every status write is conditioned on CREATING under the claim's generation" + ); + assert!(fake.heartbeats.load(std::sync::atomic::Ordering::Acquire) >= 1); + + use extenddb_core::metrics::MetricName; + let drained = runtime.metrics().unwrap().drain(Duration::ZERO); + let find = |metric: MetricName| { + drained + .iter() + .filter(|b| b.metric == metric) + .map(|b| (b.operation.clone(), b.sum)) + .collect::>() + }; + assert_eq!( + find(MetricName::BackupJobs), + [("succeeded".to_owned(), 1.0)] + ); + assert_eq!(find(MetricName::BackupJobDuration).len(), 1); + let total: u64 = mem.put_sizes.lock().unwrap().iter().map(|(_, n)| n).sum(); + assert_eq!( + find(MetricName::BackupBytesUploaded), + [(String::new(), total as f64)] + ); + assert_eq!( + find(MetricName::WorkerLastSuccess) + .iter() + .map(|(op, _)| op.as_str()) + .collect::>(), + ["BackupJob"] + ); + assert!(find(MetricName::WorkerErrorCount).is_empty()); + let beats = fake.heartbeats.load(std::sync::atomic::Ordering::Acquire); + let ages: usize = drained + .iter() + .filter(|b| b.metric == MetricName::BackupSnapshotAge) + .map(|b| b.count as usize) + .sum(); + assert!( + ages >= 1 && ages < beats, + "the snapshot's age is recorded at each confirmed timer beat and not at the re-check \ + before the manifest: {ages} recorded, {beats} beats" + ); +} + +/// The maintenance claim is checked once the row exists: held, the row is +/// FAILED with reason `maintenance`, the request is LimitExceeded naming the +/// command, the snapshot is dropped, nothing reaches the store and the slot +/// is freed. +#[tokio::test] +async fn a_store_backed_create_under_a_maintenance_claim_fails_its_row_with_reason_maintenance() { + let mem = Arc::new(MemStore::default()); + let fake = Arc::new(Fake::default()); + snapshot_fixture(&fake, &music_table(), songs(3), true, None); + *fake.maintenance_held.lock().unwrap() = true; + let runtime = job_runtime(&mem); + let err = create_music_backup(&fake, &runtime).await.unwrap_err(); + assert!( + matches!(&err, DynamoDbError::LimitExceededException(m) if m.contains("backup purge")), + "{err:?}" + ); + wait_for_jobs(&runtime).await; + assert_eq!(fake.snapshots_opened.lock().unwrap().as_slice(), ["Music"]); + let rows = fake.rows.lock().unwrap(); + assert_eq!(rows.len(), 1); + let row = rows.values().next().unwrap(); + assert_eq!(row.status, BackupStatus::Failed); + assert_eq!(row.failure_reason.as_deref(), Some("maintenance")); + assert!(row.completed_at.is_some()); + drop(rows); + assert!(mem.puts.lock().unwrap().is_empty()); + assert!(mem.deletes.lock().unwrap().is_empty()); + assert_eq!(runtime.free_slots(), 4); + assert_eq!(job_outcomes(&runtime), []); +} + +/// A snapshot the backend refuses to open answers the request with the +/// open's error (here the backend's bound, LimitExceeded) and leaves no row +/// and no object; the slot is freed. +#[tokio::test] +async fn an_open_that_fails_answers_the_request_and_leaves_nothing() { + let mem = Arc::new(MemStore::default()); + let fake = Arc::new(Fake::default()); + *fake.snapshot_error.lock().unwrap() = + Some(StorageError::LimitExceeded("too many snapshots".to_owned())); + let runtime = job_runtime(&mem); + let err = create_music_backup(&fake, &runtime).await.unwrap_err(); + assert!( + matches!(&err, DynamoDbError::LimitExceededException(m) if m == "too many snapshots"), + "{err:?}" + ); + wait_for_jobs(&runtime).await; + assert!(fake.rows.lock().unwrap().is_empty()); + assert!(fake.claims.lock().unwrap().is_empty()); + assert!(mem.keys().is_empty()); + assert_eq!(runtime.free_slots(), 4); +} + +/// The file close rule: a file closes at the first item that takes its +/// uncompressed bytes to or past `data_file_target_bytes` (a knob of exactly +/// three lines closes at the third), so every file but the last is at least +/// the knob and under the knob plus one line, and the items are split across +/// files in order. At the knob's minimum (32 MiB) with 5 MiB items, each +/// file takes the items until the sum passes 32 MiB. +#[tokio::test] +async fn a_data_file_closes_at_the_first_item_that_reaches_the_knob() { + use extenddb_core::types::AttributeValue; + let line_bytes = |item: &Item| { + extenddb_core::backup_format::encode_item_line(item) + .unwrap() + .len() as u64 + }; + + // A low knob and small items: the exact rule. + let mem = Arc::new(MemStore::default()); + let fake = Arc::new(Fake::default()); + let items = songs(10); + let line = line_bytes(&items[0]); + let target = 3 * line; + snapshot_fixture(&fake, &music_table(), items.clone(), true, None); + let runtime = job_runtime(&mem).with_job_settings(JobSettings { + data_file_target_bytes: target, + ..job_settings() + }); + let details = create_music_backup(&fake, &runtime).await.unwrap(); + wait_for_jobs(&runtime).await; + { + let row = fake.rows.lock().unwrap()[&details.backup_arn].clone(); + assert_eq!( + (row.status, row.failure_reason.clone()), + (BackupStatus::Available, None), + "{:?}", + mem.puts.lock().unwrap() + ); + } + let manifest = stored_manifest(&mem, &details.backup_arn); + let counts: Vec = manifest.data_files.iter().map(|f| f.item_count).collect(); + assert_eq!( + counts, + [3, 3, 3, 1], + "the third line takes the file to the knob, so it closes there" + ); + for file in &manifest.data_files[..3] { + assert!(file.uncompressed_bytes >= target); + assert!(file.uncompressed_bytes - line < target); + } + assert_eq!( + manifest + .data_files + .iter() + .map(|f| f.key.as_str()) + .collect::>(), + [ + "data/000001.json.gz", + "data/000002.json.gz", + "data/000003.json.gz", + "data/000004.json.gz" + ] + ); + let prefix = BackupArn::parse(&details.backup_arn) + .unwrap() + .store_prefix(); + let mut decoded = Vec::new(); + for file in &manifest.data_files { + let bytes = mem.objects.lock().unwrap()[&format!("{prefix}{}", file.key)].clone(); + let mut reader = manifest.reader(file, &bytes[..]).unwrap(); + while let Some(item) = reader.next_item().unwrap() { + decoded.push(item); + } + } + assert_eq!(decoded, items); + assert_eq!( + fake.rows.lock().unwrap()[&details.backup_arn].item_count, + 10 + ); + + // The knob's minimum, 32 MiB, with 5 MiB binary items (a 6.7 MiB + // line each): five lines pass 32 MiB, then three. + let mem = Arc::new(MemStore::default()); + let fake = Arc::new(Fake::default()); + let items: Vec = (0..8) + .map(|i| { + let mut item = song("a", &format!("t{i}")); + item.insert( + "blob".to_owned(), + AttributeValue::B(vec![0u8; 5 * 1024 * 1024]), + ); + item + }) + .collect(); + let line = line_bytes(&items[0]); + snapshot_fixture(&fake, &music_table(), items, true, None); + let runtime = job_runtime(&mem).with_job_settings(JobSettings { + data_file_target_bytes: 32 * 1024 * 1024, + ..job_settings() + }); + let limits = LimitsConfig { + max_item_size_bytes: 8 * 1024 * 1024, + ..LimitsConfig::default() + }; + let details = create_backup( + &fake, + &runtime, + ACCOUNT, + REGION, + &limits, + "Music", + "nightly", + Ok(music_table()), + ) + .await + .unwrap(); + wait_for_jobs(&runtime).await; + assert_eq!( + fake.rows.lock().unwrap()[&details.backup_arn].failure_reason, + None + ); + let manifest = stored_manifest(&mem, &details.backup_arn); + let counts: Vec = manifest.data_files.iter().map(|f| f.item_count).collect(); + assert_eq!(counts, [5, 3]); + assert!(manifest.data_files[0].uncompressed_bytes >= 32 * 1024 * 1024); + assert!(manifest.data_files[0].uncompressed_bytes - line < 32 * 1024 * 1024); +} + +/// An unordered snapshot writes `unordered` with no key ranges; an ordered +/// one `key_ordered` with a range per file. +#[tokio::test] +async fn the_manifest_records_the_snapshots_order() { + for key_ordered in [false, true] { + let mem = Arc::new(MemStore::default()); + let fake = Arc::new(Fake::default()); + snapshot_fixture(&fake, &music_table(), songs(3), key_ordered, None); + let runtime = job_runtime(&mem); + let details = create_music_backup(&fake, &runtime).await.unwrap(); + wait_for_jobs(&runtime).await; + let manifest = stored_manifest(&mem, &details.backup_arn); + let expected = if key_ordered { + extenddb_core::backup_format::ItemOrder::KeyOrdered + } else { + extenddb_core::backup_format::ItemOrder::Unordered + }; + assert_eq!(manifest.item_order, expected); + assert_eq!(manifest.data_files[0].key_range.is_some(), key_ordered); + } +} + +/// The validator's rule: with three items the wire let in, the job reads +/// and checks the whole stream past the first refusal (every gate permit is +/// taken), encodes nothing more, and fails the row with the count and the +/// first item, naming #618 for an invalid number; the FAILED write comes +/// first and the prefix is deleted after it. The stream is fed one item +/// every 20 ms, slower than the pipeline, so the encoder's stop lands +/// before the stream's end and an uploader that read that stop as its own +/// failure would end the feeder with the rest unchecked. An item refused +/// for its size gives a reason that does not name #618. +#[tokio::test] +async fn the_validator_fails_the_backup_after_the_rest_is_validated() { + let mem = Arc::new(MemStore::default()); + let fake = Arc::new(Fake::default()); + let items = vec![ + song("a", "1"), + song_with_bad_number("a", "2", "1E126"), + song("a", "3"), + song_with_bad_number("a", "4", "abc"), + song_with_bad_number("a", "5", "1E126"), + song("a", "6"), + ]; + let count = items.len(); + let gate = closed_gate(); + snapshot_fixture(&fake, &music_table(), items, true, Some(Arc::clone(&gate))); + let runtime = job_runtime(&mem); + let details = create_music_backup(&fake, &runtime).await.unwrap(); + let feed = tokio::spawn({ + let gate = Arc::clone(&gate); + async move { + for _ in 0..count { + tokio::time::sleep(Duration::from_millis(20)).await; + gate.add_permits(1); + } + } + }); + wait_for_jobs(&runtime).await; + feed.await.unwrap(); + assert_eq!(gate.available_permits(), 0, "every item was read"); + let row = fake.rows.lock().unwrap()[&details.backup_arn].clone(); + assert_eq!(row.status, BackupStatus::Failed); + let reason = row.failure_reason.unwrap(); + assert!( + reason.starts_with( + "3 item(s) cannot be restored; first: key {\"Artist\":{\"S\":\"a\"},\"SongTitle\":{\"S\":\"2\"}}, \ + attribute Year: " + ), + "{reason}" + ); + assert!(reason.ends_with(" (#618)"), "{reason}"); + let prefix = BackupArn::parse(&details.backup_arn) + .unwrap() + .store_prefix(); + assert_eq!(writes(&fake).len(), 1); + assert_eq!(writes(&fake)[0].0, JobStatus::Failed); + assert_eq!( + mem.deletes.lock().unwrap().as_slice(), + std::slice::from_ref(&prefix) + ); + assert!(objects_under(&mem, &details.backup_arn).is_empty()); + assert!( + mem.puts.lock().unwrap().is_empty(), + "after the first refused item nothing is uploaded, the first file with its one valid \ + item included: {:?}", + mem.puts.lock().unwrap() + ); + assert_eq!(job_outcomes(&runtime), [("failed".to_owned(), 1.0)]); + + // Over `max_item_size_bytes`: the reason names the size rule, not #618. + let mem = Arc::new(MemStore::default()); + let fake = Arc::new(Fake::default()); + snapshot_fixture(&fake, &music_table(), songs(2), true, None); + let runtime = job_runtime(&mem); + let limits = LimitsConfig { + max_item_size_bytes: 20, + ..LimitsConfig::default() + }; + let details = create_backup( + &fake, + &runtime, + ACCOUNT, + REGION, + &limits, + "Music", + "nightly", + Ok(music_table()), + ) + .await + .unwrap(); + wait_for_jobs(&runtime).await; + let row = fake.rows.lock().unwrap()[&details.backup_arn].clone(); + assert_eq!(row.status, BackupStatus::Failed); + let reason = row.failure_reason.unwrap(); + assert!( + reason.starts_with("2 item(s) cannot be restored; first: key "), + "{reason}" + ); + assert!(reason.contains("20 bytes"), "{reason}"); + assert!(!reason.contains("#618"), "{reason}"); +} + +/// A put that fails fails the row with the file named, and only then +/// deletes the prefix; the store error is counted. +#[tokio::test] +async fn a_put_failure_fails_the_row_then_deletes_the_prefix() { + let mem = Arc::new(MemStore::default()); + mem.fail_puts + .store(true, std::sync::atomic::Ordering::Release); + let fake = Arc::new(Fake::default()); + snapshot_fixture(&fake, &music_table(), songs(3), true, None); + let runtime = job_runtime(&mem); + let details = create_music_backup(&fake, &runtime).await.unwrap(); + wait_for_jobs(&runtime).await; + let row = fake.rows.lock().unwrap()[&details.backup_arn].clone(); + assert_eq!(row.status, BackupStatus::Failed); + assert!( + row.failure_reason + .as_deref() + .unwrap() + .starts_with("writing data/000001.json.gz: "), + "{:?}", + row.failure_reason + ); + let prefix = BackupArn::parse(&details.backup_arn) + .unwrap() + .store_prefix(); + assert_eq!( + writes(&fake), + [(JobStatus::Failed, row.failure_reason.clone())] + ); + assert_eq!(mem.deletes.lock().unwrap().as_slice(), [prefix]); + use extenddb_core::metrics::MetricName; + let drained = runtime.metrics().unwrap().drain(Duration::ZERO); + assert!(drained.iter().any(|b| { + b.metric == MetricName::BackupStoreErrors && b.operation == "put" && b.sum == 1.0 + })); + assert!( + drained + .iter() + .any(|b| { b.metric == MetricName::BackupJobs && b.operation == "failed" }) + ); + assert!( + drained + .iter() + .any(|b| { b.metric == MetricName::WorkerErrorCount && b.operation == "BackupJob" }) + ); +} + +/// A FAILED write that matches no row (the row's generation changed under +/// a heartbeat that kept confirming) ends the job with no store access: no +/// `delete_prefix`, the row untouched. +#[tokio::test] +async fn a_failed_write_that_does_not_match_makes_no_store_call() { + let mem = Arc::new(MemStore::default()); + let fake = Arc::new(Fake::default()); + let gate = closed_gate(); + snapshot_fixture( + &fake, + &music_table(), + songs(3), + true, + Some(Arc::clone(&gate)), + ); + *fake.heartbeat_answer.lock().unwrap() = HeartbeatAnswer::AlwaysConfirm; + let runtime = job_runtime(&mem); + let details = create_music_backup(&fake, &runtime).await.unwrap(); + // Another generation takes the row while the job waits on its stream; + // the put then fails. + fake.rows + .lock() + .unwrap() + .get_mut(&details.backup_arn) + .unwrap() + .owner_generation = 999; + mem.fail_puts + .store(true, std::sync::atomic::Ordering::Release); + gate.add_permits(3); + wait_for_jobs(&runtime).await; + assert_eq!(writes(&fake).len(), 1); + assert_eq!(writes(&fake)[0].0, JobStatus::Failed); + assert!( + mem.deletes.lock().unwrap().is_empty(), + "no store access after NotMatched" + ); + let row = fake.rows.lock().unwrap()[&details.backup_arn].clone(); + assert_eq!(row.status, BackupStatus::Creating); + assert_eq!(row.owner_generation, 999); + assert_eq!(job_outcomes(&runtime), [("failed".to_owned(), 1.0)]); +} + +/// A status write whose outcome is unknown is re-read, not treated as a +/// failure: an AVAILABLE write that landed is AVAILABLE with no store +/// access after it; one that was lost is written once more. The same for +/// the FAILED write: landed, the deletion follows; lost, it is written again +/// and the deletion follows. +#[tokio::test] +async fn an_unknown_write_is_re_read() { + for (unknown, writes_expected) in [(UnknownWrite::Applied, 1), (UnknownWrite::Lost, 2)] { + let mem = Arc::new(MemStore::default()); + let fake = Arc::new(Fake::default()); + snapshot_fixture(&fake, &music_table(), songs(2), true, None); + *fake.unknown_write.lock().unwrap() = Some(unknown); + let runtime = job_runtime(&mem); + let details = create_music_backup(&fake, &runtime).await.unwrap(); + wait_for_jobs(&runtime).await; + let row = fake.rows.lock().unwrap()[&details.backup_arn].clone(); + assert_eq!(row.status, BackupStatus::Available, "{unknown:?}"); + assert_eq!(writes(&fake).len(), writes_expected, "{unknown:?}"); + assert!(mem.deletes.lock().unwrap().is_empty()); + assert_eq!(objects_under(&mem, &details.backup_arn).len(), 5); + assert_eq!(job_outcomes(&runtime), [("succeeded".to_owned(), 1.0)]); + } + for (unknown, writes_expected) in [(UnknownWrite::Applied, 1), (UnknownWrite::Lost, 2)] { + let mem = Arc::new(MemStore::default()); + mem.fail_puts + .store(true, std::sync::atomic::Ordering::Release); + let fake = Arc::new(Fake::default()); + snapshot_fixture(&fake, &music_table(), songs(2), true, None); + *fake.unknown_write.lock().unwrap() = Some(unknown); + let runtime = job_runtime(&mem); + let details = create_music_backup(&fake, &runtime).await.unwrap(); + wait_for_jobs(&runtime).await; + let row = fake.rows.lock().unwrap()[&details.backup_arn].clone(); + assert_eq!(row.status, BackupStatus::Failed, "{unknown:?}"); + assert_eq!(writes(&fake).len(), writes_expected, "{unknown:?}"); + let prefix = BackupArn::parse(&details.backup_arn) + .unwrap() + .store_prefix(); + assert_eq!(mem.deletes.lock().unwrap().as_slice(), [prefix]); + } +} + +/// A heartbeat that reports the claim lost, or that does not return within +/// a period, ends the job with no put and no catalog write: the row is left +/// as it is for its owner or the reconciler. +#[tokio::test] +async fn a_lost_or_stalled_heartbeat_stops_the_job_with_no_put() { + for answer in [ + HeartbeatAnswer::Lost, + HeartbeatAnswer::Stall, + HeartbeatAnswer::Error, + ] { + let mem = Arc::new(MemStore::default()); + let fake = Arc::new(Fake::default()); + let gate = closed_gate(); + snapshot_fixture(&fake, &music_table(), songs(3), true, Some(gate)); + let runtime = job_runtime(&mem); + let details = create_music_backup(&fake, &runtime).await.unwrap(); + // The job waits on its stream (the encoder is a blocking task, so + // the clock runs: a paused clock would not advance behind it); the + // first beat, a period later, finds the claim lost (or never + // returns: a period more). + *fake.heartbeat_answer.lock().unwrap() = answer; + wait_for_jobs(&runtime).await; + assert!(mem.puts.lock().unwrap().is_empty(), "{answer:?}: no put"); + assert!( + mem.deletes.lock().unwrap().is_empty(), + "{answer:?}: no delete" + ); + assert!(writes(&fake).is_empty(), "{answer:?}: no catalog write"); + let row = fake.rows.lock().unwrap()[&details.backup_arn].clone(); + assert_eq!(row.status, BackupStatus::Creating, "{answer:?}"); + assert_eq!(runtime.free_slots(), 4); + assert_eq!(job_outcomes(&runtime), [("failed".to_owned(), 1.0)]); + } +} + +/// The pre-put check's numbers: with the claim confirming, the last +/// confirmed beat stays recent however long the job runs; a timer that +/// stopped running leaves the check true for two periods after the last +/// confirmed beat and false after; with a beat that never returns, the +/// check still passes until the timer has waited one period for the beat +/// (two periods from the last confirmation), and the claim is lost from +/// then on. +#[tokio::test(start_paused = true)] +async fn the_heartbeat_handle_confirms_the_claim_within_two_periods() { + let fake = Arc::new(Fake::default()); + fake.rows + .lock() + .unwrap() + .insert(arn(ACCOUNT), row(&arn(ACCOUNT), BackupStatus::Creating)); + let claim: Arc = Arc::from(fake.claim_on(&arn(ACCOUNT), 7)); + let period = Duration::from_secs(10); + let handle = job::heartbeat::HeartbeatHandle::start( + Arc::clone(&claim), + period, + None, + 0.0, + tokio::time::Instant::now(), + ); + // Half a period past the tenth tick, so that tick's beat has confirmed. + tokio::time::sleep(10 * period + period / 2).await; + assert!(handle.confirms_the_claim()); + assert!(fake.heartbeats.load(std::sync::atomic::Ordering::Acquire) >= 10); + // A timer that stopped running (no loss raised): the check holds for two + // periods after the last confirmed beat and then refuses. + handle.stop_timer(); + tokio::time::sleep(period).await; + assert!(handle.confirms_the_claim(), "a period and a half"); + tokio::time::sleep(period).await; + assert!(!handle.lost().is_cancelled()); + assert!( + !handle.confirms_the_claim(), + "two periods without a confirmed beat" + ); + drop(handle); + + *fake.heartbeat_answer.lock().unwrap() = HeartbeatAnswer::Stall; + let handle = job::heartbeat::HeartbeatHandle::start( + claim, + period, + None, + 0.0, + tokio::time::Instant::now(), + ); + tokio::time::sleep(period + period / 2).await; + assert!(handle.confirms_the_claim(), "the beat may still return"); + assert!(!handle.lost().is_cancelled()); + tokio::time::sleep(period).await; + assert!(!handle.confirms_the_claim()); + assert!(handle.lost().is_cancelled()); + assert!(handle.reason().contains("did not return")); +} + +/// The shutdown token runs the failure path: FAILED with reason `shutdown`, +/// then the prefix deleted; the outcome is `cancelled`. `drain_jobs` is what +/// the server calls: it cancels and waits for the job. +#[tokio::test] +async fn cancellation_runs_the_failure_path() { + let mem = Arc::new(MemStore::default()); + let fake = Arc::new(Fake::default()); + let gate = Arc::new(tokio::sync::Semaphore::new(1)); + snapshot_fixture( + &fake, + &music_table(), + songs(3), + true, + Some(Arc::clone(&gate)), + ); + let runtime = job_runtime(&mem); + let details = create_music_backup(&fake, &runtime).await.unwrap(); + // The job holds its first item and waits for the second. + let deadline = tokio::time::Instant::now() + Duration::from_secs(5); + while gate.available_permits() > 0 { + assert!(tokio::time::Instant::now() < deadline); + tokio::time::sleep(Duration::from_millis(5)).await; + } + assert_eq!(runtime.running_jobs(), 1); + let left = runtime.drain_jobs(Duration::from_secs(10)).await; + assert_eq!(left, 0); + assert_eq!(runtime.running_jobs(), 0); + let row = fake.rows.lock().unwrap()[&details.backup_arn].clone(); + assert_eq!(row.status, BackupStatus::Failed); + assert_eq!(row.failure_reason.as_deref(), Some("shutdown")); + let prefix = BackupArn::parse(&details.backup_arn) + .unwrap() + .store_prefix(); + assert_eq!( + writes(&fake), + [(JobStatus::Failed, Some("shutdown".to_owned()))] + ); + assert_eq!(mem.deletes.lock().unwrap().as_slice(), [prefix]); + assert!(objects_under(&mem, &details.backup_arn).is_empty()); + use extenddb_core::metrics::MetricName; + let drained = runtime.metrics().unwrap().drain(Duration::ZERO); + assert_eq!( + drained + .iter() + .filter(|b| b.metric == MetricName::BackupJobs) + .map(|b| (b.operation.clone(), b.sum)) + .collect::>(), + [("cancelled".to_owned(), 1.0)] + ); + assert!( + !drained + .iter() + .any(|b| b.metric == MetricName::WorkerErrorCount), + "a clean shutdown is not a worker error" + ); + assert!( + !drained + .iter() + .any(|b| b.metric == MetricName::WorkerLastSuccess), + "nor a success" + ); +} + +/// With `max_concurrent_jobs = 1`, a second CreateBackup while a job runs +/// is LimitExceeded naming the bound; once the job ends the slot is free. +#[tokio::test] +async fn a_second_create_over_the_job_bound_is_limit_exceeded() { + let mem = Arc::new(MemStore::default()); + let fake = Arc::new(Fake::default()); + let gate = closed_gate(); + snapshot_fixture( + &fake, + &music_table(), + songs(2), + true, + Some(Arc::clone(&gate)), + ); + let runtime = BackupRuntime::new(1) + .with_store(ConfiguredStore::new( + store().store_kind, + store().store_root, + Arc::clone(&mem) as Arc, + None, + Ok(store()), + )) + .with_job_settings(job_settings()); + create_music_backup(&fake, &runtime).await.unwrap(); + let err = create_music_backup(&fake, &runtime).await.unwrap_err(); + assert!( + matches!(&err, DynamoDbError::LimitExceededException(m) if m.contains("max_concurrent_jobs")), + "{err:?}" + ); + assert_eq!( + fake.snapshots_opened.lock().unwrap().len(), + 1, + "the second opened nothing" + ); + gate.add_permits(2); + wait_for_jobs(&runtime).await; + assert_eq!(runtime.free_slots(), 1); + snapshot_fixture(&fake, &music_table(), songs(1), true, None); + create_music_backup(&fake, &runtime).await.unwrap(); + wait_for_jobs(&runtime).await; + assert_eq!(fake.rows.lock().unwrap().len(), 2); +} + +/// A `tracing` subscriber that keeps every event's level and message, for +/// the tests of the job's log lines. Set as the thread's default for the +/// test's life through [`LogLines::capture`]; a `#[tokio::test]` runtime +/// runs the job's tasks on the test's thread, so their events reach it (the +/// encoders, on the blocking pool, log nothing). +#[derive(Clone, Default)] +struct LogLines(Arc>>); + +/// The process's global `tracing` subscriber while these tests run: +/// interested in every callsite and enabled for none. `tracing` caches each +/// callsite's interest the first time any thread reaches it; while exactly +/// one scoped subscriber is registered in the process, that cache is +/// computed from the default of the thread that got there first. A test +/// without a subscriber running beside a `LogLines` test on another thread +/// would so cache the job's log callsites as "never", and the `LogLines` +/// test would then see no line from them (tokio-rs/tracing#2743). With this +/// global default the cache reads "sometimes" instead, and each event asks +/// the thread's own default, the `LogLines` where one is set. +struct InterestedInEveryCallsite; + +impl tracing::Subscriber for InterestedInEveryCallsite { + fn register_callsite( + &self, + _: &'static tracing::Metadata<'static>, + ) -> tracing::subscriber::Interest { + tracing::subscriber::Interest::sometimes() + } + fn enabled(&self, _: &tracing::Metadata<'_>) -> bool { + false + } + fn new_span(&self, _: &tracing::span::Attributes<'_>) -> tracing::span::Id { + tracing::span::Id::from_u64(1) + } + fn record(&self, _: &tracing::span::Id, _: &tracing::span::Record<'_>) {} + fn record_follows_from(&self, _: &tracing::span::Id, _: &tracing::span::Id) {} + fn event(&self, _: &tracing::Event<'_>) {} + fn enter(&self, _: &tracing::span::Id) {} + fn exit(&self, _: &tracing::span::Id) {} +} + +impl LogLines { + /// A fresh `LogLines` set as this thread's `tracing` default until the + /// guard drops, with the process's global default installed once. + fn capture() -> (Self, tracing::subscriber::DefaultGuard) { + static GLOBAL: std::sync::Once = std::sync::Once::new(); + GLOBAL.call_once(|| { + // Another global may already be set by a test outside this + // module; then its interest rule applies and this one is moot. + let _ = tracing::subscriber::set_global_default(InterestedInEveryCallsite); + }); + let lines = Self::default(); + let guard = tracing::subscriber::set_default(lines.clone()); + (lines, guard) + } + + /// The messages containing `needle`, in order. + fn containing(&self, needle: &str) -> Vec { + self.0 + .lock() + .unwrap() + .iter() + .filter(|(_, m)| m.contains(needle)) + .map(|(_, m)| m.clone()) + .collect() + } +} + +struct MessageVisitor<'a>(&'a mut String); + +/// A callsite met first by a thread without a scoped subscriber is still +/// seen by a `LogLines` on another thread (the global default's interest +/// rule; the doc of `InterestedInEveryCallsite`). Without that default the +/// callsite's interest is cached as "never" whenever this is the only +/// `LogLines` alive in the process, and nothing is captured. +#[test] +fn a_callsite_first_met_on_another_thread_still_reaches_the_lines() { + fn shared_callsite(n: u32) { + tracing::error!("probe {n}: a callsite shared by two threads"); + } + let (lines, _guard) = LogLines::capture(); + std::thread::spawn(|| shared_callsite(1)).join().unwrap(); + shared_callsite(2); + assert_eq!( + lines.containing("probe "), + ["probe 2: a callsite shared by two threads"] + ); +} + +impl tracing::field::Visit for MessageVisitor<'_> { + fn record_debug(&mut self, field: &tracing::field::Field, value: &dyn std::fmt::Debug) { + if field.name() == "message" { + use std::fmt::Write as _; + let _ = write!(self.0, "{value:?}"); + } + } +} + +impl tracing::Subscriber for LogLines { + fn enabled(&self, _: &tracing::Metadata<'_>) -> bool { + true + } + fn new_span(&self, _: &tracing::span::Attributes<'_>) -> tracing::span::Id { + tracing::span::Id::from_u64(1) + } + fn record(&self, _: &tracing::span::Id, _: &tracing::span::Record<'_>) {} + fn record_follows_from(&self, _: &tracing::span::Id, _: &tracing::span::Id) {} + fn event(&self, event: &tracing::Event<'_>) { + let mut message = String::new(); + event.record(&mut MessageVisitor(&mut message)); + self.0 + .lock() + .unwrap() + .push((*event.metadata().level(), message)); + } + fn enter(&self, _: &tracing::span::Id) {} + fn exit(&self, _: &tracing::span::Id) {} +} + +/// The bytes of one item's data file line. +fn line_bytes(item: &Item) -> u64 { + extenddb_core::backup_format::encode_item_line(item) + .unwrap() + .len() as u64 +} + +/// A put that fails while the snapshot stream is between pages ends the job +/// at once: the stream holds one permit, so it yields the first item and +/// then waits; the first file closes at that item (a one-line knob), its put +/// fails, and the FAILED write lands with the stream still held, no permit +/// released. Without the uploader's stop reaching the feeder, the feeder +/// would wait for the stream's next item and the row would stay CREATING +/// with a live heartbeat for as long as the stream did. +#[tokio::test] +async fn a_put_failure_ends_the_job_without_waiting_for_the_streams_next_item() { + let mem = Arc::new(MemStore::default()); + mem.fail_puts + .store(true, std::sync::atomic::Ordering::Release); + let fake = Arc::new(Fake::default()); + let items = songs(3); + let target = line_bytes(&items[0]); + let gate = Arc::new(tokio::sync::Semaphore::new(1)); + snapshot_fixture(&fake, &music_table(), items, true, Some(Arc::clone(&gate))); + let runtime = job_runtime(&mem).with_job_settings(JobSettings { + data_file_target_bytes: target, + ..job_settings() + }); + let details = create_music_backup(&fake, &runtime).await.unwrap(); + wait_for_jobs(&runtime).await; + assert_eq!(gate.available_permits(), 0, "the stream was never released"); + let row = fake.rows.lock().unwrap()[&details.backup_arn].clone(); + assert_eq!(row.status, BackupStatus::Failed); + assert!( + row.failure_reason + .as_deref() + .unwrap() + .starts_with("writing data/000001.json.gz: "), + "{:?}", + row.failure_reason + ); + let prefix = BackupArn::parse(&details.backup_arn) + .unwrap() + .store_prefix(); + assert_eq!(mem.deletes.lock().unwrap().as_slice(), [prefix]); + assert_eq!(job_outcomes(&runtime), [("failed".to_owned(), 1.0)]); +} + +/// `BackupJobDuration` of a FAILED job covers its failure path: with the +/// store's deletion taking 300 ms, the duration recorded under `failed` is +/// at least that, so the metric measures the job to the end of its last +/// store call for either outcome. +#[tokio::test] +async fn a_failed_jobs_duration_covers_its_failed_write_and_deletion() { + use extenddb_core::metrics::MetricName; + let mem = Arc::new(MemStore::default()); + mem.fail_puts + .store(true, std::sync::atomic::Ordering::Release); + *mem.delete_delay.lock().unwrap() = Some(Duration::from_millis(300)); + let fake = Arc::new(Fake::default()); + snapshot_fixture(&fake, &music_table(), songs(2), true, None); + let runtime = job_runtime(&mem); + let details = create_music_backup(&fake, &runtime).await.unwrap(); + wait_for_jobs(&runtime).await; + let row = fake.rows.lock().unwrap()[&details.backup_arn].clone(); + assert_eq!(row.status, BackupStatus::Failed); + assert_eq!(mem.deletes.lock().unwrap().len(), 1); + let drained = runtime.metrics().unwrap().drain(Duration::ZERO); + let durations: Vec<(String, f64)> = drained + .iter() + .filter(|b| b.metric == MetricName::BackupJobDuration) + .map(|b| (b.operation.clone(), b.sum)) + .collect(); + assert_eq!(durations.len(), 1, "{durations:?}"); + assert_eq!(durations[0].0, "failed"); + assert!( + durations[0].1 >= 300_000.0, + "{} µs: the duration stops before the deletion", + durations[0].1 + ); +} + +/// The two info lines: the start line gives the table's count as the stats +/// pass last counted it (the description's figure, here 100,000), the +/// completion line the stream's count (the items written) and the bytes. +#[tokio::test] +async fn the_start_and_completion_lines_name_the_two_item_counts() { + let (lines, _guard) = LogLines::capture(); + let mem = Arc::new(MemStore::default()); + let fake = Arc::new(Fake::default()); + snapshot_fixture(&fake, &music_table(), songs(5), true, None); + let runtime = job_runtime(&mem); + let details = create_music_backup(&fake, &runtime).await.unwrap(); + wait_for_jobs(&runtime).await; + let arn = &details.backup_arn; + let started = lines.containing(": started, "); + assert_eq!( + started, + [format!( + "backup {arn}: started, 100000 items in the table as last counted by the stats pass, \ + key-ordered" + )] + ); + let total: u64 = mem.put_sizes.lock().unwrap().iter().map(|(_, n)| n).sum(); + let available = lines.containing(": AVAILABLE, "); + assert_eq!(available.len(), 1, "{available:?}"); + assert!( + available[0].starts_with(&format!( + "backup {arn}: AVAILABLE, 5 items, {total} bytes written in " + )), + "{}", + available[0] + ); +} + +/// The refusal log cap: with 1,001 refused items the first 1,000 are +/// logged one by one, one line counts the remainder, the reason carries the +/// full count and the first item, the stream is read to its end and nothing +/// is uploaded. +#[tokio::test] +async fn refusals_past_the_log_cap_are_counted_in_one_line() { + let (lines, _guard) = LogLines::capture(); + let mem = Arc::new(MemStore::default()); + let fake = Arc::new(Fake::default()); + let mut items = vec![song("a", "0")]; + items.extend((1..=1001).map(|i| song_with_bad_number("a", &format!("{i:05}"), "1E126"))); + items.push(song("a", "zzz")); + let gate = Arc::new(tokio::sync::Semaphore::new(items.len())); + snapshot_fixture(&fake, &music_table(), items, true, Some(Arc::clone(&gate))); + let runtime = job_runtime(&mem); + let details = create_music_backup(&fake, &runtime).await.unwrap(); + wait_for_jobs(&runtime).await; + assert_eq!(gate.available_permits(), 0, "every item was read"); + let logged = lines.containing(": item cannot be restored: key "); + assert_eq!(logged.len(), 1000); + assert!( + logged[0].contains(r#""SongTitle":{"S":"00001"}"#) + && logged[0].contains(", attribute Year: "), + "{}", + logged[0] + ); + assert!( + logged[999].contains(r#""SongTitle":{"S":"01000"}"#), + "{}", + logged[999] + ); + assert_eq!( + lines.containing(" more item(s) cannot be restored (not listed)"), + [format!( + "backup {}: 1 more item(s) cannot be restored (not listed)", + details.backup_arn + )] + ); + let row = fake.rows.lock().unwrap()[&details.backup_arn].clone(); + assert_eq!(row.status, BackupStatus::Failed); + let reason = row.failure_reason.unwrap(); + assert!( + reason.starts_with("1001 item(s) cannot be restored; first: key "), + "{reason}" + ); + assert!(reason.contains(r#""SongTitle":{"S":"00001"}"#), "{reason}"); + assert!(mem.puts.lock().unwrap().is_empty()); +} + +/// An item whose data file line would exceed the format's cap is refused +/// through the validator's path, not the encoder's: under a raised item +/// limit, a 600 KB item of the worst-expansion shape (a list of number +/// sets, a 40 MB line) is counted, logged with its key and the cap, the +/// rest of the stream is checked, nothing is uploaded and the FAILED +/// reason names the item without #618. The item passes the validator +/// itself (its numbers are valid, its size within the limit). The line is +/// measured through a counting writer that stops at the cap (the format's +/// own test pins that it holds no line), so the reason carries the cap and +/// not the line's length. +#[tokio::test] +async fn an_item_whose_line_exceeds_the_cap_is_refused_and_named() { + use extenddb_core::backup_format::MAX_LINE_BYTES; + use extenddb_core::types::AttributeValue; + use std::collections::BTreeSet; + let (lines, _guard) = LogLines::capture(); + // 81 two-digit values at `E-130`, 133 characters each in normalized + // form, 2 bytes of item size each (the `MAX_LINE_BYTES` doc's shape). + let set: BTreeSet = (11..100) + .filter(|m| m % 10 != 0) + .map(|m| format!("0.{}{m}", "0".repeat(129))) + .collect(); + assert_eq!(set.len(), 81); + let mut worst = song("a", "2"); + worst.insert( + "l".to_owned(), + AttributeValue::L(vec![AttributeValue::NS(set); 3_700]), + ); + let size = extenddb_core::types::item_size_bytes(&worst); + assert!(size > 409_600 && size < 2 * 1024 * 1024, "{size}"); + let line = line_bytes(&worst); + assert!(line > MAX_LINE_BYTES as u64, "{line}"); + let items = vec![song("a", "1"), worst, song("a", "3")]; + let gate = Arc::new(tokio::sync::Semaphore::new(items.len())); + let mem = Arc::new(MemStore::default()); + let fake = Arc::new(Fake::default()); + snapshot_fixture(&fake, &music_table(), items, true, Some(Arc::clone(&gate))); + let runtime = job_runtime(&mem); + let limits = LimitsConfig { + max_item_size_bytes: 2 * 1024 * 1024, + ..LimitsConfig::default() + }; + let details = create_backup( + &fake, + &runtime, + ACCOUNT, + REGION, + &limits, + "Music", + "nightly", + Ok(music_table()), + ) + .await + .unwrap(); + wait_for_jobs(&runtime).await; + assert_eq!( + gate.available_permits(), + 0, + "the rest of the stream was checked" + ); + let row = fake.rows.lock().unwrap()[&details.backup_arn].clone(); + assert_eq!(row.status, BackupStatus::Failed); + let reason = row.failure_reason.unwrap(); + assert_eq!( + reason, + format!( + "1 item(s) cannot be restored; first: key {{\"Artist\":{{\"S\":\"a\"}},\"SongTitle\":\ + {{\"S\":\"2\"}}}}: data file line exceeds the line cap of {MAX_LINE_BYTES} bytes" + ) + ); + assert_eq!(lines.containing(": item cannot be restored: key ").len(), 1); + assert!( + mem.puts.lock().unwrap().is_empty(), + "{:?}", + mem.puts.lock().unwrap() + ); + assert_eq!(job_outcomes(&runtime), [("failed".to_owned(), 1.0)]); +} + +/// A status write that errors on the claim connection makes no store call, +/// whichever write it is. The FAILED write: no `delete_prefix`, the row +/// CREATING for the reconciler. The AVAILABLE write: the manifest was +/// already put, so the five objects stay for the reconciler's probe, which +/// is the order the design rests on (the manifest is the commit record, the +/// AVAILABLE write follows it). +#[tokio::test] +async fn a_status_write_error_makes_no_store_call_and_leaves_the_manifest() { + let mem = Arc::new(MemStore::default()); + mem.fail_puts + .store(true, std::sync::atomic::Ordering::Release); + let fake = Arc::new(Fake::default()); + snapshot_fixture(&fake, &music_table(), songs(2), true, None); + fake.status_write_error + .store(true, std::sync::atomic::Ordering::Release); + let runtime = job_runtime(&mem); + let details = create_music_backup(&fake, &runtime).await.unwrap(); + wait_for_jobs(&runtime).await; + let row = fake.rows.lock().unwrap()[&details.backup_arn].clone(); + assert_eq!(row.status, BackupStatus::Creating); + assert_eq!(writes(&fake).len(), 1); + assert!( + mem.deletes.lock().unwrap().is_empty(), + "no delete after a claim error on the FAILED write" + ); + assert_eq!(job_outcomes(&runtime), [("failed".to_owned(), 1.0)]); + + let mem = Arc::new(MemStore::default()); + let fake = Arc::new(Fake::default()); + snapshot_fixture(&fake, &music_table(), songs(2), true, None); + fake.status_write_error + .store(true, std::sync::atomic::Ordering::Release); + let runtime = job_runtime(&mem); + let details = create_music_backup(&fake, &runtime).await.unwrap(); + wait_for_jobs(&runtime).await; + let row = fake.rows.lock().unwrap()[&details.backup_arn].clone(); + assert_eq!(row.status, BackupStatus::Creating); + assert_eq!(writes(&fake), [(JobStatus::Available, None)]); + assert!(mem.deletes.lock().unwrap().is_empty()); + let prefix = BackupArn::parse(&details.backup_arn) + .unwrap() + .store_prefix(); + assert_eq!( + mem.puts.lock().unwrap().as_slice(), + [ + format!("{prefix}data/000001.json.gz"), + format!("{prefix}manifest-files.json"), + format!("{prefix}manifest-summary.json"), + format!("{prefix}extenddb-manifest.sha256"), + format!("{prefix}extenddb-manifest.json"), + ], + "the manifest was put before the AVAILABLE write" + ); + assert_eq!(objects_under(&mem, &details.backup_arn).len(), 5); + assert_eq!(job_outcomes(&runtime), [("failed".to_owned(), 1.0)]); +} + +/// A second unknown outcome, on the write sent once more, counts as +/// matching no row: two writes and no third, the row left as it is, no +/// delete, the outcome `failed` (the job ended by its own path, not a +/// panic). +#[tokio::test] +async fn a_second_unknown_outcome_ends_the_job_without_a_third_write() { + let mem = Arc::new(MemStore::default()); + mem.fail_puts + .store(true, std::sync::atomic::Ordering::Release); + let fake = Arc::new(Fake::default()); + snapshot_fixture(&fake, &music_table(), songs(2), true, None); + *fake.unknown_write.lock().unwrap() = Some(UnknownWrite::Lost); + fake.unknown_write_sticky + .store(true, std::sync::atomic::Ordering::Release); + let runtime = job_runtime(&mem); + let details = create_music_backup(&fake, &runtime).await.unwrap(); + wait_for_jobs(&runtime).await; + assert_eq!(writes(&fake).len(), 2); + let row = fake.rows.lock().unwrap()[&details.backup_arn].clone(); + assert_eq!(row.status, BackupStatus::Creating); + assert!(mem.deletes.lock().unwrap().is_empty()); + assert_eq!(job_outcomes(&runtime), [("failed".to_owned(), 1.0)]); +} + +/// A second unknown outcome whose write landed counts as matched: the first +/// FAILED write is lost with its connection (the re-read shows CREATING, so +/// the write is sent once more), the second lands and its outcome is +/// unknown too, and the second re-read shows the row FAILED under this +/// claim, so the failure path goes on to the one `delete_prefix`. The row +/// is FAILED under this generation and the prefix is the job's, so the +/// deletion is right; a job that treated the second unknown outcome as +/// matching no row would leave the prefix to the reconciler for nothing. +#[tokio::test] +async fn a_second_unknown_outcome_whose_write_landed_is_matched() { + let mem = Arc::new(MemStore::default()); + mem.fail_puts + .store(true, std::sync::atomic::Ordering::Release); + let fake = Arc::new(Fake::default()); + snapshot_fixture(&fake, &music_table(), songs(2), true, None); + *fake.unknown_write.lock().unwrap() = Some(UnknownWrite::Lost); + *fake.unknown_write_then.lock().unwrap() = Some(UnknownWrite::Applied); + let runtime = job_runtime(&mem); + let details = create_music_backup(&fake, &runtime).await.unwrap(); + wait_for_jobs(&runtime).await; + assert_eq!( + writes(&fake).iter().map(|w| w.0).collect::>(), + [JobStatus::Failed, JobStatus::Failed] + ); + let row = fake.rows.lock().unwrap()[&details.backup_arn].clone(); + assert_eq!(row.status, BackupStatus::Failed); + let prefix = BackupArn::parse(&details.backup_arn) + .unwrap() + .store_prefix(); + assert_eq!( + mem.deletes.lock().unwrap().as_slice(), + [prefix], + "the second re-read shows FAILED under this claim: the deletion follows" + ); + assert!(objects_under(&mem, &details.backup_arn).is_empty()); + assert_eq!(job_outcomes(&runtime), [("failed".to_owned(), 1.0)]); +} + +/// The pre-put check in its two places: with the heartbeat timer stopped +/// without raising the loss (a timer task that died) and two periods gone, +/// neither a data file's put nor a document's put starts, each stopping the +/// job as a lost claim naming the object. +#[tokio::test] +async fn a_stopped_heartbeat_timer_starts_no_put() { + let mem = Arc::new(MemStore::default()); + let fake = Fake::default(); + let arn = arn(ACCOUNT); + fake.rows + .lock() + .unwrap() + .insert(arn.clone(), row(&arn, BackupStatus::Creating)); + let claim: Arc = Arc::from(fake.claim_on(&arn, 7)); + let period = Duration::from_millis(50); + let heartbeat = job::heartbeat::HeartbeatHandle::start( + claim, + period, + None, + 0.0, + tokio::time::Instant::now(), + ); + heartbeat.stop_timer(); + tokio::time::sleep(3 * period).await; + assert!(!heartbeat.lost().is_cancelled()); + let shutdown = extenddb_storage::CancellationToken::new(); + let parsed = BackupArn::parse(&arn).unwrap(); + let prefix = parsed.store_prefix(); + let pipeline = job::writer::Pipeline { + store: &*mem, + prefix: &prefix, + arn: &arn, + heartbeat: &heartbeat, + shutdown: &shutdown, + metrics: None, + target_bytes: 64 * 1024 * 1024, + heartbeat_period: period, + key_prefix: "", + bucket: &store().store_root, + }; + let table = music_table(); + let key = extenddb_core::validation::RestorableKeySchema::resolve( + &table.key_schema, + &table.attribute_definitions, + ) + .unwrap(); + let stream: BoxStream<'static, Result> = + Box::pin(futures::stream::iter(songs(1).into_iter().map(Ok))); + let limits = extenddb_core::validation::RestorableLimits::from(&LimitsConfig::default()); + let stop = job::writer::write_data_files(&pipeline, stream, key, limits, None) + .await + .unwrap_err(); + assert_eq!( + stop, + job::writer::Stop::ClaimLost( + "no heartbeat confirmed the claim recently enough for data file 1".to_owned() + ) + ); + assert!(mem.puts.lock().unwrap().is_empty()); + let stop = job::put_document(&pipeline, "manifest-files.json", b"{}") + .await + .unwrap_err(); + assert_eq!( + stop, + job::writer::Stop::ClaimLost( + "no heartbeat confirmed the claim recently enough for manifest-files.json".to_owned() + ) + ); + assert!(mem.puts.lock().unwrap().is_empty()); + assert!(mem.keys().is_empty()); +} + +/// The pre-put check's clock is the claim's insert, not the handle's start: +/// with a period of 100 ms, `stale_job_after` of 300 ms (the floor's ratio) +/// and a maintenance check of 400 ms between the insert and the job, the +/// first data file's put finds the last confirmed beat four periods old and +/// is refused as a lost claim naming the file; the job ends with no store +/// or catalog access, the row left CREATING for the reconciler. A check +/// within two periods admits the put. +#[tokio::test] +async fn a_slow_maintenance_check_refuses_the_first_put() { + let (lines, _guard) = LogLines::capture(); + let period = Duration::from_millis(100); + let settings = JobSettings { + heartbeat_interval: period, + stale_job_after: 3 * period, + ..job_settings() + }; + let mem = Arc::new(MemStore::default()); + let fake = Arc::new(Fake::default()); + snapshot_fixture(&fake, &music_table(), songs(2), true, None); + *fake.maintenance_delay.lock().unwrap() = Some(4 * period); + let runtime = job_runtime(&mem).with_job_settings(settings); + let details = create_music_backup(&fake, &runtime).await.unwrap(); + wait_for_jobs(&runtime).await; + let row = fake.rows.lock().unwrap()[&details.backup_arn].clone(); + assert_eq!( + (row.status, row.failure_reason), + (BackupStatus::Creating, None), + "left to the reconciler" + ); + assert!(mem.puts.lock().unwrap().is_empty(), "no put started"); + assert!(mem.keys().is_empty()); + assert!(writes(&fake).is_empty(), "no catalog write"); + assert_eq!( + lines.containing("no heartbeat confirmed the claim"), + [format!( + "backup {}: stopped with no further store or catalog access: no heartbeat confirmed \ + the claim recently enough for data file 1; the row is left to its owner or to the \ + reconciler", + details.backup_arn + )] + ); + assert_eq!(job_outcomes(&runtime), [("failed".to_owned(), 1.0)]); + assert_eq!(runtime.free_slots(), 4); + + // A check of half a period: the first put is admitted and the backup + // completes. + let mem = Arc::new(MemStore::default()); + let fake = Arc::new(Fake::default()); + snapshot_fixture(&fake, &music_table(), songs(2), true, None); + *fake.maintenance_delay.lock().unwrap() = Some(period / 2); + let runtime = job_runtime(&mem).with_job_settings(settings); + let details = create_music_backup(&fake, &runtime).await.unwrap(); + wait_for_jobs(&runtime).await; + let row = fake.rows.lock().unwrap()[&details.backup_arn].clone(); + assert_eq!(row.status, BackupStatus::Available); + assert_eq!(objects_under(&mem, &details.backup_arn).len(), 5); +} + +/// The claim re-check before the manifest: the heartbeat answers `Lost` +/// from the sidecar's put on, with a period long enough that no timer beat +/// runs during the job, so the re-check is the one beat the job sends; it +/// finds the claim lost, the manifest is not written, the row is left +/// CREATING with its four objects and no catalog write. +#[tokio::test] +async fn the_claim_re_check_before_the_manifest_stops_a_lost_claim() { + let mem = Arc::new(MemStore::default()); + let fake = Arc::new(Fake::default()); + snapshot_fixture(&fake, &music_table(), songs(2), true, None); + let answer = Arc::clone(&fake.heartbeat_answer); + *mem.on_put.lock().unwrap() = Some(Box::new(move |key: &str| { + if key.ends_with("extenddb-manifest.sha256") { + *answer.lock().unwrap() = HeartbeatAnswer::Lost; + } + })); + let runtime = job_runtime(&mem).with_job_settings(JobSettings { + heartbeat_interval: Duration::from_secs(10), + ..job_settings() + }); + let details = create_music_backup(&fake, &runtime).await.unwrap(); + wait_for_jobs(&runtime).await; + assert_eq!( + fake.heartbeats.load(std::sync::atomic::Ordering::Acquire), + 1, + "the re-check is the one beat" + ); + let row = fake.rows.lock().unwrap()[&details.backup_arn].clone(); + assert_eq!(row.status, BackupStatus::Creating); + assert!( + writes(&fake).is_empty(), + "no catalog write after a lost claim" + ); + assert!(mem.deletes.lock().unwrap().is_empty()); + let objects = objects_under(&mem, &details.backup_arn); + assert_eq!(objects.len(), 4, "{objects:?}"); + assert!( + !objects + .iter() + .any(|k| k.ends_with("extenddb-manifest.json")), + "the manifest was not written" + ); + assert_eq!(job_outcomes(&runtime), [("failed".to_owned(), 1.0)]); +} + +// --- the reconciler's duties over the store's rows ---------------------------- + +/// A CREATING row whose owner is dead, with a complete, verified backup +/// under its prefix (the golden manifest's documents and data files of the +/// recorded sizes), in `mem` and `fake`; returns its ARN. +fn seed_dead_creating_with_manifest(fake: &Fake, mem: &MemStore, complete: bool) -> String { + use extenddb_core::backup_format::{BackupDocuments, ExtenddbManifest, ManifestHeader}; + let golden = ExtenddbManifest::from_json_bytes(include_bytes!( + "../../../core/testdata/extenddb-manifest/v1.json" + )) + .unwrap(); + let docs = BackupDocuments::build( + ManifestHeader { + backup_arn: golden.backup_arn().unwrap(), + backup_name: golden.backup_name.clone(), + source_backend: golden.source_backend.clone(), + source_extenddb_version: golden.source_extenddb_version.clone(), + snapshot: golden.snapshot.clone(), + item_order: golden.item_order, + source_table: golden.source_table.clone(), + }, + golden + .data_files + .iter() + .cloned() + .map(|entry| (entry, None)) + .collect(), + "", + &store().store_root, + ) + .unwrap(); + let arn = golden.backup_arn.clone(); + let prefix = golden.backup_arn().unwrap().store_prefix(); + for (i, file) in golden.data_files.iter().enumerate() { + let size = if complete || i == 0 { + file.bytes + } else { + file.bytes - 1 + }; + mem.insert( + &format!("{prefix}{}", file.key), + &vec![0u8; usize::try_from(size).unwrap()], + ); + } + for (name, bytes) in docs.documents() { + mem.insert(&format!("{prefix}{name}"), bytes); + } + let mut dead = row(&arn, BackupStatus::Creating); + dead.heartbeat_at = 1234.5; + dead.size_bytes = 0; + dead.item_count = 0; + fake.rows.lock().unwrap().insert(arn.clone(), dead); + fake.dead.lock().unwrap().insert(arn.clone()); + arn +} + +/// One cycle on `fake` and `mem`, the store healthy. +async fn cycle_on(fake: &Fake, runtime: &BackupRuntime) -> reconciler::CycleReport { + reconciler::cycle( + fake, + runtime, + Duration::from_secs(300), + "test", + &mut reconciler::ReconcilerState::default(), + ) + .await + .unwrap() +} + +/// A dead CREATING row whose prefix holds a complete, verified manifest is +/// AVAILABLE with the manifest's size, count and `source_table` (the row's +/// own `source_table_details_json`, the stats pass's, is replaced: it is +/// seeded here with another object so the write is seen to carry the +/// manifest's); the takeover leaves `heartbeat_at` as the dead owner wrote +/// it and heartbeats nothing; the objects stay. A row whose owner is alive +/// is left alone. +#[tokio::test] +async fn the_reconciler_resolves_a_dead_creating_row_with_a_manifest_to_available() { + let mem = Arc::new(MemStore::default()); + mem.insert(STORE_IDENTITY_KEY, &identity_file("store-1", "catalog-1")); + let fake = Fake::default(); + let dead = seed_dead_creating_with_manifest(&fake, &mem, true); + let stats_pass = serde_json::json!({"TableName": "Music", "ItemCount": 0}); + fake.rows + .lock() + .unwrap() + .get_mut(&dead) + .unwrap() + .source_table_details_json = stats_pass.clone(); + let live = arn(ACCOUNT); + fake.rows + .lock() + .unwrap() + .insert(live.clone(), row(&live, BackupStatus::Creating)); + let objects_before = mem.keys().len(); + let runtime = runtime_on(&mem, store()); + let report = cycle_on(&fake, &runtime).await; + assert_eq!( + report.creating_resolved, + [(dead.clone(), BackupStatus::Available)] + ); + assert_eq!( + fake.takeovers.lock().unwrap().as_slice(), + std::slice::from_ref(&dead) + ); + let manifest = stored_manifest(&mem, &dead); + let row = fake.rows.lock().unwrap()[&dead].clone(); + assert_eq!(row.status, BackupStatus::Available); + assert_eq!(row.size_bytes, manifest.backup_size_bytes); + assert_eq!(row.item_count, 100_000); + assert_ne!( + stats_pass, + serde_json::to_value(&manifest.source_table).unwrap() + ); + assert_eq!( + row.source_table_details_json, + serde_json::to_value(&manifest.source_table).unwrap(), + "the AVAILABLE write carries the manifest's source_table" + ); + assert_eq!(row.heartbeat_at, 1234.5, "the takeover left the heartbeat"); + assert_ne!(row.owner_generation, 7, "a fresh generation"); + assert!(row.completed_at.is_some()); + assert_eq!( + fake.heartbeats.load(std::sync::atomic::Ordering::Acquire), + 0, + "the reconciler heartbeats no takeover claim" + ); + assert_eq!(mem.keys().len(), objects_before, "nothing deleted"); + assert_eq!( + fake.rows.lock().unwrap()[&live].status, + BackupStatus::Creating + ); + assert!(mem.deletes.lock().unwrap().is_empty()); +} + +/// A dead CREATING row without a complete manifest is FAILED with the +/// owner-dead reason and `completed_at`, and the resolution itself deletes +/// nothing; the FAILED-prefix duty deletes the prefix once it runs, the +/// heartbeat the owner left being stale. Every guard of the probe has its +/// shape: no manifest; a manifest without its sidecar; a sidecar that does +/// not match; a manifest the reader refuses under a sidecar computed over +/// its bytes (a sidecar matches what was stored, not what parses); a data +/// file missing; a data file shorter than recorded; one larger than +/// recorded; a manifest naming another backup under the row's prefix. A +/// store error during the probe (not an absent object) is none of these: +/// the row stays CREATING for the next cycle, nothing written. +#[tokio::test] +async fn the_reconciler_fails_a_dead_creating_row_without_a_complete_manifest() { + for shape in [ + "none", + "no sidecar", + "sidecar mismatch", + "unparseable manifest", + "missing data file", + "short file", + "oversized file", + "foreign ARN", + ] { + let mem = Arc::new(MemStore::default()); + mem.insert(STORE_IDENTITY_KEY, &identity_file("store-1", "catalog-1")); + let fake = Fake::default(); + let mut dead = seed_dead_creating_with_manifest(&fake, &mem, shape != "short file"); + let prefix = BackupArn::parse(&dead).unwrap().store_prefix(); + let golden = stored_manifest(&mem, &dead); + match shape { + "none" => { + mem.objects + .lock() + .unwrap() + .remove(&format!("{prefix}extenddb-manifest.json")); + } + "no sidecar" => { + mem.objects + .lock() + .unwrap() + .remove(&format!("{prefix}extenddb-manifest.sha256")); + } + "sidecar mismatch" => { + mem.insert(&format!("{prefix}extenddb-manifest.sha256"), b"0000"); + } + "unparseable manifest" => { + use sha2::Digest as _; + let garbage = b"{\"format_version\":1,\"backup_arn\":"; + mem.insert(&format!("{prefix}extenddb-manifest.json"), garbage); + let sidecar = format!( + "{} extenddb-manifest.json\n", + hex::encode(sha2::Sha256::digest(garbage)) + ); + mem.insert( + &format!("{prefix}extenddb-manifest.sha256"), + sidecar.as_bytes(), + ); + } + "missing data file" => { + mem.objects + .lock() + .unwrap() + .remove(&format!("{prefix}{}", golden.data_files[1].key)); + } + "oversized file" => { + let key = format!("{prefix}{}", golden.data_files[1].key); + let bytes = usize::try_from(golden.data_files[1].bytes).unwrap() + 1; + mem.insert(&key, &vec![0u8; bytes]); + } + "foreign ARN" => { + // The golden backup's objects, whole, under another row's + // prefix: that row's manifest names the golden backup. + let other = arn_n(3); + let other_prefix = BackupArn::parse(&other).unwrap().store_prefix(); + let objects: Vec<(String, Vec)> = mem + .objects + .lock() + .unwrap() + .iter() + .filter(|(k, _)| k.starts_with(&prefix)) + .map(|(k, v)| (k.clone(), v.clone())) + .collect(); + for (k, v) in objects { + mem.objects.lock().unwrap().remove(&k); + mem.insert(&format!("{other_prefix}{}", &k[prefix.len()..]), &v); + } + fake.rows.lock().unwrap().remove(&dead); + fake.dead.lock().unwrap().remove(&dead); + let mut r = row(&other, BackupStatus::Creating); + r.heartbeat_at = 1234.5; + fake.rows.lock().unwrap().insert(other.clone(), r); + fake.dead.lock().unwrap().insert(other.clone()); + dead = other; + } + _ => {} + } + let objects_before = mem.keys().len(); + // Under the maintenance claim, so the FAILED-prefix duty of the same + // cycle does not run and the resolution's own effect is visible. + *fake.maintenance_held.lock().unwrap() = true; + let runtime = runtime_on(&mem, store()); + let report = cycle_on(&fake, &runtime).await; + assert_eq!( + report.creating_resolved, + [(dead.clone(), BackupStatus::Failed)], + "{shape}" + ); + assert!(report.failed_prefixes_deleted.is_empty()); + let row = fake.rows.lock().unwrap()[&dead].clone(); + assert_eq!(row.status, BackupStatus::Failed, "{shape}"); + assert_eq!( + row.failure_reason.as_deref(), + Some(reconciler::OWNER_DEAD_NO_MANIFEST) + ); + assert!(row.completed_at.is_some()); + assert_eq!(row.heartbeat_at, 1234.5); + assert_eq!(mem.keys().len(), objects_before, "{shape}: nothing deleted"); + assert!(mem.deletes.lock().unwrap().is_empty(), "{shape}"); + + // The next cycle without the claim: the FAILED row's heartbeat is + // stale (the dead owner's), the prefix lists objects, so it is + // deleted. + *fake.maintenance_held.lock().unwrap() = false; + let report = cycle_on(&fake, &runtime).await; + assert_eq!( + report.failed_prefixes_deleted, + std::slice::from_ref(&dead), + "{shape}" + ); + assert!(objects_under(&mem, &dead).is_empty(), "{shape}"); + assert_eq!(mem.keys(), [STORE_IDENTITY_KEY]); + } + + // A store error during the probe: the row is taken over and left + // CREATING, no status write, nothing deleted; the next cycle, with the + // store answering, resolves it. + let mem = Arc::new(MemStore::default()); + mem.insert(STORE_IDENTITY_KEY, &identity_file("store-1", "catalog-1")); + let fake = Fake::default(); + let dead = seed_dead_creating_with_manifest(&fake, &mem, true); + *mem.fail_gets_under.lock().unwrap() = Some(BackupArn::parse(&dead).unwrap().store_prefix()); + let runtime = runtime_on(&mem, store()); + let report = cycle_on(&fake, &runtime).await; + assert!(report.creating_resolved.is_empty()); + assert!( + report.creating_left_to_owner.is_empty(), + "a probe that failed is neither resolved nor left to an owner" + ); + assert_eq!( + fake.takeovers.lock().unwrap().as_slice(), + std::slice::from_ref(&dead) + ); + let row = fake.rows.lock().unwrap()[&dead].clone(); + assert_eq!( + row.status, + BackupStatus::Creating, + "left for the next cycle" + ); + assert!(fake.status_writes.lock().unwrap().is_empty()); + assert!(mem.deletes.lock().unwrap().is_empty()); + *mem.fail_gets_under.lock().unwrap() = None; + let report = cycle_on(&fake, &runtime).await; + assert_eq!(report.creating_resolved, [(dead, BackupStatus::Available)]); +} + +/// A dead CREATING row whose status write fails on the claim connection is +/// counted neither as resolved nor as left to its owner: the row stays +/// CREATING for the next cycle (its takeover done, one write attempted), +/// nothing deleted; with the connection back the next cycle resolves it. +#[tokio::test] +async fn a_dead_creating_row_whose_status_write_errors_is_left_for_the_next_cycle() { + let mem = Arc::new(MemStore::default()); + mem.insert(STORE_IDENTITY_KEY, &identity_file("store-1", "catalog-1")); + let fake = Fake::default(); + let dead = seed_dead_creating_with_manifest(&fake, &mem, true); + fake.status_write_error + .store(true, std::sync::atomic::Ordering::Release); + let runtime = runtime_on(&mem, store()); + let report = cycle_on(&fake, &runtime).await; + assert!(report.creating_resolved.is_empty(), "{report:?}"); + assert!(report.creating_left_to_owner.is_empty(), "{report:?}"); + assert!(!report.acted(), "nothing changed"); + assert_eq!( + fake.takeovers.lock().unwrap().as_slice(), + std::slice::from_ref(&dead) + ); + assert_eq!(writes(&fake), [(JobStatus::Available, None)]); + let row = fake.rows.lock().unwrap()[&dead].clone(); + assert_eq!(row.status, BackupStatus::Creating); + assert!(mem.deletes.lock().unwrap().is_empty()); + fake.status_write_error + .store(false, std::sync::atomic::Ordering::Release); + let report = cycle_on(&fake, &runtime).await; + assert_eq!(report.creating_resolved, [(dead, BackupStatus::Available)]); +} + +/// Within one cycle the dead-owner duty runs before the FAILED-prefix duty, +/// so a row the cycle just failed, whose heartbeat the dead owner left +/// stale, loses its prefix in that same cycle (the design states this as a +/// property of the cycle, not a convenience). +#[tokio::test] +async fn a_row_failed_by_the_cycle_loses_its_prefix_in_that_cycle() { + let mem = Arc::new(MemStore::default()); + mem.insert(STORE_IDENTITY_KEY, &identity_file("store-1", "catalog-1")); + let fake = Fake::default(); + let dead = seed_dead_creating_with_manifest(&fake, &mem, false); + assert!(!objects_under(&mem, &dead).is_empty()); + let runtime = runtime_on(&mem, store()); + let report = cycle_on(&fake, &runtime).await; + assert_eq!( + report.creating_resolved, + [(dead.clone(), BackupStatus::Failed)] + ); + assert_eq!( + report.failed_prefixes_deleted, + std::slice::from_ref(&dead), + "the same cycle" + ); + assert!(objects_under(&mem, &dead).is_empty()); + assert_eq!( + fake.rows.lock().unwrap()[&dead].status, + BackupStatus::Failed + ); +} + +/// A dead CREATING row whose status write matches no row (another +/// reconciler took the row over in the gap before this one's write, item +/// 29's race on MongoDB and Cassandra) is left to that owner: counted as +/// such and not as resolved, the row still CREATING under the other +/// generation, nothing deleted. +#[tokio::test] +async fn a_dead_creating_row_taken_over_by_another_is_left_to_that_owner() { + let mem = Arc::new(MemStore::default()); + mem.insert(STORE_IDENTITY_KEY, &identity_file("store-1", "catalog-1")); + let fake = Fake::default(); + let dead = seed_dead_creating_with_manifest(&fake, &mem, true); + fake.takeover_lost_to_another + .store(true, std::sync::atomic::Ordering::Release); + let runtime = runtime_on(&mem, store()); + let report = cycle_on(&fake, &runtime).await; + assert!(report.creating_resolved.is_empty()); + assert_eq!(report.creating_left_to_owner, std::slice::from_ref(&dead)); + assert!(!report.acted(), "nothing changed"); + let writes = fake.status_writes.lock().unwrap().clone(); + assert_eq!(writes.len(), 1); + assert_eq!(writes[0].to, JobStatus::Available); + let row = fake.rows.lock().unwrap()[&dead].clone(); + assert_eq!(row.status, BackupStatus::Creating); + assert_ne!(row.owner_generation, writes[0].generation); + assert!(mem.deletes.lock().unwrap().is_empty()); +} + +/// The deleting duties and their gates. A FAILED row's prefix is deleted +/// only once its heartbeat is older than `stale_job_after` and only while +/// no maintenance claim is held; a DELETED row's objects are swept; a FAILED +/// or DELETED row whose prefix is empty is removed once its `completed_at` +/// is older than `failed_row_retention` (a prefix emptied earlier in the +/// cycle counts), and kept while the retention has not passed or while its +/// prefix still lists objects (a FAILED row past its retention whose +/// heartbeat is fresh: no duty empties its prefix, so the tombstone duty +/// keeps the row); an AVAILABLE row is no tombstone, whatever its clock and +/// prefix. The dead-CREATING duty runs under the maintenance claim; +/// the deleting duties and the registrations sweep do not. With the store +/// unhealthy none of them runs. +#[tokio::test] +async fn the_reconciler_deletes_failed_and_deleted_prefixes_and_removes_tombstones_on_their_clocks() +{ + let mem = Arc::new(MemStore::default()); + mem.insert(STORE_IDENTITY_KEY, &identity_file("store-1", "catalog-1")); + let fake = Fake::default(); + let now = seconds_since_epoch(); + let stale_failed = arn_n(1); + let fresh_failed = arn_n(2); + let deleted_with_objects = arn_n(3); + let old_failed_tombstone = arn_n(4); + let old_deleted_tombstone = arn_n(5); + let young_tombstone = arn_n(6); + let old_tombstone_with_objects = arn_n(7); + let old_failed_with_objects_and_a_live_heartbeat = arn_n(8); + for (a, status, heartbeat_at, completed_at, objects) in [ + ( + &stale_failed, + BackupStatus::Failed, + now - 1000.0, + Some(now - 1000.0), + true, + ), + (&fresh_failed, BackupStatus::Failed, now, Some(now), true), + ( + &deleted_with_objects, + BackupStatus::Deleted, + now - 1000.0, + Some(now - 1000.0), + true, + ), + ( + &old_failed_tombstone, + BackupStatus::Failed, + now - 1000.0, + Some(now - 1000.0), + false, + ), + ( + &old_deleted_tombstone, + BackupStatus::Deleted, + now - 1000.0, + Some(now - 1000.0), + false, + ), + ( + &young_tombstone, + BackupStatus::Failed, + now - 1000.0, + Some(now - 10.0), + false, + ), + ( + &old_tombstone_with_objects, + BackupStatus::Deleted, + now - 1000.0, + Some(now - 1000.0), + true, + ), + ( + &old_failed_with_objects_and_a_live_heartbeat, + BackupStatus::Failed, + now, + Some(now - 1000.0), + true, + ), + ] { + let mut r = row(a, status); + r.heartbeat_at = heartbeat_at; + r.completed_at = completed_at; + fake.rows.lock().unwrap().insert(a.clone(), r); + if objects { + let prefix = BackupArn::parse(a).unwrap().store_prefix(); + mem.insert(&format!("{prefix}data/000001.json.gz"), b"x"); + } + } + let retention = Duration::from_secs(600); + let settings = JobSettings { + failed_row_retention: retention, + ..job_settings() + }; + + // Unhealthy (the root's identity file names another store while the + // store answers): no duty over the store's rows runs, the dead-CREATING + // one included. + let dead = seed_dead_creating_with_manifest(&fake, &mem, true); + mem.insert( + STORE_IDENTITY_KEY, + &identity_file("another-store", "catalog-1"), + ); + let runtime = runtime_on(&mem, store()).with_job_settings(settings); + let report = cycle_on(&fake, &runtime).await; + assert_eq!(report.store_healthy, Some(false)); + assert!(report.creating_resolved.is_empty()); + assert!(report.failed_prefixes_deleted.is_empty()); + assert!(report.deleted_prefixes_swept.is_empty()); + assert!(report.tombstones_removed.is_empty()); + assert!(mem.deletes.lock().unwrap().is_empty()); + assert_eq!(fake.rows.lock().unwrap().len(), 9); + assert_eq!( + fake.rows.lock().unwrap()[&dead].status, + BackupStatus::Creating + ); + mem.insert(STORE_IDENTITY_KEY, &identity_file("store-1", "catalog-1")); + + // Under the maintenance claim, the store healthy again: the + // dead-CREATING duty runs, the deleting duties and the registrations + // sweep (which the backend has something for) do not. + fake.swept_registrations.lock().unwrap().push(arn_n(9)); + *fake.maintenance_held.lock().unwrap() = true; + let report = cycle_on(&fake, &runtime).await; + assert_eq!(report.store_healthy, Some(true)); + assert!(report.maintenance_held); + assert_eq!(report.creating_resolved, [(dead, BackupStatus::Available)]); + assert!(report.failed_prefixes_deleted.is_empty()); + assert!(report.deleted_prefixes_swept.is_empty()); + assert!(report.tombstones_removed.is_empty()); + assert!( + report.registrations_swept.is_empty(), + "the sweep does not run under the claim" + ); + assert!(mem.deletes.lock().unwrap().is_empty()); + assert_eq!(fake.rows.lock().unwrap().len(), 9); + + // An AVAILABLE row past the retention with an empty prefix: a tombstone + // is a FAILED or DELETED row and nothing else. It enters the catalog + // here, older than the row the cycle above resolved, because the + // store's check heads the newest AVAILABLE row's manifest and this row + // has none. + let old_available_with_an_empty_prefix = + format!("arn:aws:dynamodb:{REGION}:{ACCOUNT}:table/Music/backup/01700000000000-00000000"); + let mut r = row(&old_available_with_an_empty_prefix, BackupStatus::Available); + r.heartbeat_at = now - 1000.0; + r.completed_at = Some(now - 1000.0); + fake.rows + .lock() + .unwrap() + .insert(old_available_with_an_empty_prefix.clone(), r); + + // The claim released: each duty on its clock. + *fake.maintenance_held.lock().unwrap() = false; + let report = cycle_on(&fake, &runtime).await; + assert_eq!( + report.failed_prefixes_deleted, + std::slice::from_ref(&stale_failed) + ); + assert_eq!( + report.deleted_prefixes_swept, + [ + deleted_with_objects.clone(), + old_tombstone_with_objects.clone() + ] + ); + // The tombstone duty runs after the two deleting duties, so a prefix + // they emptied this cycle counts as empty here. + let mut removed = report.tombstones_removed.clone(); + removed.sort(); + assert_eq!( + removed, + [ + stale_failed.clone(), + deleted_with_objects.clone(), + old_failed_tombstone.clone(), + old_deleted_tombstone.clone(), + old_tombstone_with_objects.clone(), + ] + ); + assert_eq!(report.registrations_swept, [arn_n(9)]); + assert!(report.acted()); + assert_eq!( + objects_under(&mem, &fresh_failed).len(), + 1, + "a FAILED row with a live heartbeat keeps its prefix" + ); + assert!(objects_under(&mem, &stale_failed).is_empty()); + assert!(objects_under(&mem, &deleted_with_objects).is_empty()); + assert_eq!( + objects_under(&mem, &old_failed_with_objects_and_a_live_heartbeat).len(), + 1, + "no duty empties the prefix of a FAILED row with a live heartbeat" + ); + { + let rows = fake.rows.lock().unwrap(); + assert!(rows.contains_key(&fresh_failed), "its heartbeat is live"); + assert!( + rows.contains_key(&young_tombstone), + "its retention has not passed" + ); + assert!( + rows.contains_key(&old_failed_with_objects_and_a_live_heartbeat), + "its prefix still lists an object" + ); + assert!( + rows.contains_key(&old_available_with_an_empty_prefix), + "AVAILABLE is not a tombstone status" + ); + assert_eq!( + rows[&old_available_with_an_empty_prefix].status, + BackupStatus::Available + ); + assert_eq!(rows.len(), 5, "the four kept and the resolved CREATING row"); + } + assert!( + !fake + .remove_attempts + .lock() + .unwrap() + .contains(&old_available_with_an_empty_prefix), + "the duty asks the catalog to remove FAILED and DELETED rows only; the backends would \ + refuse an AVAILABLE one, and the duty does not lean on that" + ); + + // The next cycle has nothing left to do. + let report = cycle_on(&fake, &runtime).await; + assert!(report.tombstones_removed.is_empty()); + assert!(report.deleted_prefixes_swept.is_empty()); + assert!(report.failed_prefixes_deleted.is_empty()); + assert!(report.creating_resolved.is_empty()); +} diff --git a/crates/engine/src/lib.rs b/crates/engine/src/lib.rs index 2fb8b63a5..f56df9813 100755 --- a/crates/engine/src/lib.rs +++ b/crates/engine/src/lib.rs @@ -9,9 +9,10 @@ mod backup; pub use backup::{ - BackupRuntime, ConfiguredStore, HeldClaim, LOCAL_CHECK_TIMEOUT, S3_CHECK_TIMEOUT, - STORE_IDENTITY_KEY, Spool, StoreHealth, StoreHealthReport, StoreIdentityFile, StoreProblem, - check_configured_store, maintenance, read_identity_bounded, reconciler, store_identity, + BackupRuntime, ConfiguredStore, HeldClaim, JobSettings, KillPoint, LOCAL_CHECK_TIMEOUT, + S3_CHECK_TIMEOUT, STORE_IDENTITY_KEY, Spool, StoreHealth, StoreHealthReport, StoreIdentityFile, + StoreProblem, check_configured_store, maintenance, read_identity_bounded, reconciler, + store_identity, }; mod batch_get_item; mod batch_write_item; diff --git a/crates/server/Cargo.toml b/crates/server/Cargo.toml index c6e58acca..1b58f02b5 100755 --- a/crates/server/Cargo.toml +++ b/crates/server/Cargo.toml @@ -9,6 +9,9 @@ license.workspace = true [features] mongodb-test-hooks = [] +# The backup job's kill-point pauses (`EXTENDDB_BACKUP_PAUSE`), for the +# binary-level crash tests. +backup-test-hooks = ["extenddb-engine/backup-test-hooks"] [dependencies] extenddb-cache = { workspace = true } diff --git a/crates/server/src/backup_store.rs b/crates/server/src/backup_store.rs index b0962e9b4..0a08a688b 100644 --- a/crates/server/src/backup_store.rs +++ b/crates/server/src/backup_store.rs @@ -75,7 +75,9 @@ use std::sync::Arc; use extenddb_backup_store::{BackupStore, BackupStoreKind, StoreError}; use extenddb_config::backup::{BackupConfig, EffectiveStore, Unconfigured}; pub use extenddb_engine::LOCAL_CHECK_TIMEOUT; -use extenddb_engine::{BackupRuntime, ConfiguredStore, Spool, StoreProblem, store_identity}; +use extenddb_engine::{ + BackupRuntime, ConfiguredStore, JobSettings, Spool, StoreProblem, store_identity, +}; use extenddb_storage::backup_catalog::{StoreIdentity, StoreKind}; use extenddb_storage::{BackupCatalog, StorageEngine}; @@ -214,10 +216,28 @@ pub async fn configure( Ok((setup.runtime, setup.spool_lock)) } -/// Build the backup runtime for this process: the job slots and, when the -/// configuration and the backend select one, the store with its health; -/// otherwise why there is none. `catalog` is the catalog the identity check -/// reads and the rows rule of the module doc consults. +/// The `[backup]` settings the jobs and the reconciler read, as the +/// configuration resolves them: the heartbeat period is +/// [`BackupConfig::effective_heartbeat_interval_secs`]. +#[must_use] +pub fn job_settings(backup: &BackupConfig) -> JobSettings { + JobSettings { + data_file_target_bytes: backup.data_file_target_bytes, + heartbeat_interval: std::time::Duration::from_secs( + backup.effective_heartbeat_interval_secs(), + ), + stale_job_after: std::time::Duration::from_secs(backup.stale_job_after_secs), + failed_row_retention: std::time::Duration::from_secs(backup.failed_row_retention_secs), + } +} + +/// Build the backup runtime for this process: the job slots, the job +/// settings and, when the configuration and the backend select one, the +/// store with its health and the names the DynamoDB-shaped manifests carry +/// for it; otherwise why there is none. `catalog` is the catalog the +/// identity check reads and the rows rule of the module doc consults. The +/// server adds the metrics collector, the shutdown token and the source +/// names to the runtime it gets back. /// /// # Errors /// @@ -231,7 +251,7 @@ pub async fn configure_described( ) -> anyhow::Result { let runtime = BackupRuntime::new(backup.max_concurrent_jobs) .with_legacy_create_only(backend == "cassandra") - .with_stale_job_after(std::time::Duration::from_secs(backup.stale_job_after_secs)); + .with_job_settings(job_settings(backup)); let resolved = backup .resolve_store(database_file) .map_err(|e| anyhow::anyhow!("[backup] configuration: {e}"))?; @@ -324,6 +344,15 @@ pub async fn configure_described( BackupStoreKind::S3 => StoreKind::S3, BackupStoreKind::Disabled => unreachable!("open_described never returns Disabled"), }; + // `manifest-summary.json`'s `s3Bucket` and the key prefix every key of + // the DynamoDB-shaped manifests starts with: the bucket and the + // configured prefix on S3, the root path and no prefix on a filesystem + // store. + let (bucket, key_prefix) = match (kind, &config.s3) { + (StoreKind::S3, Some(s3)) => (s3.bucket.clone(), s3.prefix.clone()), + _ => (opened.root.clone(), String::new()), + }; + let configured = |store: ConfiguredStore| store.with_manifest_names(bucket, key_prefix); // The spool and its lock: configuration errors on every backend. let (spool, spool_lock) = match (resolved.spool_path, caller) { @@ -342,19 +371,28 @@ pub async fn configure_described( // listener is up, in the reconciler's first cycle. A read-only command // runs no check at all. if caller == Caller::ReadOnlyCommand { - let configured = ConfiguredStore::checking(kind, opened.root, opened.store, spool); + let configured = configured(ConfiguredStore::checking( + kind, + opened.root, + opened.store, + spool, + )); return Ok(StoreSetup::with_store(runtime, configured, spool_lock)); } if kind == StoreKind::S3 { if caller == Caller::Server { tracing::info!( "backup store: {kind} {}{spool_line}; its validation and identity check run \ - after the listener is up, and CreateBackup writes in-catalog backups in this \ - release", + after the listener is up, and CreateBackup writes to it once a check passes", opened.root ); } - let configured = ConfiguredStore::checking(kind, opened.root, opened.store, spool); + let configured = configured(ConfiguredStore::checking( + kind, + opened.root, + opened.store, + spool, + )); return Ok(StoreSetup::with_store(runtime, configured, spool_lock)); } @@ -391,15 +429,19 @@ pub async fn configure_described( opened.root, LOCAL_CHECK_TIMEOUT.as_secs() ); - let configured = ConfiguredStore::checking(kind, opened.root, opened.store, spool); + let configured = configured(ConfiguredStore::checking( + kind, + opened.root, + opened.store, + spool, + )); return Ok(StoreSetup::with_store(runtime, configured, spool_lock)); } }; match &outcome { Ok(identity) => tracing::info!( - "backup store: {kind} {} (store id {}){spool_line}; CreateBackup writes in-catalog \ - backups in this release, and the store serves restores and deletes of the backups \ - it holds", + "backup store: {kind} {} (store id {}){spool_line}; CreateBackup writes to it, and \ + it serves restores and deletes of the backups it holds", opened.root, identity.store_id ), @@ -422,7 +464,13 @@ pub async fn configure_described( rows_note(defaulted) ), } - let configured = ConfiguredStore::new(kind, opened.root, opened.store, spool, outcome); + let configured = configured(ConfiguredStore::new( + kind, + opened.root, + opened.store, + spool, + outcome, + )); Ok(StoreSetup::with_store(runtime, configured, spool_lock)) } @@ -810,7 +858,7 @@ mod tests { use extenddb_storage::BoxedFuture; use extenddb_storage::backup_catalog::{ BackupKey, BackupRow, BackupStatus, DeleteStart, JobClaim, MaintenanceClaim, RestoreJobRow, - RestoreTargetKey, TimeBounds, + RestoreTargetKey, StoreIdentity, TimeBounds, }; use extenddb_storage::error::StorageError; use futures::stream::BoxStream; @@ -842,6 +890,22 @@ mod tests { ) -> BoxedFuture<'a, Result, StorageError>> { unused() } + fn claim_dead_backup<'a>( + &'a self, + _key: &'a BackupKey, + _stale_after: std::time::Duration, + ) -> BoxedFuture<'a, Result>, StorageError>> { + unused() + } + fn list_backup_rows_older_than<'a>( + &'a self, + _status: BackupStatus, + _store: &'a StoreIdentity, + _clock: extenddb_storage::backup_catalog::StaleClock, + _age: std::time::Duration, + ) -> BoxStream<'a, Result> { + Box::pin(futures::stream::empty()) + } fn list_backup_rows<'a>( &'a self, _status: Option, diff --git a/crates/server/src/metrics_endpoint.rs b/crates/server/src/metrics_endpoint.rs index d625c5d92..52bb5c282 100755 --- a/crates/server/src/metrics_endpoint.rs +++ b/crates/server/src/metrics_endpoint.rs @@ -363,4 +363,45 @@ mod tests { "GSI-name dimension must be retained, got {dims:?}" ); } + + /// The backup job's metric names round-trip through the store's string + /// column: a row written under each name renders as that name and not + /// as the fallback `SystemErrors`, with its outcome or store-call + /// dimension kept. + #[test] + fn backup_job_metric_names_render_from_the_store() { + use extenddb_core::metrics::MetricName; + let names = [ + MetricName::BackupJobs, + MetricName::BackupJobDuration, + MetricName::BackupBytesUploaded, + MetricName::BackupSnapshotAge, + MetricName::BackupStoreErrors, + ]; + let rows: Vec = names + .iter() + .map(|name| MetricsRow { + bucket: time::OffsetDateTime::UNIX_EPOCH, + metric: name.to_string(), + table_name: None, + index_name: None, + operation: Some("succeeded".to_owned()), + sum: 1.0, + count: 1, + min: 1.0, + max: 1.0, + }) + .collect(); + let (snapshots, _buckets) = aggregate_rows(&rows, 60, TimeWindow::Last5Minutes); + let mut rendered: Vec = snapshots.iter().map(|s| s.metric).collect(); + rendered.sort_by_key(|m| m.to_string()); + let mut expected = names.to_vec(); + expected.sort_by_key(|m| m.to_string()); + assert_eq!(rendered, expected); + assert!(snapshots.iter().all(|s| { + s.dimensions + .iter() + .any(|d| matches!(d, Dimension::Operation(o) if o == "succeeded")) + })); + } } diff --git a/crates/server/src/serve.rs b/crates/server/src/serve.rs index 96d951dcc..15b6fb7f2 100644 --- a/crates/server/src/serve.rs +++ b/crates/server/src/serve.rs @@ -632,6 +632,11 @@ async fn serve_inner(params: ServeParams, port: u16) -> anyhow::Result<()> { // checked. A configuration error stops startup here; a transport or // identity failure starts the server with the store unhealthy // (`backup_store` module doc). The spool lock lives until shutdown. + // Workers and backup jobs run until `shutdown` is cancelled, which + // happens after the HTTP server stops accepting; the jobs are cancelled + // on a child token first, so their failure paths run and record their + // metrics before the workers (the metrics flush among them) are drained. + let shutdown = CancellationToken::new(); let backend_name = app_config.storage.backend.clone(); let (backup_runtime, _spool_lock) = crate::backup_store::configure( &app_config.backup, @@ -641,7 +646,12 @@ async fn serve_inner(params: ServeParams, port: u16) -> anyhow::Result<()> { crate::backup_store::Caller::Server, ) .await?; - let backup_runtime = Arc::new(backup_runtime); + let backup_runtime = Arc::new( + backup_runtime + .with_metrics(metrics.clone()) + .with_shutdown(shutdown.child_token()) + .with_source(&backend_name, build.version), + ); let backup_runtime_for_reconciler = Arc::clone(&backup_runtime); let state = AppState { storage: storage.clone(), @@ -671,11 +681,9 @@ async fn serve_inner(params: ServeParams, port: u16) -> anyhow::Result<()> { docs_store, }; - // Workers run until `shutdown` is cancelled, which happens after the HTTP - // server stops accepting. Handles are collected so shutdown can drain them - // (the metrics flush worker persists its final bucket on the way out) - // instead of leaving the work to a runtime drop. - let shutdown = CancellationToken::new(); + // Handles are collected so shutdown can drain the workers (the metrics + // flush worker persists its final bucket on the way out) instead of + // leaving the work to a runtime drop. let mut worker_handles = vec![ // D-22: Poll log_level from the settings table. tokio::spawn(workers::poll_log_level( @@ -749,8 +757,14 @@ async fn serve_inner(params: ServeParams, port: u16) -> anyhow::Result<()> { let server_result = crate::start_server(listener, state, pid_file, tls_config).await; - // The HTTP server has stopped accepting; drain the workers so in-flight - // cycles finish and the metrics flush worker writes its final bucket. + // The HTTP server has stopped accepting; the backup jobs are cancelled + // and awaited first, within their own bound (a cancelled job's failure + // path deletes its prefix, which on S3 is what sweeps an upload still + // in flight), then the workers, so in-flight cycles finish and the + // metrics flush worker writes its final bucket. + backup_runtime_for_reconciler + .drain_jobs(JOB_DRAIN_TIMEOUT) + .await; drain_workers(&shutdown, worker_handles).await; server_result?; @@ -758,6 +772,11 @@ async fn serve_inner(params: ServeParams, port: u16) -> anyhow::Result<()> { Ok(()) } +/// How long shutdown waits for the backup jobs' failure paths: a +/// `delete_prefix` of a large S3 prefix takes longer than the workers' +/// drain allows, and this runs before the binary's own runtime bound. +const JOB_DRAIN_TIMEOUT: Duration = Duration::from_secs(30); + /// Cancel the shutdown token and wait for every worker to return. /// /// Bounded by `DRAIN_TIMEOUT` so a worker stuck inside a backend call cannot diff --git a/crates/storage-cassandra/src/backup_catalog.rs b/crates/storage-cassandra/src/backup_catalog.rs index 1cfab090f..76d1ada05 100644 --- a/crates/storage-cassandra/src/backup_catalog.rs +++ b/crates/storage-cassandra/src/backup_catalog.rs @@ -22,7 +22,12 @@ //! later write by the job is a lightweight transaction `IF owner_generation = //! `, and a heartbeat that does not apply means another owner has //! the row. Times are the coordinator's clock; staleness is measured against -//! it (`coordinator_now_millis`), never an application host's. +//! it (`coordinator_now_millis`), never an application host's. A dead +//! owner's row (`claim_dead_backup`) is taken over by an LWT on the +//! generation, the status and the heartbeat as read at `LOCAL_SERIAL`, the +//! heartbeat left as the owner wrote it. `sweep_restore_registrations` +//! removes, entry by entry under an LWT, the `active_restores` entries whose +//! target is no longer a CREATING restore target of the backup. //! //! DeleteBackup's lock is the lightweight transaction on the `backups_by_arn` //! row, `IF backup_status = 'AVAILABLE' AND active_restores = null`, on the @@ -69,8 +74,9 @@ use cdrs_tokio::types::value::Value; use extenddb_core::backup_format::BackupArn; use extenddb_storage::backup_catalog::{ BackupCatalog, BackupKey, BackupRow, BackupStatus, DeleteStart, Heartbeat, JobClaim, JobRow, - JobStatus, MaintenanceClaim, RestoreJobRow, RestoreKind, RestoreTargetKey, StatusFields, - StoreIdentity, StoreKind, TimeBounds, WriteOutcome, billing_mode_name, new_generation, + JobStatus, MaintenanceClaim, RestoreJobRow, RestoreKind, RestoreTargetKey, StaleClock, + StatusFields, StoreIdentity, StoreKind, TimeBounds, WriteOutcome, billing_mode_name, + new_generation, }; use extenddb_storage::error::StorageError; use futures::future::BoxFuture; @@ -337,6 +343,10 @@ impl JobClaim for CassandraBackupClaim { if fields.completed { assignments.push("completed_at = toTimestamp(now())".to_owned()); } + if let Some(source_table) = fields.source_table_details_json.as_ref() { + assignments.push("source_table_details_json = ?".to_owned()); + values.push(Value::from(source_table.to_string())); + } values.extend([ Value::from(self.key.account_id.as_str()), Value::from(self.key.backup_arn.as_str()), @@ -1072,6 +1082,237 @@ impl BackupCatalog for CassandraEngine { }) } + fn claim_dead_backup<'a>( + &'a self, + key: &'a BackupKey, + stale_after: Duration, + ) -> BoxFuture<'a, Result>, StorageError>> { + Box::pin(async move { + // The owner is dead when the row's heartbeat is older than the + // window on the coordinator's clock. The row is read at + // LOCAL_SERIAL, so a heartbeat an LWT just wrote is seen, and the + // takeover is conditioned on the generation, the status and the + // heartbeat as read: a beat landing between the read and the + // write (the owner alive after all), or another taker's + // generation, leaves the row and answers `None`. The heartbeat is + // not refreshed: the FAILED-prefix window counts from the dead + // owner's last beat. + let catalog = self.catalog_keyspace(); + let query = format!( + "SELECT backup_status, owner_generation, heartbeat_at, store_kind, store_root, \ + store_id FROM {catalog}.backups_by_arn WHERE account_id = ? AND backup_arn = ?" + ); + let rows = crate::cassandra_util::query_rows_serial::( + &self.session, + &query, + cdrs_tokio::query_values!(key.account_id.as_str(), key.backup_arn.as_str()), + "dead backup row", + ) + .await?; + let Some(row) = rows.first() else { + return Ok(None); + }; + let status: Option = row.get_by_name("backup_status").ok().flatten(); + if status.as_deref() != Some(BackupStatus::Creating.as_str()) { + return Ok(None); + } + // An in-catalog row (no store) is the legacy duty's. + let store_kind: Option = row.get_by_name("store_kind").ok().flatten(); + let Some(store_kind) = store_kind.as_deref().and_then(StoreKind::parse) else { + return Ok(None); + }; + let store = StoreIdentity { + store_kind, + store_root: crate::cassandra_util::get_column( + row, + "store_root", + "dead backup row", + )?, + store_id: crate::cassandra_util::get_column(row, "store_id", "dead backup row")?, + }; + let owner_generation: Option = row.get_by_name("owner_generation").ok().flatten(); + let heartbeat_at: Option = row.get_by_name("heartbeat_at").ok().flatten(); + let now = coordinator_now_millis(self).await?; + let stale_ms = i64::try_from(stale_after.as_millis()).unwrap_or(i64::MAX); + if heartbeat_at.is_some_and(|at| at >= now.saturating_sub(stale_ms)) { + return Ok(None); + } + let generation = new_generation(); + let take_over = format!( + "UPDATE {catalog}.backups_by_arn SET owner_generation = ? \ + WHERE account_id = ? AND backup_arn = ? \ + IF owner_generation = ? AND backup_status = 'CREATING' AND heartbeat_at = ?" + ); + let taken = conditional_write( + &self.session, + &take_over, + cdrs_tokio::query_values!( + generation_column(generation), + key.account_id.as_str(), + key.backup_arn.as_str(), + owner_generation, + heartbeat_at + ), + ) + .await?; + let claimed = match taken { + WriteOutcome::Matched => true, + WriteOutcome::NotMatched => false, + // A takeover whose answer was lost may have committed: a row + // carrying this call's generation is this call's claim. + WriteOutcome::Unknown => { + let rows = crate::cassandra_util::query_rows_serial::( + &self.session, + &format!( + "SELECT owner_generation FROM {catalog}.backups_by_arn \ + WHERE account_id = ? AND backup_arn = ?" + ), + cdrs_tokio::query_values!(key.account_id.as_str(), key.backup_arn.as_str()), + "dead backup row", + ) + .await?; + let current: Option = rows + .first() + .and_then(|r| r.get_by_name("owner_generation").ok().flatten()); + current == Some(generation_column(generation)) + } + }; + if !claimed { + return Ok(None); + } + Ok(Some(Box::new(CassandraBackupClaim { + engine: self.clone(), + key: key.clone(), + store, + generation, + }) as Box)) + }) + } + + fn list_backup_rows_older_than<'a>( + &'a self, + status: BackupStatus, + store: &'a StoreIdentity, + clock: StaleClock, + age: Duration, + ) -> BoxStream<'a, Result> { + rows_stream(async move { + // The coordinator's clock is read once, before the status + // partition is scanned, so a row's column is compared with the + // clock that stamped it and never with this host's; a heartbeat + // written while the rows are re-read is later than `now` and the + // row is not listed. A row with no `completed_at` has no age on + // that clock and is not listed. + let now = coordinator_now_millis(self).await?; + let age_ms = i64::try_from(age.as_millis()).unwrap_or(i64::MAX); + let cutoff = now.saturating_sub(age_ms); + let mut rows = Vec::new(); + for (account_id, backup_arn) in self.status_index_arns(store, status).await? { + let Some(row) = self.read_index_row(&account_id, &backup_arn).await? else { + continue; + }; + if row.status != status || row.store != *store { + continue; + } + let at = match clock { + StaleClock::Heartbeat => Some(seconds_to_millis(row.heartbeat_at)), + StaleClock::Completed => row.completed_at.map(seconds_to_millis), + }; + if at.is_some_and(|at| at < cutoff) { + rows.push(row); + } + } + sort_listing(&mut rows); + Ok(rows) + }) + } + + fn sweep_restore_registrations(&self) -> BoxFuture<'_, Result, StorageError>> { + Box::pin(async move { + // A registration is live while its target is a CREATING restore + // target with the registered table id restoring from this backup + // (the rule the in-use check applies); any other entry is one a + // restore left behind, its removal lost with its owner, and is + // removed by a lightweight transaction on the entry as read. The + // map has no index, so the scan is a range read of `backups_by_arn` + // at quorum; rows carrying an entry are few. + use cdrs_tokio::types::AsRust; + use cdrs_tokio::types::map::Map; + let catalog = self.catalog_keyspace(); + let scan = format!( + "SELECT account_id, backup_arn, active_restores FROM {catalog}.backups_by_arn" + ); + let rows = crate::cassandra_util::query_rows_quorum::( + &self.session, + &scan, + cdrs_tokio::query_values!(), + "restore registrations", + ) + .await?; + let remove = format!( + "DELETE active_restores[?] FROM {catalog}.backups_by_arn \ + WHERE account_id = ? AND backup_arn = ? IF active_restores[?] = ?" + ); + let mut swept = Vec::new(); + for row in rows { + let map: Option = row.get_by_name("active_restores").ok().flatten(); + let Some(map) = map else { + continue; + }; + let entries: std::collections::HashMap = map + .as_rust() + .map_err(|e| StorageError::Internal(format!("Parse active_restores: {e}")))? + .unwrap_or_default(); + if entries.is_empty() { + continue; + } + let account_id: String = + crate::cassandra_util::get_column(&row, "account_id", "restore registrations")?; + let backup_arn: String = + crate::cassandra_util::get_column(&row, "backup_arn", "restore registrations")?; + let mut touched = false; + for (table_id, target_name) in entries { + let live = self + .restore_target_row(&account_id, &target_name) + .await? + .is_some_and(|target| { + target.table_id == table_id + && target.is_creating_target() + && target.source_backup_arn.as_deref() == Some(backup_arn.as_str()) + }); + if live { + continue; + } + let removed = conditional_write( + &self.session, + &remove, + cdrs_tokio::query_values!( + table_id.as_str(), + account_id.as_str(), + backup_arn.as_str(), + table_id.as_str(), + target_name.as_str() + ), + ) + .await?; + if removed == WriteOutcome::Matched { + tracing::info!( + "backup {backup_arn}: removed the restore registration of table \ + {target_name} ({table_id}); the target is no longer being restored \ + from it" + ); + touched = true; + } + } + if touched { + swept.push(backup_arn); + } + } + swept.sort(); + Ok(swept) + }) + } + fn backup_in_use<'a>( &'a self, key: &'a BackupKey, diff --git a/crates/storage-cassandra/tests/direct/backup_engine.rs b/crates/storage-cassandra/tests/direct/backup_engine.rs index 3daa5be2c..2db5b466e 100644 --- a/crates/storage-cassandra/tests/direct/backup_engine.rs +++ b/crates/storage-cassandra/tests/direct/backup_engine.rs @@ -20,8 +20,8 @@ use extenddb_core::types::{ TimeToLiveStatus, }; use extenddb_storage::backup_catalog::{ - BackupRow, BackupStatus, DeleteStart, Heartbeat, JobStatus, RestoreTargetKey, StatusFields, - StoreIdentity, StoreKind, TimeBounds, UNOWNED_GENERATION, WriteOutcome, + BackupRow, BackupStatus, DeleteStart, Heartbeat, JobStatus, RestoreTargetKey, StaleClock, + StatusFields, StoreIdentity, StoreKind, TimeBounds, UNOWNED_GENERATION, WriteOutcome, }; use extenddb_storage::error::StorageError; use extenddb_storage::{ @@ -2315,6 +2315,7 @@ async fn test_claim_backup_excludes_a_second_owner_and_fences_its_writes() { size_bytes: Some(1234), item_count: Some(5), failure_reason: None, + source_table_details_json: None, completed: true, }; assert_eq!( @@ -4542,3 +4543,490 @@ async fn test_the_maintenance_claim_excludes_a_second_holder_and_expires_when_st taker.release().await.expect("release"); assert!(!check().await.expect("check")); } + +/// The index row of `row`, read back through the catalog. +async fn stored_index_row(engine: &CassandraEngine, row: &BackupRow) -> BackupRow { + engine + .read_backup_row(&arn_of(row)) + .await + .expect("read") + .expect("the row exists") +} + +/// `claim_dead_backup`: refused while the owner's heartbeat is within the +/// window, on the coordinator's clock; once the window has passed (waited +/// out, not hand-dated) the row is taken over under a fresh generation with +/// its heartbeat left as the dead owner wrote it, the old claim finds its +/// heartbeat lost and its writes unmatched, and the successor's AVAILABLE +/// write carries the source table and a completion time and moves the status +/// index. A successor that heartbeats is a live owner to a third taker; a row +/// that is not CREATING, and a missing row, answer `None`. +#[tokio::test] +async fn test_claim_dead_backup_takes_over_a_stale_row_and_refuses_a_live_one() { + if crate::helpers::skip_without_cassandra() { + return; + } + let engine = setup_engine().await; + let account = TestAccount::new(&engine, "extenddb_ttl_test").await; + let account_id = account.account_id.clone(); + let store = unique_store(); + let row = index_row(&account_id, "Music", "dead-owner", &store); + let key = row.key(); + let window = Duration::from_secs(1); + assert!( + engine + .claim_dead_backup(&key, window) + .await + .expect("claim") + .is_none(), + "a missing row" + ); + let owner = engine.claim_backup(&row).await.expect("claim"); + let before = stored_index_row(&engine, &row).await; + assert!( + engine + .claim_dead_backup(&key, Duration::from_secs(300)) + .await + .expect("claim") + .is_none(), + "a live owner within a long window" + ); + assert!( + engine + .claim_dead_backup(&key, window) + .await + .expect("claim") + .is_none(), + "a heartbeat written an instant ago is within a one-second window" + ); + tokio::time::sleep(Duration::from_millis(1_300)).await; + let successor = engine + .claim_dead_backup(&key, window) + .await + .expect("claim") + .expect("the window passed on the coordinator's clock: taken over"); + assert_ne!(successor.generation(), owner.generation()); + assert_ne!(successor.generation(), UNOWNED_GENERATION); + let after = stored_index_row(&engine, &row).await; + assert_eq!(after.owner_generation, successor.generation()); + assert_eq!(after.status, BackupStatus::Creating); + assert_eq!( + after.heartbeat_at, before.heartbeat_at, + "the takeover left the heartbeat as the dead owner wrote it" + ); + assert_eq!( + owner.heartbeat().await.expect("heartbeat"), + Heartbeat::Lost, + "the old owner's heartbeat finds another generation" + ); + assert_eq!( + owner + .set_status( + JobStatus::Creating, + JobStatus::Failed, + &StatusFields::default() + ) + .await + .expect("write"), + WriteOutcome::NotMatched, + "the old generation's write is fenced" + ); + + // The successor heartbeats: a third taker finds a live owner. + assert_eq!( + successor.heartbeat().await.expect("heartbeat"), + Heartbeat::Confirmed + ); + assert!( + engine + .claim_dead_backup(&key, window) + .await + .expect("claim") + .is_none(), + "the successor's heartbeat is fresh" + ); + + // The successor's AVAILABLE write carries the manifest's source table. + let mut source_table = row.source_table_details_json.clone(); + source_table["item_count"] = serde_json::json!(41); + let fields = StatusFields { + size_bytes: Some(4_096), + item_count: Some(41), + failure_reason: None, + source_table_details_json: Some(source_table.clone()), + completed: true, + }; + assert_eq!( + successor + .set_status(JobStatus::Creating, JobStatus::Available, &fields) + .await + .expect("write"), + WriteOutcome::Matched + ); + let available = stored_index_row(&engine, &row).await; + assert_eq!(available.status, BackupStatus::Available); + assert_eq!(available.size_bytes, 4_096); + assert_eq!(available.item_count, 41); + assert_eq!( + available.source_table_details_json, source_table, + "the write applied the source table" + ); + assert!(available.completed_at.is_some(), "completed is stamped"); + let listed: Vec = engine + .list_backup_rows(Some(BackupStatus::Available), &store) + .try_collect() + .await + .unwrap(); + assert_eq!(listed.len(), 1, "the status index followed: {listed:?}"); + assert!( + engine + .claim_dead_backup(&key, Duration::ZERO) + .await + .expect("claim") + .is_none(), + "an AVAILABLE row is not taken over, whatever its heartbeat's age" + ); + + // A FAILED write without the member leaves the stored source table. + let failed = index_row(&account_id, "Music", "dead-owner-failed", &store); + let claim = engine.claim_backup(&failed).await.expect("claim"); + assert_eq!( + claim + .set_status( + JobStatus::Creating, + JobStatus::Failed, + &StatusFields { + failure_reason: Some("owner dead".to_owned()), + completed: true, + ..StatusFields::default() + } + ) + .await + .expect("write"), + WriteOutcome::Matched + ); + let stored = stored_index_row(&engine, &failed).await; + assert_eq!( + stored.source_table_details_json, + failed.source_table_details_json + ); + assert_eq!(stored.failure_reason.as_deref(), Some("owner dead")); +} + +/// `list_backup_rows_older_than`: the rows of a store with a status whose +/// named clock column is older than the age on the coordinator's clock. A +/// row FAILED after its heartbeat aged is listed on the heartbeat clock and +/// not on the completed clock until that window too has passed (waited out +/// on the coordinator's clock, not hand-dated); a row with no `completed_at` +/// is never listed on the completed clock; another status and another store +/// are not listed. +#[tokio::test] +async fn test_rows_older_than_are_listed_on_the_named_clock() { + if crate::helpers::skip_without_cassandra() { + return; + } + let engine = setup_engine().await; + let account = TestAccount::new(&engine, "extenddb_ttl_test").await; + let account_id = account.account_id.clone(); + let store = unique_store(); + let window = Duration::from_secs(1); + let listed = |status: BackupStatus, clock: StaleClock, age: Duration| { + let engine = &engine; + let store = &store; + async move { + let mut arns: Vec = engine + .list_backup_rows_older_than(status, store, clock, age) + .map_ok(|row| row.backup_arn) + .try_collect() + .await + .expect("listing"); + arns.sort(); + arns + } + }; + let completed_failure = StatusFields { + failure_reason: Some("owner dead".to_owned()), + completed: true, + ..StatusFields::default() + }; + let open_failure = StatusFields { + failure_reason: Some("owner dead".to_owned()), + completed: false, + ..StatusFields::default() + }; + let stamped = index_row(&account_id, "Music", "stamped", &store); + let unstamped = index_row(&account_id, "Music", "unstamped", &store); + let creating = index_row(&account_id, "Music", "creating", &store); + let mut other_store = store.clone(); + other_store.store_id = "store-2".to_owned(); + let elsewhere = index_row(&account_id, "Music", "elsewhere", &other_store); + let stamped_claim = engine.claim_backup(&stamped).await.expect("claim"); + let unstamped_claim = engine.claim_backup(&unstamped).await.expect("claim"); + let _creating_claim = engine.claim_backup(&creating).await.expect("claim"); + let elsewhere_claim = engine.claim_backup(&elsewhere).await.expect("claim"); + assert_eq!( + elsewhere_claim + .set_status(JobStatus::Creating, JobStatus::Failed, &completed_failure) + .await + .expect("write"), + WriteOutcome::Matched + ); + assert!( + listed(BackupStatus::Failed, StaleClock::Heartbeat, Duration::ZERO) + .await + .is_empty(), + "nothing of this store is FAILED yet" + ); + + // The heartbeats age past the window; then the two rows fail, one with a + // completion time and one without. + tokio::time::sleep(Duration::from_millis(1_300)).await; + assert_eq!( + stamped_claim + .set_status(JobStatus::Creating, JobStatus::Failed, &completed_failure) + .await + .expect("write"), + WriteOutcome::Matched + ); + assert_eq!( + unstamped_claim + .set_status(JobStatus::Creating, JobStatus::Failed, &open_failure) + .await + .expect("write"), + WriteOutcome::Matched + ); + let mut both = vec![stamped.backup_arn.clone(), unstamped.backup_arn.clone()]; + both.sort(); + assert_eq!( + listed(BackupStatus::Failed, StaleClock::Heartbeat, window).await, + both, + "a row just failed is listed on the heartbeat clock: its dead owner's beat is stale" + ); + assert!( + listed(BackupStatus::Failed, StaleClock::Completed, window) + .await + .is_empty(), + "the completion time is an instant old" + ); + tokio::time::sleep(Duration::from_millis(50)).await; + assert_eq!( + listed(BackupStatus::Failed, StaleClock::Completed, Duration::ZERO).await, + std::slice::from_ref(&stamped.backup_arn), + "a row with no completed_at is not listed on the completed clock" + ); + assert!( + listed( + BackupStatus::Failed, + StaleClock::Heartbeat, + Duration::from_secs(300) + ) + .await + .is_empty(), + "a long window lists nothing" + ); + assert_eq!( + listed(BackupStatus::Creating, StaleClock::Heartbeat, window).await, + std::slice::from_ref(&creating.backup_arn), + "the CREATING row is listed under its own status only" + ); + assert!( + listed( + BackupStatus::Creating, + StaleClock::Completed, + Duration::ZERO + ) + .await + .is_empty(), + "a CREATING row has no completion time" + ); + assert!( + listed(BackupStatus::Deleted, StaleClock::Completed, Duration::ZERO) + .await + .is_empty() + ); + + // The completed window crossed on the coordinator's clock. + tokio::time::sleep(Duration::from_millis(1_300)).await; + assert_eq!( + listed(BackupStatus::Failed, StaleClock::Completed, window).await, + std::slice::from_ref(&stamped.backup_arn) + ); + assert_eq!( + listed(BackupStatus::Failed, StaleClock::Heartbeat, window).await, + both, + "the other store's FAILED row is not listed" + ); +} + +/// The `active_restores` of a `backups_by_arn` row, as the map's entries. +async fn registrations( + engine: &CassandraEngine, + account_id: &str, + backup_arn: &str, +) -> std::collections::HashMap { + use cdrs_tokio::types::AsRust; + use cdrs_tokio::types::IntoRustByName; + let rows = extenddb_storage_cassandra::cassandra_util::query_rows_quorum::( + &engine.session_arc(), + &format!( + "SELECT active_restores FROM {}.backups_by_arn \ + WHERE account_id = ? AND backup_arn = ?", + engine.catalog_keyspace() + ), + cdrs_tokio::query_values!(account_id, backup_arn), + "registrations", + ) + .await + .expect("read the row"); + let map: Option = rows + .first() + .expect("the row exists") + .get_by_name("active_restores") + .ok() + .flatten(); + map.map(|map| map.as_rust().expect("map").unwrap_or_default()) + .unwrap_or_default() +} + +/// `sweep_restore_registrations`: an entry whose target is a CREATING +/// restore target of the backup with the registered table id is kept; one +/// whose target is ACTIVE, one whose target restores from another backup and +/// one whose target has no row are removed, and the swept backup's ARN is +/// returned once. A second sweep finds nothing to do. +#[tokio::test] +async fn test_the_registrations_sweep_removes_stale_entries_and_keeps_a_live_one() { + if crate::helpers::skip_without_cassandra() { + return; + } + let (engine, account_id, _drop_keyspaces) = isolated_engine("sw", false).await; + let store = unique_store(); + let catalog = engine.catalog_keyspace(); + let row = index_row(&account_id, "Music", "registered", &store); + let claim = engine.claim_backup(&row).await.expect("claim"); + assert_eq!( + claim + .set_status( + JobStatus::Creating, + JobStatus::Available, + &StatusFields::default() + ) + .await + .expect("write"), + WriteOutcome::Matched + ); + let other = index_row(&account_id, "Music", "other", &store); + let other_claim = engine.claim_backup(&other).await.expect("claim"); + assert_eq!( + other_claim + .set_status( + JobStatus::Creating, + JobStatus::Available, + &StatusFields::default() + ) + .await + .expect("write"), + WriteOutcome::Matched + ); + let mut targets = Vec::new(); + for name in ["SwLive", "SwActive", "SwForeign"] { + targets.push( + create_active_table( + &engine, + &account_id, + CreateTableInput { + table_name: name.to_owned(), + key_schema: vec![ks("id", KeyType::Hash)], + attribute_definitions: vec![attr("id", ScalarAttributeType::S)], + billing_mode: Some(BillingMode::PayPerRequest), + ..Default::default() + }, + ) + .await, + ); + } + let (live, active, foreign) = (&targets[0], &targets[1], &targets[2]); + // A CREATING target restoring from the backup; a CREATING target + // restoring from the other backup; an ACTIVE table; a name with no row. + for (target, source) in [(live, &row), (foreign, &other)] { + apply( + &engine, + &format!( + "UPDATE {catalog}.tables SET table_status = 'CREATING', \ + status_transition_at = null, restore_source_backup_arn = ? \ + WHERE account_id = ? AND table_name = ? IF table_id = ?" + ), + cdrs_tokio::query_values!( + source.backup_arn.as_str(), + account_id.as_str(), + target.table_name.as_str(), + target.table_id.as_str() + ), + ) + .await; + } + let missing_id = uuid::Uuid::new_v4().to_string(); + for (table_id, name) in [ + (live.table_id.as_str(), "SwLive"), + (active.table_id.as_str(), "SwActive"), + (foreign.table_id.as_str(), "SwForeign"), + (missing_id.as_str(), "SwMissing"), + ] { + apply( + &engine, + &format!( + "UPDATE {catalog}.backups_by_arn SET active_restores[?] = ? \ + WHERE account_id = ? AND backup_arn = ? IF EXISTS" + ), + cdrs_tokio::query_values!(table_id, name, account_id.as_str(), row.backup_arn.as_str()), + ) + .await; + } + // The other backup registers its own live restore: nothing to sweep. + apply( + &engine, + &format!( + "UPDATE {catalog}.backups_by_arn SET active_restores[?] = ? \ + WHERE account_id = ? AND backup_arn = ? IF EXISTS" + ), + cdrs_tokio::query_values!( + foreign.table_id.as_str(), + "SwForeign", + account_id.as_str(), + other.backup_arn.as_str() + ), + ) + .await; + assert_eq!( + registrations(&engine, &account_id, &row.backup_arn) + .await + .len(), + 4 + ); + + let swept = engine.sweep_restore_registrations().await.expect("sweep"); + assert_eq!(swept, std::slice::from_ref(&row.backup_arn)); + let left = registrations(&engine, &account_id, &row.backup_arn).await; + assert_eq!( + left, + std::collections::HashMap::from([(live.table_id.clone(), "SwLive".to_owned())]), + "the live entry is kept and the three stale ones are removed" + ); + assert_eq!( + registrations(&engine, &account_id, &other.backup_arn).await, + std::collections::HashMap::from([(foreign.table_id.clone(), "SwForeign".to_owned())]), + "the other backup's live entry is untouched" + ); + assert!( + engine + .sweep_restore_registrations() + .await + .expect("sweep") + .is_empty(), + "a second sweep finds nothing" + ); + assert_eq!( + engine.backup_in_use(&row.key()).await.expect("in use"), + Some("SwLive".to_owned()), + "the kept entry is the in-use rule's" + ); +} diff --git a/crates/storage-mongodb/src/backup_catalog.rs b/crates/storage-mongodb/src/backup_catalog.rs index d65b5dfef..77244b666 100644 --- a/crates/storage-mongodb/src/backup_catalog.rs +++ b/crates/storage-mongodb/src/backup_catalog.rs @@ -11,7 +11,11 @@ //! "$$NOW"`, so no row ever exists unowned and a second claim of the same //! backup trips the unique index on (`account_id`, `backup_id`). Every write //! the job makes carries its generation in the filter; a heartbeat that -//! matches other than exactly one document means the claim is lost. +//! matches other than exactly one document means the claim is lost. The +//! heartbeat is also the liveness test: the reconciler takes over a CREATING +//! row whose heartbeat is older than the stale window on the server's clock +//! with one conditional update that sets a fresh generation and leaves +//! `heartbeat_at` as the dead owner wrote it. //! //! The lock on a row for DeleteBackup is a session transaction that `$inc`s //! `lock_seq` on the row before it reads the status and the in-use rule, so @@ -38,8 +42,8 @@ use std::time::Duration; use extenddb_core::backup_format::BackupArn; use extenddb_storage::backup_catalog::{ BackupCatalog, BackupKey, BackupRow, BackupStatus, DeleteStart, Heartbeat, JobClaim, JobRow, - JobStatus, MaintenanceClaim, RestoreJobRow, RestoreKind, RestoreTargetKey, StatusFields, - StoreIdentity, StoreKind, TimeBounds, WriteOutcome, new_generation, + JobStatus, MaintenanceClaim, RestoreJobRow, RestoreKind, RestoreTargetKey, StaleClock, + StatusFields, StoreIdentity, StoreKind, TimeBounds, WriteOutcome, new_generation, }; use extenddb_storage::error::StorageError; use futures::TryStreamExt; @@ -204,6 +208,46 @@ fn key_filter(key: &BackupKey) -> Document { doc! { "account_id": &key.account_id, "backup_id": &key.backup_id } } +/// `set` as the `$set` stage of a pipeline update: `$$NOW` stays the server's +/// clock and every other value is wrapped in `$literal`, so a string, or a +/// key inside the source table object, is never read as a field path. +fn literal_set(set: Document) -> Document { + set.into_iter() + .map(|(k, v)| { + let v = match v { + Bson::String(s) if s == "$$NOW" => Bson::String(s), + other => doc! { "$literal": other }.into(), + }; + (k, v) + }) + .collect() +} + +/// The filter of the rows of `store`. +fn store_filter(store: &StoreIdentity) -> Document { + doc! { + "store_kind": store.store_kind.as_str(), + "store_root": &store.store_root, + "store_id": &store.store_id, + } +} + +/// The `$expr` of [`BackupCatalog::list_backup_rows_older_than`]: the +/// `clock` column older than `age` on the server's clock. `age_is_stale` +/// reads a missing column as the epoch, which is the rule the dead-job +/// listing wants and not this one: a row with `completed_at` unset has not +/// reached its terminal status's clock, so the column must also be present. +fn older_than_filter(clock: StaleClock, age: Duration) -> Document { + let column = match clock { + StaleClock::Heartbeat => "heartbeat_at", + StaleClock::Completed => "completed_at", + }; + doc! { + column: { "$type": "date" }, + "$expr": age_is_stale(&[column], age), + } +} + /// The sort every listing applies: `(created_at, backup_arn)`. fn listing_sort() -> Document { doc! { "created_at": 1, "backup_arn": 1 } @@ -310,22 +354,19 @@ impl JobClaim for MongoBackupClaim { if let Some(reason) = &fields.failure_reason { set.insert("failure_reason", reason.as_str()); } + if let Some(source) = &fields.source_table_details_json { + let source = bson::to_bson(source).map_err(|e| { + StorageError::Internal(format!("serialize source_table_details_json: {e}")) + })?; + set.insert("source_table_details_json", source); + } if fields.completed { set.insert("completed_at", "$$NOW"); } - // A pipeline update, so `$$NOW` is the server's clock; the - // literal strings are wrapped so the pipeline does not read them - // as field paths. - let set: Document = set - .into_iter() - .map(|(k, v)| { - let v = match v { - Bson::String(s) if s == "$$NOW" => Bson::String(s), - other => doc! { "$literal": other }.into(), - }; - (k, v) - }) - .collect(); + // A pipeline update, so `$$NOW` is the server's clock; every other + // value is wrapped in `$literal` so the pipeline does not read a + // string, or a key of the source table object, as a field path. + let set = literal_set(set); match self .index .update_one(filter, vec![doc! { "$set": set }]) @@ -493,16 +534,7 @@ impl BackupCatalog for MongoEngine { let mut set = row_fields(row)?; set.insert("owner_generation", generation_field(generation)); set.insert("heartbeat_at", "$$NOW"); - let set: Document = set - .into_iter() - .map(|(k, v)| { - let v = match v { - Bson::String(s) if s == "$$NOW" => Bson::String(s), - other => doc! { "$literal": other }.into(), - }; - (k, v) - }) - .collect(); + let set = literal_set(set); match self .backup_index() .update_one(filter, vec![doc! { "$set": set }]) @@ -538,11 +570,7 @@ impl BackupCatalog for MongoEngine { store: &'a StoreIdentity, ) -> BoxStream<'a, Result> { rows_stream(async move { - let mut filter = doc! { - "store_kind": store.store_kind.as_str(), - "store_root": &store.store_root, - "store_id": &store.store_id, - }; + let mut filter = store_filter(store); if let Some(status) = status { filter.insert("status", status.as_str()); } @@ -674,6 +702,67 @@ impl BackupCatalog for MongoEngine { }) } + fn claim_dead_backup<'a>( + &'a self, + key: &'a BackupKey, + stale_after: Duration, + ) -> BoxFuture<'a, Result>, StorageError>> { + Box::pin(async move { + // The heartbeat is the liveness test: a live owner refreshes it + // every period, so a CREATING row whose heartbeat is older than + // the window on the server's clock has no owner, and the one + // write below both tests that and takes the row. The filter + // binds no generation, so the write takes a row whatever + // generation it carries; `heartbeat_at` is left as the dead + // owner wrote it. A second taker before this one resolves the + // row (another process's reconciler; one process runs one + // reconciler) finds the same stale heartbeat and takes the row + // again under its own generation, which fences this claim's + // status write as any takeover fences the dead owner's. + let generation = new_generation(); + let mut filter = key_filter(key); + filter.insert("status", BackupStatus::Creating.as_str()); + filter.insert("$expr", age_is_stale(&["heartbeat_at"], stale_after)); + let update = vec![doc! { "$set": literal_set(doc! { + "owner_generation": generation_field(generation), + }) }]; + let taken = self + .backup_index() + .find_one_and_update(filter, update) + .await + .map_err(internal)?; + Ok(taken.map(|_| { + Box::new(MongoBackupClaim { + index: self.backup_index(), + key: key.clone(), + generation, + }) as Box + })) + }) + } + + fn list_backup_rows_older_than<'a>( + &'a self, + status: BackupStatus, + store: &'a StoreIdentity, + clock: StaleClock, + age: Duration, + ) -> BoxStream<'a, Result> { + rows_stream(async move { + let mut filter = store_filter(store); + filter.insert("status", status.as_str()); + filter.extend(older_than_filter(clock, age)); + self.backup_index() + .find(filter) + .sort(listing_sort()) + .await + .map_err(internal)? + .try_collect() + .await + .map_err(internal) + }) + } + fn backup_in_use<'a>( &'a self, key: &'a BackupKey, @@ -1105,4 +1194,58 @@ mod tests { assert_eq!(generation_from_field(&doc, "owner_generation"), g); assert_eq!(generation_from_field(&doc! {}, "owner_generation"), 0); } + + /// The older-than filter reads the named column alone against the + /// server's clock and requires the column to be a date, so a row whose + /// `completed_at` was never stamped is not read as old as the epoch. + #[test] + fn the_older_than_filter_names_one_clock_column_and_requires_it_set() { + let heartbeat = older_than_filter(StaleClock::Heartbeat, Duration::from_secs(90)); + assert_eq!( + heartbeat.get_document("heartbeat_at").unwrap(), + &doc! { "$type": "date" } + ); + assert!(heartbeat.get("completed_at").is_none()); + let expr = heartbeat.get_document("$expr").unwrap().to_string(); + assert!(expr.contains("$heartbeat_at"), "{expr}"); + assert!(!expr.contains("completed_at"), "{expr}"); + assert!(expr.contains("$$NOW") && expr.contains("90000"), "{expr}"); + + let completed = older_than_filter(StaleClock::Completed, Duration::from_secs(90)); + assert_eq!( + completed.get_document("completed_at").unwrap(), + &doc! { "$type": "date" } + ); + assert!(completed.get("heartbeat_at").is_none()); + let expr = completed.get_document("$expr").unwrap().to_string(); + assert!(expr.contains("$completed_at"), "{expr}"); + assert!(!expr.contains("heartbeat_at"), "{expr}"); + } + + /// A pipeline `$set` keeps `$$NOW` as the server's clock and wraps every + /// other value, the source table object included, so none is read as a + /// field path. + #[test] + fn a_pipeline_set_wraps_every_value_but_now_in_a_literal() { + let source = bson::to_bson(&fixture_source_table()).unwrap(); + let set = literal_set(doc! { + "status": "AVAILABLE", + "completed_at": "$$NOW", + "owner_generation": 7_i64, + "source_table_details_json": source.clone(), + }); + assert_eq!(set.get_str("completed_at"), Ok("$$NOW")); + assert_eq!( + set.get_document("status").unwrap(), + &doc! { "$literal": "AVAILABLE" } + ); + assert_eq!( + set.get_document("owner_generation").unwrap(), + &doc! { "$literal": 7_i64 } + ); + assert_eq!( + set.get_document("source_table_details_json").unwrap(), + &doc! { "$literal": source } + ); + } } diff --git a/crates/storage-mongodb/src/backup_live_tests.rs b/crates/storage-mongodb/src/backup_live_tests.rs index 2bf7fe17f..e139eb513 100644 --- a/crates/storage-mongodb/src/backup_live_tests.rs +++ b/crates/storage-mongodb/src/backup_live_tests.rs @@ -31,7 +31,8 @@ use extenddb_core::types::{ }; use extenddb_storage::backup_catalog::{ BackupRow, BackupStatus, DeleteStart, Heartbeat, JobStatus, RestoreKind, RestoreTargetKey, - StatusFields, StoreIdentity, StoreKind, TimeBounds, UNOWNED_GENERATION, WriteOutcome, + StaleClock, StatusFields, StoreIdentity, StoreKind, TimeBounds, UNOWNED_GENERATION, + WriteOutcome, }; use extenddb_storage::error::StorageError; use extenddb_storage::{ @@ -601,6 +602,7 @@ async fn live_claim_backup_excludes_a_second_owner_and_fences_its_writes() { size_bytes: Some(1234), item_count: Some(5), failure_reason: None, + source_table_details_json: None, completed: true, }; assert_eq!( @@ -804,6 +806,293 @@ async fn live_a_dead_creating_row_is_removed_and_a_live_or_moved_one_is_kept() { fx.finish().await; } +/// The `backup_index` collection read at `majority`, for the observations +/// a test makes between the writes under test. +fn index_majority(engine: &MongoEngine) -> mongodb::Collection { + engine.catalog_db.collection_with_options( + BACKUP_INDEX, + mongodb::options::CollectionOptions::builder() + .read_concern(mongodb::options::ReadConcern::majority()) + .build(), + ) +} + +async fn index_doc(engine: &MongoEngine, row: &BackupRow) -> Document { + index_majority(engine) + .find_one(doc! { "account_id": &row.account_id, "backup_id": &row.backup_id }) + .await + .expect("read the row") + .expect("the row exists") +} + +/// `claim_dead_backup`: a CREATING row is refused while its owner's heartbeat +/// is within the window, and taken over once the window has passed on the +/// server's clock (no hand-dating: the test waits), under a fresh generation +/// and with `heartbeat_at` as the dead owner left it; the old claim then +/// finds its heartbeat lost and its status write unmatched; the successor's +/// heartbeat makes it a live owner a further taker is refused, and its +/// AVAILABLE write carries the source table object and stamps +/// `completed_at`. An AVAILABLE row and a missing row are refused. +#[tokio::test(flavor = "multi_thread")] +#[ignore = "needs a MongoDB replica set in EXTENDDB_TEST_MONGODB_URI"] +async fn live_claim_dead_backup_takes_over_a_stale_row_and_refuses_a_live_one() { + let fx = Fixture::new().await; + let window = Duration::from_secs(3); + let dead = row(ACCOUNT, "Music", "a"); + let key = dead.key(); + let owner = fx.engine.claim_backup(&dead).await.expect("claim"); + assert_eq!(owner.heartbeat().await.unwrap(), Heartbeat::Confirmed); + assert!( + fx.engine + .claim_dead_backup(&key, STALE_AFTER) + .await + .unwrap() + .is_none(), + "a live owner" + ); + assert!( + fx.engine + .claim_dead_backup(&key, window) + .await + .unwrap() + .is_none(), + "a heartbeat within the window" + ); + let before = index_doc(&fx.engine, &dead).await; + let heartbeat_before = *before.get_datetime("heartbeat_at").expect("heartbeat_at"); + tokio::time::sleep(window + Duration::from_millis(300)).await; + let successor = fx + .engine + .claim_dead_backup(&key, window) + .await + .unwrap() + .expect("the window has passed on the server's clock"); + assert_ne!(successor.generation(), owner.generation()); + assert_ne!(successor.generation(), UNOWNED_GENERATION); + let after = index_doc(&fx.engine, &dead).await; + assert_eq!( + *after.get_datetime("heartbeat_at").expect("heartbeat_at"), + heartbeat_before, + "the takeover left the heartbeat as the dead owner wrote it" + ); + assert_eq!(after.get_str("status"), Ok("CREATING")); + assert_eq!( + after.get_i64("owner_generation").unwrap(), + i64::try_from(successor.generation()).unwrap() + ); + assert_eq!(owner.heartbeat().await.unwrap(), Heartbeat::Lost); + assert_eq!( + owner + .set_status( + JobStatus::Creating, + JobStatus::Available, + &StatusFields::default() + ) + .await + .unwrap(), + WriteOutcome::NotMatched, + "the old generation is fenced" + ); + assert_eq!(successor.heartbeat().await.unwrap(), Heartbeat::Confirmed); + + // A second taker while the row is held: the heartbeat the successor + // just refreshed is within the window, so it is refused; another + // process's reconciler racing the takeover itself, before any + // heartbeat, would take the row again and fence the successor. + assert!( + fx.engine + .claim_dead_backup(&key, window) + .await + .unwrap() + .is_none(), + "a heartbeating successor is a live owner" + ); + + let mut source = dead.source_table_details_json.clone(); + source["item_count"] = serde_json::json!(4242); + let fields = StatusFields { + size_bytes: Some(777), + item_count: Some(4242), + failure_reason: None, + source_table_details_json: Some(source.clone()), + completed: true, + }; + assert_eq!( + successor + .set_status(JobStatus::Creating, JobStatus::Available, &fields) + .await + .unwrap(), + WriteOutcome::Matched + ); + let stored = fx + .engine + .read_backup_row(&arn_of(&dead)) + .await + .unwrap() + .unwrap(); + assert_eq!(stored.status, BackupStatus::Available); + assert_eq!(stored.size_bytes, 777); + assert_eq!(stored.item_count, 4242); + assert_eq!( + stored.source_table_details_json, source, + "the AVAILABLE write carries the source table object" + ); + assert_eq!(stored.source_table().unwrap().item_count, 4242); + assert!(stored.completed_at.is_some()); + assert_eq!(stored.owner_generation, successor.generation()); + + tokio::time::sleep(window + Duration::from_millis(300)).await; + assert!( + fx.engine + .claim_dead_backup(&key, window) + .await + .unwrap() + .is_none(), + "an AVAILABLE row with a stale heartbeat is not CREATING" + ); + let missing = row(ACCOUNT, "Music", "none").key(); + assert!( + fx.engine + .claim_dead_backup(&missing, window) + .await + .unwrap() + .is_none(), + "a missing row" + ); + drop(owner); + drop(successor); + fx.finish().await; +} + +/// `list_backup_rows_older_than`: the rows of the store with the status +/// whose named clock column is older than the age on the server's clock. A +/// row FAILED just now by a dead owner is listed on the heartbeat clock (its +/// heartbeat is the owner's last) and not yet on the completed clock, which +/// tells the two columns apart; the completed window is then crossed by +/// waiting. A FAILED row with no `completed_at` is never listed on the +/// completed clock. A CREATING row, and a FAILED row of another store, are +/// not listed under FAILED; a DELETED row is listed under DELETED on its +/// `completed_at`. The listing keeps the catalog's order. +#[tokio::test(flavor = "multi_thread")] +#[ignore = "needs a MongoDB replica set in EXTENDDB_TEST_MONGODB_URI"] +async fn live_rows_older_than_are_listed_on_the_named_clock() { + let fx = Fixture::new().await; + let window = Duration::from_secs(3); + let names = |listed: Vec| -> Vec { + listed.into_iter().map(|r| r.backup_name).collect() + }; + let older = |status: BackupStatus, store: StoreIdentity, clock: StaleClock| { + let engine = &fx.engine; + async move { + let rows: Vec = engine + .list_backup_rows_older_than(status, &store, clock, window) + .try_collect() + .await + .expect("listing"); + names(rows) + } + }; + let mut other_store = store(); + other_store.store_id = "store-2".to_owned(); + + let completed = row(ACCOUNT, "Music", "completed"); + let completed_claim = fx.engine.claim_backup(&completed).await.expect("claim"); + tokio::time::sleep(Duration::from_millis(5)).await; + let unstamped = row(ACCOUNT, "Music", "unstamped"); + let unstamped_claim = fx.engine.claim_backup(&unstamped).await.expect("claim"); + tokio::time::sleep(Duration::from_millis(5)).await; + let creating = row(ACCOUNT, "Music", "creating"); + let _creating_claim = fx.engine.claim_backup(&creating).await.expect("claim"); + let mut elsewhere = row(ACCOUNT, "Music", "elsewhere"); + elsewhere.store = other_store.clone(); + let elsewhere_claim = fx.engine.claim_backup(&elsewhere).await.expect("claim"); + let deleted = row(ACCOUNT, "Music", "deleted"); + let deleted_claim = fx.engine.claim_backup(&deleted).await.expect("claim"); + let done = StatusFields { + completed: true, + ..StatusFields::default() + }; + assert_eq!( + deleted_claim + .set_status(JobStatus::Creating, JobStatus::Available, &done) + .await + .unwrap(), + WriteOutcome::Matched + ); + assert_eq!( + fx.engine.start_backup_delete(&deleted.key()).await.unwrap(), + DeleteStart::Deleting + ); + assert!(fx.engine.set_backup_deleted(&deleted.key()).await.unwrap()); + assert!( + older(BackupStatus::Failed, store(), StaleClock::Heartbeat) + .await + .is_empty(), + "no row is FAILED yet" + ); + + tokio::time::sleep(window + Duration::from_millis(300)).await; + // The owners die here: FAILED writes with the heartbeat left as it was. + for (claim, fields) in [ + (&completed_claim, done.clone()), + (&unstamped_claim, StatusFields::default()), + (&elsewhere_claim, done.clone()), + ] { + assert_eq!( + claim + .set_status(JobStatus::Creating, JobStatus::Failed, &fields) + .await + .unwrap(), + WriteOutcome::Matched + ); + } + let unstamped_doc = index_doc(&fx.engine, &unstamped).await; + assert!( + unstamped_doc.get("completed_at").is_none(), + "a write with `completed: false` stamps nothing: {unstamped_doc}" + ); + assert_eq!( + older(BackupStatus::Failed, store(), StaleClock::Heartbeat).await, + ["completed", "unstamped"], + "rows FAILED just now are listed on their dead owners' heartbeats, in the catalog's order" + ); + assert!( + older(BackupStatus::Failed, store(), StaleClock::Completed) + .await + .is_empty(), + "a `completed_at` stamped just now is within the window; an unset one is never older" + ); + assert_eq!( + older(BackupStatus::Deleted, store(), StaleClock::Completed).await, + ["deleted"] + ); + assert_eq!( + older(BackupStatus::Creating, store(), StaleClock::Heartbeat).await, + ["creating"] + ); + assert_eq!( + older( + BackupStatus::Failed, + other_store.clone(), + StaleClock::Heartbeat + ) + .await, + ["elsewhere"] + ); + + tokio::time::sleep(window + Duration::from_millis(300)).await; + assert_eq!( + older(BackupStatus::Failed, store(), StaleClock::Completed).await, + ["completed"], + "the completed window crossed on the server's clock; no `completed_at` is never older" + ); + assert_eq!( + older(BackupStatus::Failed, store(), StaleClock::Heartbeat).await, + ["completed", "unstamped"] + ); + fx.finish().await; +} + /// The maintenance claim: taken once, refused to a second holder naming the /// first, heartbeated on the server's clock, released; a holder whose /// heartbeat is older than the stale window is dead, so its claim is not diff --git a/crates/storage-mongodb/src/snapshot_engine.rs b/crates/storage-mongodb/src/snapshot_engine.rs index 75b744aad..aa2cd0878 100644 --- a/crates/storage-mongodb/src/snapshot_engine.rs +++ b/crates/storage-mongodb/src/snapshot_engine.rs @@ -69,8 +69,11 @@ //! //! Memory: a batch is at most `batch_size` documents //! (`paging::DEFAULT_FETCH_BATCH`, 1000) and at most the server's 16 MiB -//! reply limit, whichever is reached first, so unlike the SQL backends' -//! row-bounded fetches one batch here is bounded in bytes too. +//! reply limit, whichever is reached first, so one batch here is bounded in +//! bytes by the server. The stream applies no cut of its own at +//! `paging::SNAPSHOT_FETCH_BYTES` (64 MiB): the reply limit is already under +//! that budget, and the byte cut the SQL backends make exists because their +//! row-bounded fetches have no such limit. use extenddb_core::types::Item; use extenddb_storage::error::StorageError; diff --git a/crates/storage-postgres/src/backup_catalog.rs b/crates/storage-postgres/src/backup_catalog.rs index 468f63eb7..e39246962 100644 --- a/crates/storage-postgres/src/backup_catalog.rs +++ b/crates/storage-postgres/src/backup_catalog.rs @@ -44,8 +44,8 @@ use std::time::Duration; use extenddb_core::backup_format::BackupArn; use extenddb_storage::backup_catalog::{ BackupCatalog, BackupKey, BackupRow, BackupStatus, DeleteStart, Heartbeat, JobClaim, JobRow, - JobStatus, MaintenanceClaim, RestoreJobRow, RestoreKind, RestoreTargetKey, StatusFields, - StoreIdentity, StoreKind, TimeBounds, WriteOutcome, + JobStatus, MaintenanceClaim, RestoreJobRow, RestoreKind, RestoreTargetKey, StaleClock, + StatusFields, StoreIdentity, StoreKind, TimeBounds, WriteOutcome, }; use extenddb_storage::error::StorageError; use futures::future::BoxFuture; @@ -249,7 +249,8 @@ impl JobClaim for PostgresBackupClaim { size_bytes = COALESCE($5, size_bytes), \ item_count = COALESCE($6, item_count), \ failure_reason = COALESCE($7, failure_reason), \ - completed_at = CASE WHEN $8 THEN now() ELSE completed_at END \ + completed_at = CASE WHEN $8 THEN now() ELSE completed_at END, \ + source_table_details_json = COALESCE($10, source_table_details_json) \ WHERE account_id = $1 AND backup_id = $2 AND owner_generation = $3 \ AND status = $9", ) @@ -262,6 +263,7 @@ impl JobClaim for PostgresBackupClaim { .bind(fields.failure_reason.as_deref()) .bind(fields.completed) .bind(from.as_str()) + .bind(fields.source_table_details_json.as_ref()) .execute(session.connection()) .await; match written { @@ -534,6 +536,76 @@ impl BackupCatalog for PostgresEngine { }) } + fn claim_dead_backup<'a>( + &'a self, + key: &'a BackupKey, + stale_after: Duration, + ) -> BoxFuture<'a, Result>, StorageError>> { + Box::pin(async move { + // The same advisory lock the owner took: a session that still + // holds it is alive whatever its heartbeat says (the lock goes + // with the session), and a second taker is refused here. + let Some(mut session) = + try_claim(&self.pool, &backup_lock_key(&key.backup_id), None).await? + else { + return Ok(None); + }; + let generation = session.generation(); + let taken = sqlx::query( + "UPDATE backup_index SET owner_generation = $3 \ + WHERE account_id = $1 AND backup_id = $2 AND status = 'CREATING' \ + AND heartbeat_at < now() - make_interval(secs => $4)", + ) + .bind(&key.account_id) + .bind(&key.backup_id) + .bind(generation_column(generation)) + .bind(stale_after.as_secs_f64()) + .execute(session.connection()) + .await + .map_err(db_err)? + .rows_affected(); + if taken != 1 { + session.close().await; + return Ok(None); + } + Ok(Some(Box::new(PostgresBackupClaim { + session: Mutex::new(session), + generation, + key: key.clone(), + pool: self.pool.clone(), + }) as Box)) + }) + } + + fn list_backup_rows_older_than<'a>( + &'a self, + status: BackupStatus, + store: &'a StoreIdentity, + clock: StaleClock, + age: Duration, + ) -> BoxStream<'a, Result> { + let column = match clock { + StaleClock::Heartbeat => "heartbeat_at", + StaleClock::Completed => "completed_at", + }; + rows_stream(async move { + sqlx::query_as(&format!( + "SELECT {ROW_COLUMNS} FROM backup_index \ + WHERE store_kind = $1 AND store_root = $2 AND store_id = $3 AND status = $4 \ + AND {column} < now() - make_interval(secs => $5) \ + ORDER BY created_at, backup_arn" + )) + .bind(store.store_kind.as_str()) + .bind(&store.store_root) + .bind(&store.store_id) + .bind(status.as_str()) + .bind(age.as_secs_f64()) + .fetch_all(&self.pool) + .await + .map_err(db_err) + }) + } + fn backup_in_use<'a>( &'a self, key: &'a BackupKey, diff --git a/crates/storage-postgres/src/backup_engine.rs b/crates/storage-postgres/src/backup_engine.rs index b501810a1..6677f65cc 100755 --- a/crates/storage-postgres/src/backup_engine.rs +++ b/crates/storage-postgres/src/backup_engine.rs @@ -190,6 +190,17 @@ pub(crate) fn backup_lock_key(backup_id: &str) -> String { format!("backup:{backup_id}") } +/// The TCP keepalive settings a claim session asks the server for, as +/// `(setting, seconds)`: idle 30 s before the first probe, then probes 10 s +/// apart, three unanswered before the connection is dropped. Public for the +/// test that reads them back with `SHOW`. +#[doc(hidden)] +pub const CLAIM_KEEPALIVES: [(&str, u32); 3] = [ + ("tcp_keepalives_idle", 30), + ("tcp_keepalives_interval", 10), + ("tcp_keepalives_count", 3), +]; + /// Take the lock `key` on a new dedicated connection, with a fresh /// generation. `Ok(None)`: another session holds it. /// @@ -222,6 +233,25 @@ pub(crate) async fn try_claim( the server setting: {e}" ); } + // The server's TCP keepalive probes toward this process, so the + // advisory lock of a claim whose host vanished (no FIN, no RST) is + // released about a minute after the loss (30 s idle, then three probes + // 10 s apart) instead of after the kernel's default two hours, and the + // row's successor can take the lock once the row's heartbeat is stale. + // The owner's own liveness check is the heartbeat, not these. Each is a + // session setting a managed service may refuse; a refusal is a warning + // and the server's defaults stay. + for (name, value) in CLAIM_KEEPALIVES { + if let Err(e) = sqlx::query(&format!("SET {name} = {value}")) + .execute(&mut conn) + .await + { + tracing::warn!( + "could not set {name} for the claim session; continuing with the server \ + setting: {e}" + ); + } + } let taken: bool = sqlx::query_scalar("SELECT pg_try_advisory_lock(hashtextextended($1, $2))") .bind(key) .bind(RESTORE_LOCK_SEED) @@ -235,6 +265,36 @@ pub(crate) async fn try_claim( })) } +/// Open a claim session on `key` as a job would and read `settings` back +/// from it with `SHOW`, then end the session: for the test that checks the +/// session settings [`try_claim`] applies. `None` when another session holds +/// the lock. +/// +/// # Errors +/// +/// The claim's own errors, and `Internal` when a `SHOW` fails. +#[cfg(feature = "test-hooks")] +#[doc(hidden)] +pub async fn claim_session_settings( + pool: &sqlx::PgPool, + key: &str, + settings: &[&str], +) -> Result>, StorageError> { + let Some(mut session) = try_claim(pool, key, None).await? else { + return Ok(None); + }; + let mut values = Vec::with_capacity(settings.len()); + for name in settings { + let value: String = sqlx::query_scalar(&format!("SHOW {name}")) + .fetch_one(session.connection()) + .await + .map_err(|e| StorageError::Internal(format!("SHOW {name}: {e}")))?; + values.push(value); + } + session.close().await; + Ok(Some(values)) +} + /// Take the restore lock for `table_name` on a new dedicated connection. async fn try_restore_lock( pool: &sqlx::PgPool, diff --git a/crates/storage-postgres/src/lib.rs b/crates/storage-postgres/src/lib.rs index 757eee26a..a8f74f4ee 100755 --- a/crates/storage-postgres/src/lib.rs +++ b/crates/storage-postgres/src/lib.rs @@ -27,7 +27,9 @@ mod operations; mod pg_util; mod snapshot_engine; #[doc(hidden)] -pub use snapshot_engine::map_fetch_error; +pub use snapshot_engine::{ + MIN_NEXT_PAGE_ROWS, MIN_PAGE_ROWS, SNAPSHOT_SLOT_BOUND, map_fetch_error, page_sql_for, +}; mod stream_engine; #[doc(hidden)] pub mod stream_records_sql; @@ -40,8 +42,11 @@ mod vector_search; mod worker_store; mod workers; +#[cfg(feature = "test-hooks")] +#[doc(hidden)] +pub use backup_engine::claim_session_settings; #[doc(hidden)] -pub use backup_engine::RESTORE_LOCK_SEED; +pub use backup_engine::{CLAIM_KEEPALIVES, RESTORE_LOCK_SEED}; pub use bootstrapper::PostgresBootstrapper; pub use catalog_store::PostgresCatalogStore; pub use config::PostgresStorageConfig; @@ -292,6 +297,9 @@ pub struct PostgresEngine { /// wait; the open needs only a data connection by then, so they delay it /// no further. pub(crate) snapshot_open_slots: Arc, + /// The permits `snapshot_open_slots` was built with, for the refusal + /// that names the bound. + pub(crate) snapshot_slot_limit: usize, /// Whether `data_pool` is the catalog pool (no separate data database). /// The snapshot open uses one transaction on that layout, and /// `DeleteTable` runs its in-use probe and its synchronous drop on the @@ -422,6 +430,7 @@ impl PostgresEngine { snapshot_open_slots: Arc::new(tokio::sync::Semaphore::new(snapshot_slot_count( pool_size, ))), + snapshot_slot_limit: snapshot_slot_count(pool_size), shared_data_pool, #[cfg(feature = "test-hooks")] drop_attempts: Arc::default(), diff --git a/crates/storage-postgres/src/snapshot_engine.rs b/crates/storage-postgres/src/snapshot_engine.rs index 3fbf6f83f..1f6bbeb90 100644 --- a/crates/storage-postgres/src/snapshot_engine.rs +++ b/crates/storage-postgres/src/snapshot_engine.rs @@ -91,46 +91,164 @@ //! layout the first catalog read already did. Its text is the manifest //! marker: the `xmin:xmax:xip` of the snapshot, recorded so a later //! change-log archive can tell which commits the snapshot includes; nothing -//! reopens a snapshot from it. Items are then read through a server-side -//! cursor in batches. +//! reopens a snapshot from it. Items are then read in keyset pages on the +//! same transaction: each page is one query that seeks past the last row's +//! primary key (`WHERE (pk, sk) > ($1, $2) ORDER BY pk, sk LIMIT n`, an +//! index scan on the primary key) and cuts itself at the byte budget +//! ([`PageQuery`]). //! -//! Memory: one `FETCH` is `paging::DEFAULT_FETCH_BATCH` (1000) rows, all of -//! them held as item text at once. At the default 400 KB item limit one -//! row's text can reach 27.7 MB (the measured worst case in #394's line-cap -//! table), so a fetch is bounded by rows, not bytes. A byte budget (keyset -//! paging with a byte cutoff) is required before the stream gets a caller; -//! the backup design assigns it to the backup job PR. +//! Memory: a page holds at most `paging::fetch_batch()` rows (1000) and +//! stops after the row that takes its item text to or past +//! `paging::SNAPSHOT_FETCH_BYTES` (64 MiB), so the stream holds at most the +//! budget plus one row of text in decoded form, plus the one row's text +//! being parsed: the rows are pulled from the connection one at a time and +//! parsed as they arrive (`fetch_page`), so the page's text is never held +//! whole on this side. The cut is made in the page query, so the server +//! returns no row past it; the rows it reads past the cut before `LIMIT n` +//! stops it are at most `n - 1`, and each of them costs the server one +//! serialization of its item text and two spills of it (the two window +//! functions buffer the `n` rows before the outer `WHERE` drops them), so a +//! page costs the server `n` rows of text, not the rows it returns. `n` is +//! [`MIN_PAGE_ROWS`] for the first page and is then sized from the largest +//! row of the last pages, those holding at least a row bound's worth of rows +//! ([`RowBoundMemory`], [`next_page_rows`]: the budget divided by that row, +//! never below [`MIN_NEXT_PAGE_ROWS`]), so a table of large rows pages at +//! the rows the budget holds plus one, and the over-read that remains is the +//! second page, and then the page that follows a row bound's worth of small +//! rows: each asks for the rows the small rows before it justified (the +//! first page's 16, then a row bound's worth) and may meet large ones, so it +//! reads up to its `n` rows of up to the line cap (32 MiB) each; `n` then +//! stays at the floor while large rows keep arriving within a row bound's +//! worth of rows and returns to the row bound a row bound's worth of small +//! rows after the last large one, so such a page recurs at most once per row +//! bound's worth of small rows, on a table whose large rows are rarer than +//! that. The bound is the rows the page reads, `n` of up to the line cap +//! each (measured on a 20,000-row table with 30 such rows every 33rd key +//! among the first 1,000: `docs/design/15-backup-restore-bench-pr8.md`); +//! nothing in this form stops the server at the budget, since a window +//! function runs over the `n` rows before the outer `WHERE` drops them. The +//! floor has a cost of its own: after a page holding a row at the line cap +//! the bound stays at the floor for a row bound's worth of rows, 250 pages of +//! four, whatever those rows hold, so a table whose large rows are rarer +//! than one per row bound pays 250 round trips per large row where one page +//! of memory would page it at the row bound (measured on a 20,000-row table +//! with a 27 MB row at every 1,100th key: 2,585 page queries against 38 with +//! one page of memory, 25.6 against 25.1 s, 1.06 against 1.09 GB of temp; +//! the record). //! -//! Order: the cursor orders by the primary key (`key_order_clause`) only when -//! the text key columns (`pk`, and `sk_s` for an `S` range key) have a -//! collation named "C" or "POSIX", read from `pg_attribute` inside the -//! snapshot transaction. The rule is the name: any other collation, -//! including "default" on a database whose own collation is C, streams -//! unordered, which is complete. A table created before the key columns were -//! declared `COLLATE "C"` keeps the database collation, and ordering it -//! bytewise is a Seq Scan and an external sort of the whole table before the -//! first FETCH (7 MB on disk for 100k rows at `work_mem` 4 MB), with the -//! lock, the connection and the permit held throughout. Such a table is -//! streamed with a plain `SELECT`, reported `key_ordered = false`, and a -//! warning names it. +//! The plan is the planner's choice, not the query's promise. With the +//! planner's estimate of the rows after the seek above `n`, the page is +//! `Limit` over an `Index Scan` on the primary key index with no sort (the +//! 100k-row test table's first page, a page in its middle and a page ten +//! rows from its end, `the_page_query_is_an_index_scan` and +//! `the_last_page_sorts_the_rows_after_the_seek`). On a table of about a +//! thousand rows, a page near its end whose `n` is above the rows left +//! (1,060 rows, 57 left, `n` = 810) gets a `Bitmap Heap Scan` driven by a +//! `Bitmap Index Scan` on the same index and a `Sort` of those rows under +//! the `Limit` (PostgreSQL 16, the same test), so such a page sorts the rows +//! after the seek, at most the rows left, with their text; the seek is +//! served by the primary key index either way. +//! +//! Order: the page orders by the primary key in the key columns' own +//! collation, which is the order the primary key index has, so every table +//! pages by its index. The stream is reported `key_ordered` only when the +//! key shape has a format-defined order (`key_schema_is_orderable`) and the +//! text key columns (`pk`, `sk_s`) have a collation named "C" or "POSIX", +//! read from `pg_attribute` inside the snapshot transaction; the rule is the +//! name, so "default" on a database whose own collation is C streams +//! unordered, which is complete (`open_stream` says why the table is not +//! sorted bytewise instead). use std::sync::Arc; use extenddb_core::types::ScalarAttributeType; use extenddb_core::types::{Item, TableDescription}; +use extenddb_storage::backup_catalog::{JobBound, limit_exceeded_message}; use extenddb_storage::error::StorageError; use extenddb_storage::snapshot::{drop_deleting_indexes, key_schema_is_orderable, paging}; -use extenddb_storage::util::sk_column; +use extenddb_storage::util::sk_column_n; use extenddb_storage::{SnapshotEngine, SnapshotInfo, TableSnapshot}; use futures::future::BoxFuture; use futures::stream::BoxStream; +use sqlx::Row as _; +use sqlx::postgres::PgRow; use sqlx::{PgConnection, Postgres, Transaction}; use tokio::sync::{Notify, OwnedSemaphorePermit}; use crate::PostgresEngine; use crate::data::{all_sort_key_info, data_table_name, index_table_name}; -const CURSOR: &str = "extenddb_snapshot"; +/// Rows the first page asks for, before anything is known about the row +/// size. The server serializes every row the inner `LIMIT` reads, so this +/// bounds the first page's cost at 16 rows of up to the line cap (32 MiB) +/// each; a table of small rows pays one short round trip for it, since the +/// second page is sized from the first ([`next_page_rows`]). +#[doc(hidden)] +pub const MIN_PAGE_ROWS: usize = 16; + +/// The fewest rows a page after the first asks for. The budget divided by +/// the largest row is two for 27 MB rows, while such a page keeps three (the +/// row that reaches the budget is kept), so without a floor every page of +/// such a table would be two rows and no page would reach its cut; at four, +/// a page of such rows reads one row past its cut. +#[doc(hidden)] +pub const MIN_NEXT_PAGE_ROWS: usize = 4; + +/// The largest rows of the last pages, which size the next page's row bound +/// ([`next_page_rows`]): the memory keeps the pages that together hold at +/// least [`RowBoundMemory::rows`] rows (the row bound, 1000 in production), +/// so a large row keeps the bound low for at least a row bound's worth of +/// rows after it rather than for one page. With one page of memory a large +/// row sets the next bound to 4, four small rows set the one after back to +/// 1000, and on a table whose large rows are scattered among small ones the +/// 1000-row page meets large rows again and the cycle repeats every other +/// page (5,555 rows read for 1,000 kept on the measured shape); with the +/// memory the bound stays at 4 while large rows keep arriving within a row +/// bound's worth of rows, so that table pages at 4 after its first 1000-row +/// page (1,900 read for 1,000 kept). +#[derive(Debug, Clone)] +struct RowBoundMemory { + /// How many rows the kept pages must hold at least. + rows: usize, + /// The largest row and the row count of each kept page, oldest first. + pages: std::collections::VecDeque<(usize, usize)>, +} + +impl RowBoundMemory { + fn new(rows: usize) -> Self { + Self { + rows, + pages: std::collections::VecDeque::new(), + } + } + + /// Record a page's largest row (`max_row`, 0 for an empty page) and its + /// row count, and forget the oldest pages the memory no longer needs: + /// the oldest page goes while the pages after it hold at least `rows`. + fn record(&mut self, max_row: usize, rows: usize) { + self.pages.push_back((max_row, rows)); + while self.pages.len() > 1 { + let after_oldest: usize = self.pages.iter().skip(1).map(|&(_, n)| n).sum(); + if after_oldest < self.rows { + break; + } + self.pages.pop_front(); + } + } + + /// The largest row among the kept pages; `None` when none held a row. + fn largest_row(&self) -> Option { + self.pages + .iter() + .map(|&(max_row, _)| max_row) + .max() + .filter(|&b| b > 0) + } +} + +/// The name the snapshot-slot refusal gives its bound: the setting the slot +/// count is derived from (half of `pool_size`). +pub const SNAPSHOT_SLOT_BOUND: &str = "[storage.postgres] pool_size snapshot-slot"; const BEGIN_SNAPSHOT: &str = "BEGIN ISOLATION LEVEL REPEATABLE READ READ ONLY"; @@ -239,17 +357,17 @@ impl PostgresEngine { let slot = Arc::clone(&self.snapshot_open_slots) .try_acquire_owned() .map_err(|_| { - StorageError::LimitExceeded( - "too many table snapshots are open on this server; retry when one ends" - .to_owned(), - ) + StorageError::LimitExceeded(limit_exceeded_message(JobBound::Counted { + name: SNAPSHOT_SLOT_BOUND, + limit: self.snapshot_slot_limit, + })) })?; let pinned = if self.shared_data_pool { self.pin_on_one_transaction(account_id, table_name).await? } else { self.pin_behind_barrier(account_id, table_name).await? }; - open_cursor(pinned, slot, table_name).await + open_stream(pinned, slot, table_name).await } /// Shared layout (module doc): one `REPEATABLE READ READ ONLY` @@ -462,53 +580,60 @@ async fn leave_out_gsis_without_data_table( Ok(()) } -/// The common half of the open: decide the order, then `DECLARE` the cursor -/// on the held transaction. -async fn open_cursor( +/// The common half of the open: decide the order and the collation, then +/// hand the held transaction to the keyset stream. +async fn open_stream( mut pinned: Pinned, slot: OwnedSemaphorePermit, table_name: &str, ) -> Result { let ddb_table = data_table_name(&pinned.table_id); - // Ordered only when the text key columns already sort bytewise: on a - // table whose key columns keep the database collation the ORDER BY would - // be a Seq Scan and a full sort before the first FETCH, with the lock, - // the connection and the permit held throughout (module doc). - let mut order_by = key_order_clause(&pinned.description); - if order_by.is_some() { - let not_bytewise: Vec = sqlx::query_scalar( - "SELECT a.attname::text FROM pg_attribute a \ - LEFT JOIN pg_collation c ON c.oid = a.attcollation \ - WHERE a.attrelid = to_regclass($1) AND a.attname = ANY($2) \ - AND NOT a.attisdropped \ - AND COALESCE(c.collname, '') NOT IN ('C', 'POSIX')", - ) - .bind(&ddb_table) - .bind(key_text_columns(&pinned.description)) - .fetch_all(pinned.held.open_conn()?) - .await - .map_err(|e| db_err(&e))?; - if !not_bytewise.is_empty() { - tracing::warn!( - table = table_name, - table_id = %pinned.table_id, - columns = ?not_bytewise, - "table snapshot: key columns do not have collation \"C\"; streaming the \ - table unordered (key_ordered = false) rather than sorting it" - ); - order_by = None; - } + let columns = key_columns(&pinned.description); + // Whether the text key columns already sort bytewise: a collation named + // "C" or "POSIX" in `pg_attribute`, read inside the snapshot transaction. + // The page orders by the primary key in the columns' own collation either + // way, which is the order the primary key index has, so a table created + // before the key columns were declared `TEXT COLLATE "C"` (it keeps the + // database collation, `en_US.utf8` in the default setups, where `a < B`) + // pages by its index and is reported unordered rather than sorted + // bytewise (a Seq Scan and an external sort of the whole table before the + // first row, with the lock, the connection and the permit held + // throughout). `COLLATE "C"` is written into the page query only when the + // columns have it, where it changes nothing and states the order the + // stream relies on (`pk` and `sk_s` bytewise, `sk_n` and `sk_b` by value: + // the order `extenddb_core::backup_format::key_order` defines), and the + // stream is reported ordered only then. + let text_columns: Vec = columns + .iter() + .filter(|c| c.kind == KeyKind::Text) + .map(|c| c.name.clone()) + .collect(); + let not_bytewise: Vec = sqlx::query_scalar( + "SELECT a.attname::text FROM pg_attribute a \ + LEFT JOIN pg_collation c ON c.oid = a.attcollation \ + WHERE a.attrelid = to_regclass($1) AND a.attname = ANY($2) \ + AND NOT a.attisdropped \ + AND COALESCE(c.collname, '') NOT IN ('C', 'POSIX')", + ) + .bind(&ddb_table) + .bind(&text_columns) + .fetch_all(pinned.held.open_conn()?) + .await + .map_err(|e| db_err(&e))?; + let collate_c = not_bytewise.is_empty(); + let orderable = key_schema_is_orderable(&pinned.description); + if orderable && !collate_c { + tracing::warn!( + table = table_name, + table_id = %pinned.table_id, + columns = ?not_bytewise, + "table snapshot: key columns do not have collation \"C\"; streaming the \ + table unordered (key_ordered = false) rather than sorting it" + ); } - let key_ordered = order_by.is_some(); - let select = match &order_by { - Some(cols) => format!("SELECT item_data::text FROM {ddb_table} ORDER BY {cols}"), - None => format!("SELECT item_data::text FROM {ddb_table}"), - }; + let key_ordered = orderable && collate_c; + let query = PageQuery::new(&ddb_table, columns, collate_c); paging::open_pause(paging::OpenStage::BeforeItems).await; - sqlx::query(&format!("DECLARE {CURSOR} NO SCROLL CURSOR FOR {select}")) - .execute(pinned.held.open_conn()?) - .await - .map_err(|e| db_err(&e))?; Ok(TableSnapshot { description: pinned.description, @@ -517,21 +642,10 @@ async fn open_cursor( marker: pinned.marker, }, key_ordered, - items: cursor_stream(pinned.held, slot), + items: keyset_stream(pinned.held, slot, query), }) } -/// The text key columns whose collation decides the order: `pk`, and `sk_s` -/// for an `S` range key (`sk_n` is `NUMERIC` and `sk_b` is `bytea`). -fn key_text_columns(description: &TableDescription) -> Vec { - let sks = all_sort_key_info(&description.key_schema, &description.attribute_definitions); - let mut columns = vec!["pk".to_owned()]; - if matches!(sks.as_slice(), [(_, ScalarAttributeType::S)]) { - columns.push("sk_s".to_owned()); - } - columns -} - /// Drop attribute definitions no key schema (table, GSI, LSI) names, after /// an index has been left out of a description. `CreateTable` refuses a /// definition no key uses. @@ -553,45 +667,334 @@ fn prune_unused_attribute_definitions(description: &mut TableDescription) { .retain(|d| used.contains(&d.attribute_name)); } -/// `ORDER BY` columns for the format's key order, or `None` when the key -/// shape has none (`key_schema_is_orderable`). The open uses the clause only -/// after checking that the text key columns (`pk`, and `sk_s` for an `S` -/// range key) have collation "C" or "POSIX" (`pg_attribute.attcollation`); -/// a table created before the key columns were declared `TEXT COLLATE "C"` -/// keeps the database collation (`en_US.utf8` in the default setups, where -/// `a < B`) and is streamed unordered instead, because `ORDER BY ... COLLATE -/// "C"` on it is a full sort before the first row. On the tables that reach -/// the clause the explicit `COLLATE "C"` therefore does not change the order; -/// it is kept as defence in depth (no test can tell it from its absence), so -/// the query states the order it relies on. `pk` and `sk_s` sort bytewise and -/// `sk_n` (`NUMERIC`) and `sk_b` (`BYTEA`) sort by value: the order -/// `extenddb_core::backup_format::key_order` defines. -fn key_order_clause(description: &TableDescription) -> Option { - if !key_schema_is_orderable(description) { - return None; - } +/// The type a primary-key column of the data table has, which decides how +/// the seek binds its value. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum KeyKind { + /// `pk`, `sk_s`, `sk_s`: `TEXT`. + Text, + /// `sk_n`, `sk_n`: `NUMERIC`, carried as its text. + Numeric, + /// `sk_b`, `sk_b`: `BYTEA`. + Bytes, +} + +/// A primary-key column of the data table. +#[derive(Debug, Clone)] +struct KeyColumn { + name: String, + kind: KeyKind, +} + +/// The primary-key columns of a table's data table, in the index's order: +/// `pk`, then one typed column per range key (`sk_column_n`). +fn key_columns(description: &TableDescription) -> Vec { let sks = all_sort_key_info(&description.key_schema, &description.attribute_definitions); - match sks.as_slice() { - [] => Some("pk COLLATE \"C\"".to_owned()), - [(_, ScalarAttributeType::S)] => Some("pk COLLATE \"C\", sk_s COLLATE \"C\"".to_owned()), - [(_, sk_type)] => Some(format!("pk COLLATE \"C\", {}", sk_column(*sk_type))), - _ => None, + let mut columns = vec![KeyColumn { + name: "pk".to_owned(), + kind: KeyKind::Text, + }]; + for (i, &(_, sk_type)) in sks.iter().enumerate() { + columns.push(KeyColumn { + name: sk_column_n(i, sk_type), + kind: match sk_type { + ScalarAttributeType::S => KeyKind::Text, + ScalarAttributeType::N => KeyKind::Numeric, + ScalarAttributeType::B => KeyKind::Bytes, + }, + }); } + columns } -/// Stream items from the open cursor, one `FETCH` per batch. The held +/// One key column's value from the last row of a page: where the next page +/// starts. +#[derive(Debug, Clone)] +enum KeyValue { + Text(String), + Numeric(String), + Bytes(Vec), +} + +/// The page query of one table: the first page's text and a following +/// page's, built once per open. +/// +/// Three layers. The innermost reads the rows after the seek in primary-key +/// order and stops at `LIMIT n`; the middle computes, over those `n` rows +/// only, the running total of item text in the same order and the count of +/// rows the limit returned (`count(*) OVER ()`, the end-of-stream signal); +/// the outer keeps every row whose predecessors' total is under the byte +/// budget and orders the result, since a window function's output order is +/// not promised to the client. The layering matters: a window function runs +/// before `LIMIT`, so a running total written beside the `LIMIT` would be +/// computed over the whole rest of the table. The innermost `ORDER BY ... +/// LIMIT` behind a row comparison on the primary-key columns is an index +/// scan on the primary key; `item_data::text` is computed once, in the +/// innermost layer, and the other layers read the alias. +#[derive(Debug, Clone)] +struct PageQuery { + first: String, + after: String, + columns: Vec, +} + +impl PageQuery { + fn new(ddb_table: &str, columns: Vec, collate_c: bool) -> Self { + Self { + first: page_sql(ddb_table, &columns, collate_c, false), + after: page_sql(ddb_table, &columns, collate_c, true), + columns, + } + } +} + +/// The text of a page query (see [`PageQuery`]). With `after_key` the query +/// seeks past a key bound as `$1..$k` (text, numeric as text, bytes) and +/// takes the row bound and the byte budget as `$k+1` and `$k+2`; without, as +/// `$1` and `$2`. +fn page_sql(ddb_table: &str, columns: &[KeyColumn], collate_c: bool, after_key: bool) -> String { + let order = |prefix: &str| -> String { + columns + .iter() + .enumerate() + .map(|(i, c)| { + let name = if prefix.is_empty() { + format!("t.{}", c.name) + } else { + format!("{prefix}.k{i}") + }; + if collate_c && c.kind == KeyKind::Text { + format!("{name} COLLATE \"C\"") + } else { + name + } + }) + .collect::>() + .join(", ") + }; + let seek = if after_key { + let left = columns + .iter() + .map(|c| format!("t.{}", c.name)) + .collect::>() + .join(", "); + let right = columns + .iter() + .enumerate() + .map(|(i, c)| { + let p = i + 1; + match c.kind { + KeyKind::Text => format!("${p}::text"), + KeyKind::Numeric => format!("${p}::text::numeric"), + KeyKind::Bytes => format!("${p}::bytea"), + } + }) + .collect::>() + .join(", "); + format!("WHERE ({left}) > ({right}) ") + } else { + String::new() + }; + let k = if after_key { columns.len() } else { 0 }; + let n_param = k + 1; + let budget_param = k + 2; + let inner_select = columns + .iter() + .enumerate() + .map(|(i, c)| format!("t.{} AS k{i}", c.name)) + .collect::>() + .join(", "); + let keys = (0..columns.len()) + .map(|i| format!("k{i}")) + .collect::>() + .join(", "); + let shown = columns + .iter() + .enumerate() + .map(|(i, c)| match c.kind { + KeyKind::Numeric => format!("page.k{i}::text AS c{i}"), + KeyKind::Text | KeyKind::Bytes => format!("page.k{i} AS c{i}"), + }) + .collect::>() + .join(", "); + format!( + "SELECT {shown}, page.item, page.n_total FROM (\ + SELECT {keys}, item, \ + sum(octet_length(item)) OVER (ORDER BY {lim_order} \ + ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS running, \ + count(*) OVER () AS n_total \ + FROM (\ + SELECT {inner_select}, t.item_data::text AS item FROM {ddb_table} t \ + {seek}ORDER BY {t_order} LIMIT ${n_param}\ + ) lim\ + ) page \ + WHERE page.running - octet_length(page.item) < ${budget_param} \ + ORDER BY {page_order}", + lim_order = order("lim"), + t_order = order(""), + page_order = order("page"), + ) +} + +/// The page query of `description`'s table, for the crate's tests: the +/// text [`PageQuery`] runs for a page after a key when `after_key`, else the +/// first page's, with `COLLATE "C"` on the text key columns when +/// `collate_c` (what `open_stream` decides from `pg_attribute`). +#[doc(hidden)] +#[must_use] +pub fn page_sql_for( + description: &TableDescription, + table_id: &str, + collate_c: bool, + after_key: bool, +) -> String { + page_sql( + &data_table_name(table_id), + &key_columns(description), + collate_c, + after_key, + ) +} + +/// One page as the database returned it, decoded. +struct Page { + /// The items the byte cut kept, in key order. + items: Vec, + /// The last kept row's key: where the next page starts. `None` for an + /// empty page. + last_key: Option>, + /// Each kept row's text, in bytes, in key order: what the next page's + /// row bound is sized from. + row_bytes: Vec, + /// Rows the innermost `LIMIT` returned, cut or not; fewer than the row + /// bound means the table is exhausted. + n_total: usize, +} + +/// Run the page query and decode its rows as they arrive: the rows are +/// pulled from the connection one at a time (`fetch`), each row's text is +/// parsed into its `Item` and dropped, so the page's text is never held +/// whole; what the page holds is its items. A row whose text does not parse +/// ends the page with `Internal`; a database error is mapped by +/// [`map_fetch_error`]. +async fn fetch_page( + conn: &mut PgConnection, + query: &PageQuery, + after: Option<&[KeyValue]>, + n: usize, + budget: usize, +) -> Result { + use futures::TryStreamExt as _; + let sql = if after.is_some() { + &query.after + } else { + &query.first + }; + let mut q = sqlx::query(sql); + for value in after.into_iter().flatten() { + q = match value { + KeyValue::Text(s) | KeyValue::Numeric(s) => q.bind(s.as_str()), + KeyValue::Bytes(b) => q.bind(b.as_slice()), + }; + } + let mut rows = q + .bind(i64::try_from(n).unwrap_or(i64::MAX)) + .bind(i64::try_from(budget).unwrap_or(i64::MAX)) + .fetch(conn); + let mut page = Page { + items: Vec::new(), + last_key: None, + row_bytes: Vec::new(), + n_total: 0, + }; + while let Some(row) = rows.try_next().await.map_err(|e| map_fetch_error(&e))? { + let row: PgRow = row; + let n_total: i64 = row.try_get("n_total").map_err(|e| db_err(&e))?; + page.n_total = usize::try_from(n_total).unwrap_or(0); + let mut key = Vec::with_capacity(query.columns.len()); + for (i, column) in query.columns.iter().enumerate() { + let name = format!("c{i}"); + key.push(match column.kind { + KeyKind::Text => { + KeyValue::Text(row.try_get(name.as_str()).map_err(|e| db_err(&e))?) + } + KeyKind::Numeric => { + KeyValue::Numeric(row.try_get(name.as_str()).map_err(|e| db_err(&e))?) + } + KeyKind::Bytes => { + KeyValue::Bytes(row.try_get(name.as_str()).map_err(|e| db_err(&e))?) + } + }); + } + let data: String = row.try_get("item").map_err(|e| db_err(&e))?; + drop(row); + let item = serde_json::from_str::(&data) + .map_err(|e| StorageError::Internal(format!("Parse item: {e}")))?; + page.row_bytes.push(data.len()); + drop(data); + page.items.push(item); + page.last_key = Some(key); + } + Ok(page) +} + +/// Rows the next page asks for, from the largest row of the last pages +/// (`memory`, [`RowBoundMemory`]): the rows the budget holds at that row +/// (`budget / largest`), between [`MIN_NEXT_PAGE_ROWS`] and the row bound. +/// The largest row and not the average, because the server serializes every +/// row the inner `LIMIT` reads: after a page of small rows with a few large +/// ones among them the average is small and a bound from it asks the next +/// page for hundreds of rows of a table that may now be large rows only (one +/// such page measured at 5.9 GB of server temp files for the three rows it +/// returned), where the largest row asks for the rows the budget holds of +/// it. A table of small rows pages at the row bound; a table of 27 MB rows +/// at [`MIN_NEXT_PAGE_ROWS`], of which one row per page is read past the cut +/// and dropped by the outer `WHERE`; a table whose large rows are scattered +/// among small ones pages at the row bound until the first page that meets +/// a large row, which reads up to its `n` rows of up to the line cap, and at +/// [`MIN_NEXT_PAGE_ROWS`] from there while large rows keep arriving within a +/// row bound's worth of rows, the bound returning to the row bound a row +/// bound's worth of small rows after the last large one; so every page that +/// follows a row bound's worth of small rows can read `n` rows of up to the +/// line cap, and no page can read more. The floor's cost: after a page +/// holding a row at the line cap the bound stays at the floor for a row +/// bound's worth of rows, 250 pages of four, whatever those rows hold, so a +/// table whose large rows are rarer than one per row bound pays 250 round +/// trips per large row (2,585 page queries against 38 with one page of +/// memory on a 20,000-row table with a 27 MB row at every 1,100th key, 25.6 +/// against 25.1 s, the record). A memory that held no row (an empty page) +/// gives the row bound. +fn next_page_rows(memory: &RowBoundMemory, budget: usize, batch: usize) -> usize { + let Some(max_row) = memory.largest_row() else { + return batch; + }; + (budget / max_row).max(MIN_NEXT_PAGE_ROWS).min(batch) +} + +/// Stream items by keyset pages on the held transaction. The held /// transaction lives in the stream's state: the last page commits it (waking /// the poller first), an error rolls it back, and dropping the stream before /// its end rolls it back through `HeldSnapshot`'s drop; each wakes the -/// control-plane poller. -fn cursor_stream( +/// control-plane poller. The row bound and the byte budget are read once +/// here, when the stream is built. +fn keyset_stream( held: HeldSnapshot, slot: OwnedSemaphorePermit, + query: PageQuery, ) -> BoxStream<'static, Result> { struct State { held: HeldSnapshot, /// Released with the transaction: the slot counts the connection. slot: Option, + query: PageQuery, + /// The last row's key; `None` before the first page. + after: Option>, + /// Rows the next page asks for. + n: usize, + batch: usize, + budget: usize, + /// The largest rows of the last pages, which size `n`. + memory: RowBoundMemory, buffer: std::vec::IntoIter, done: bool, } @@ -604,13 +1007,19 @@ fn cursor_stream( self.slot = None; } } + let batch = paging::fetch_batch(); let state = State { held, slot: Some(slot), + query, + after: None, + n: MIN_PAGE_ROWS.min(batch), + batch, + budget: paging::fetch_bytes(), + memory: RowBoundMemory::new(batch), buffer: Vec::new().into_iter(), done: false, }; - let batch = paging::fetch_batch(); Box::pin(futures::stream::unfold( state, move |mut state| async move { @@ -623,38 +1032,43 @@ fn cursor_stream( } let conn = state.held.conn()?; paging::record_fetch(); - let rows: Result, sqlx::Error> = - sqlx::query_as(&format!("FETCH {batch} FROM {CURSOR}")) - .fetch_all(conn) - .await; - let rows = match rows { - Ok(rows) => rows, + let page = fetch_page( + conn, + &state.query, + state.after.as_deref(), + state.n, + state.budget, + ) + .await; + let page = match page { + Ok(page) => page, Err(e) => { state.abandon(); - return Some((Err(map_fetch_error(&e)), state)); + return Some((Err(e), state)); } }; - if rows.len() < batch { + let returned = page.items.len(); + paging::record_page_bytes(page.row_bytes.iter().sum()); + paging::record_rows_past_cut( + u64::try_from(page.n_total.saturating_sub(returned)).unwrap_or(u64::MAX), + ); + // The table is exhausted when the limit returned fewer rows + // than asked and the cut dropped none of them; a short page + // the cut shortened still has rows past its last one. + if page.n_total < state.n && returned == page.n_total { state.done = true; // Close the transaction now rather than at drop so vacuum is // released as soon as the last row is out. let _ = state.held.finish().await; state.slot = None; + } else { + state.after = page.last_key; + state + .memory + .record(page.row_bytes.iter().copied().max().unwrap_or(0), returned); + state.n = next_page_rows(&state.memory, state.budget, state.batch); } - let mut items = Vec::with_capacity(rows.len()); - for (data,) in rows { - match serde_json::from_str::(&data) { - Ok(item) => items.push(item), - Err(e) => { - state.abandon(); - return Some(( - Err(StorageError::Internal(format!("Parse item: {e}"))), - state, - )); - } - } - } - state.buffer = items.into_iter(); + state.buffer = page.items.into_iter(); } }, )) @@ -679,3 +1093,140 @@ pub fn map_fetch_error(e: &sqlx::Error) -> StorageError { } db_err(e) } + +#[cfg(test)] +mod tests { + use super::*; + + fn columns(kinds: &[(&str, KeyKind)]) -> Vec { + kinds + .iter() + .map(|(name, kind)| KeyColumn { + name: (*name).to_owned(), + kind: *kind, + }) + .collect() + } + + fn memory_of(pages: &[(usize, usize)], rows: usize) -> RowBoundMemory { + let mut memory = RowBoundMemory::new(rows); + for &(max_row, n) in pages { + memory.record(max_row, n); + } + memory + } + + /// The row bound follows the largest row of the remembered pages within + /// its clamps: small rows page at the row bound, 27 MB rows at the floor + /// (the budget holds two, the floor is four), a memory that held no row + /// (an empty page) at the row bound, a row bound under the floor (a + /// test's small page) is the bound; and a page of 984 small rows and 3 + /// large ones is sized from the large row, not from the average (which + /// is 82 KB and would ask the next page for about 800 rows of a table + /// that is now large rows only). + #[test] + fn the_row_bound_adapts_within_its_clamps() { + let budget = 64 * 1024 * 1024; + assert_eq!( + next_page_rows(&memory_of(&[(300, 16)], 1000), budget, 1000), + 1000 + ); + // The floor as its value: the budget holds two such rows, a page + // keeps three, and four reads one past the cut. + assert_eq!( + next_page_rows(&memory_of(&[(27_000_000, 3)], 1000), budget, 1000), + 4 + ); + assert_eq!( + next_page_rows(&memory_of(&[(1024 * 1024, 8)], 1000), budget, 1000), + 64 + ); + assert_eq!( + next_page_rows(&memory_of(&[(0, 0)], 1000), budget, 1000), + 1000 + ); + assert_eq!(next_page_rows(&memory_of(&[(300, 3)], 3), budget, 3), 3); + // 984 rows of 300 bytes and 3 of 27 MB: the page's largest row sizes + // the next page. + let next = next_page_rows(&memory_of(&[(27_000_000, 987)], 1000), budget, 1000); + assert!( + next < 8, + "the next page asks for {next} rows after a page of 984 small rows and 3 large ones" + ); + assert!(next >= MIN_NEXT_PAGE_ROWS); + } + + /// The memory keeps the pages that together hold at least the row bound: + /// after a page with a 27 MB row, pages of four small rows keep the bound + /// at the floor until a row bound's worth of them have passed (250 pages + /// of four at a row bound of 1000), and the page after that is sized from + /// the small rows alone; a large row met within that span starts the + /// count again; the memory holds at most the pages it needs (the oldest + /// goes once the pages after it hold the row bound), so it does not grow + /// with the table. + #[test] + fn the_memory_covers_a_row_bound_of_rows() { + let budget = 64 * 1024 * 1024; + let mut memory = RowBoundMemory::new(1000); + memory.record(27_000_000, 83); + for _ in 0..249 { + memory.record(300, 4); + assert_eq!(next_page_rows(&memory, budget, 1000), MIN_NEXT_PAGE_ROWS); + } + // 249 pages of four hold 996 rows after the large page: still kept. + assert_eq!(memory.pages.len(), 250); + memory.record(300, 4); + // 1000 rows after it: the large page is forgotten. + assert_eq!(next_page_rows(&memory, budget, 1000), 1000); + assert_eq!(memory.pages.len(), 250); + // A large row within the span starts the count again. + memory.record(27_000_000, 4); + for _ in 0..249 { + memory.record(300, 4); + } + assert_eq!(next_page_rows(&memory, budget, 1000), MIN_NEXT_PAGE_ROWS); + memory.record(300, 4); + assert_eq!(next_page_rows(&memory, budget, 1000), 1000); + // Pages of the row bound: one page of memory is enough. + let mut memory = RowBoundMemory::new(1000); + memory.record(27_000_000, 3); + memory.record(300, 1000); + assert_eq!(memory.pages.len(), 1); + assert_eq!(next_page_rows(&memory, budget, 1000), 1000); + } + + /// The seek compares the key columns as their own types (numeric through + /// text, bytes as bytea), the first page has no seek, the text columns + /// carry `COLLATE "C"` only when asked, the row bound and the budget + /// take the parameters after the key, and `item_data::text` is computed + /// once in the innermost layer. + #[test] + fn the_page_query_binds_the_key_by_type() { + let cols = columns(&[("pk", KeyKind::Text), ("sk_n", KeyKind::Numeric)]); + let after = page_sql("\"_ddb_t\"", &cols, true, true); + assert!( + after.contains("WHERE (t.pk, t.sk_n) > ($1::text, $2::text::numeric) ORDER BY t.pk COLLATE \"C\", t.sk_n LIMIT $3"), + "{after}" + ); + assert!( + after.contains("< $4 ORDER BY page.k0 COLLATE \"C\", page.k1"), + "{after}" + ); + assert!(after.contains("page.k1::text AS c1"), "{after}"); + assert_eq!(after.matches("item_data::text").count(), 1, "{after}"); + let first = page_sql("\"_ddb_t\"", &cols, false, false); + assert!(!first.contains("WHERE (t.pk"), "{first}"); + assert!(first.contains("ORDER BY t.pk, t.sk_n LIMIT $1"), "{first}"); + assert!(first.contains("< $2 ORDER BY page.k0, page.k1"), "{first}"); + let bytes = columns(&[ + ("pk", KeyKind::Text), + ("sk_b", KeyKind::Bytes), + ("sk2_s", KeyKind::Text), + ]); + let after = page_sql("\"_ddb_t\"", &bytes, true, true); + assert!( + after.contains("> ($1::text, $2::bytea, $3::text) ORDER BY t.pk COLLATE \"C\", t.sk_b, t.sk2_s COLLATE \"C\" LIMIT $4"), + "{after}" + ); + } +} diff --git a/crates/storage-postgres/tests/backup_restore.rs b/crates/storage-postgres/tests/backup_restore.rs index af3c8fa46..6581e38a9 100644 --- a/crates/storage-postgres/tests/backup_restore.rs +++ b/crates/storage-postgres/tests/backup_restore.rs @@ -2736,7 +2736,7 @@ mod catalog_index { use super::*; - fn store() -> StoreIdentity { + pub(super) fn store() -> StoreIdentity { StoreIdentity { store_kind: StoreKind::Filesystem, store_root: "/var/lib/extenddb/backups".to_owned(), @@ -2881,6 +2881,7 @@ mod catalog_index { size_bytes: Some(1234), item_count: Some(5), failure_reason: None, + source_table_details_json: None, completed: true, }; assert_eq!( @@ -4414,6 +4415,381 @@ mod catalog_index { /// run against this catalog and real filesystem stores: two instances of one /// catalog starting against one empty root, and two instances with /// host-local roots under one `store_root` label. +/// The catalog methods the store-backed backup job and the reconciler call: +/// the takeover of a dead owner's row, the age listings on the database's +/// clock, the AVAILABLE write's `source_table_details_json`, the claim +/// session's settings. +mod backup_job { + use std::time::Duration; + + use extenddb_core::backup_format::BackupArn; + use extenddb_storage::BackupCatalog; + use extenddb_storage::backup_catalog::{ + BackupRow, BackupStatus, JobStatus, StaleClock, StatusFields, StoreIdentity, WriteOutcome, + }; + use futures::TryStreamExt; + + use super::catalog_index::{row as new_row, store}; + use super::*; + + fn arn_of(row: &BackupRow) -> BackupArn { + BackupArn::parse(&row.backup_arn).unwrap() + } + + async fn stored(s: &Scratch, row: &BackupRow) -> BackupRow { + s.engine + .read_backup_row(&arn_of(row)) + .await + .expect("read") + .expect("the row exists") + } + + /// Wait until no session holds the backup's advisory lock (the server + /// releases it a moment after the holding connection closes). + async fn await_lock_free(s: &Scratch, backup_id: &str) { + let deadline = tokio::time::Instant::now() + Duration::from_secs(10); + while backup_lock_holder(s, backup_id).await.is_some() { + assert!( + tokio::time::Instant::now() < deadline, + "the advisory lock outlived its session" + ); + tokio::time::sleep(Duration::from_millis(20)).await; + } + } + + /// `claim_dead_backup`: refused while the owner's session holds the + /// lock, refused while the row's heartbeat is within the stale window + /// though the lock is free (an owner that died a moment ago), taken over + /// once the heartbeat is older than the window on the database's clock + /// (the window is crossed by waiting, not by dating the row), with a + /// fresh generation and `heartbeat_at` as the dead owner left it; the + /// successor's AVAILABLE write carries the manifest's `source_table` + /// into the row; a row that is no longer CREATING is not taken over. + #[tokio::test] + async fn claim_dead_backup_takes_over_a_stale_row_and_refuses_a_live_one() { + if base_conn().is_none() { + eprintln!( + "SKIP claim_dead_backup_takes_over_a_stale_row_and_refuses_a_live_one: no \ + PostgreSQL" + ); + return; + } + let s = scratch().await; + let window = Duration::from_secs(1); + let row = new_row(ACCOUNT, "Music", "nightly"); + let key = row.key(); + let owner = s.engine.claim_backup(&row).await.expect("claim"); + // Alive: its session holds the lock. + assert!( + s.engine + .claim_dead_backup(&key, window) + .await + .expect("takeover attempt") + .is_none(), + "a row whose owner holds the lock was taken over" + ); + assert_eq!(stored(&s, &row).await.owner_generation, owner.generation()); + // The owner's session ends (a crash): the lock goes, the heartbeat + // stays as written. Within the window the row is not yet dead. + drop(owner); + await_lock_free(&s, &row.backup_id).await; + let left = stored(&s, &row).await; + assert!( + s.engine + .claim_dead_backup(&key, window) + .await + .expect("takeover attempt") + .is_none(), + "a row whose heartbeat is within the stale window was taken over" + ); + assert_eq!( + backup_lock_holder(&s, &row.backup_id).await, + None, + "a refused takeover left its lock session behind" + ); + tokio::time::sleep(window + Duration::from_millis(300)).await; + let successor = s + .engine + .claim_dead_backup(&key, window) + .await + .expect("takeover") + .expect("a stale CREATING row is taken over"); + assert_ne!(successor.generation(), left.owner_generation); + let taken = stored(&s, &row).await; + assert_eq!(taken.status, BackupStatus::Creating); + assert_eq!(taken.owner_generation, successor.generation()); + assert_eq!( + taken.heartbeat_at, left.heartbeat_at, + "the takeover refreshed heartbeat_at" + ); + assert!( + backup_lock_holder(&s, &row.backup_id).await.is_some(), + "the successor holds the lock" + ); + // A second taker is refused by the lock while the successor lives. + assert!( + s.engine + .claim_dead_backup(&key, window) + .await + .expect("second takeover attempt") + .is_none() + ); + // The successor's writes go through: the manifest's source table + // replaces the row's, the FAILED write's `None` leaves it. + let manifest_source = serde_json::json!({"table_name": "Music", "item_count": 42}); + let fields = StatusFields { + size_bytes: Some(10), + item_count: Some(42), + source_table_details_json: Some(manifest_source.clone()), + completed: true, + ..StatusFields::default() + }; + assert_eq!( + successor + .set_status(JobStatus::Creating, JobStatus::Available, &fields) + .await + .expect("write"), + WriteOutcome::Matched + ); + let available = stored(&s, &row).await; + assert_eq!(available.status, BackupStatus::Available); + assert_eq!(available.source_table_details_json, manifest_source); + assert_eq!(available.item_count, 42); + drop(successor); + await_lock_free(&s, &row.backup_id).await; + tokio::time::sleep(window + Duration::from_millis(300)).await; + assert!( + s.engine + .claim_dead_backup(&key, window) + .await + .expect("takeover attempt") + .is_none(), + "an AVAILABLE row was taken over" + ); + // A row that is gone. + assert!( + s.engine + .claim_dead_backup(&new_row(ACCOUNT, "Music", "other").key(), window) + .await + .expect("takeover attempt") + .is_none() + ); + s.cleanup().await; + } + + /// An owner whose claim session the server ended (`pg_terminate_backend`) + /// gets `Unknown` from its next status write, not an error, and its + /// re-read shows the row as it is: the fence the job and the reconciler + /// act on. Once the heartbeat is stale the row is a successor's. + #[tokio::test] + async fn a_terminated_claim_session_reports_unknown_and_the_row_is_reread() { + if base_conn().is_none() { + eprintln!( + "SKIP a_terminated_claim_session_reports_unknown_and_the_row_is_reread: no \ + PostgreSQL" + ); + return; + } + let s = scratch().await; + let row = new_row(ACCOUNT, "Music", "nightly"); + let owner = s.engine.claim_backup(&row).await.expect("claim"); + let pid = backup_lock_holder(&s, &row.backup_id) + .await + .expect("the claim session holds the lock"); + let terminated: bool = sqlx::query_scalar("SELECT pg_terminate_backend($1)") + .bind(pid) + .fetch_one(&s.catalog) + .await + .expect("terminate"); + assert!(terminated); + await_lock_free(&s, &row.backup_id).await; + let outcome = owner + .set_status( + JobStatus::Creating, + JobStatus::Available, + &StatusFields { + size_bytes: Some(1), + completed: true, + ..StatusFields::default() + }, + ) + .await + .expect("a lost connection is an outcome, not an error"); + assert_eq!(outcome, WriteOutcome::Unknown); + // The re-read runs on the pool, not the dead session. + let read = owner.read().await.expect("re-read"); + assert_eq!(read.status, JobStatus::Creating); + assert_eq!(read.owner_generation, owner.generation()); + // The heartbeat on the dead session is an error the caller treats as + // `Lost`. + assert!(owner.heartbeat().await.is_err()); + s.cleanup().await; + } + + /// `list_backup_rows_older_than` compares the named column with the + /// database's `now()`: a FAILED row whose heartbeat is older than the age + /// but whose completion is not is listed on the heartbeat clock and not + /// on the completed clock (the two columns are told apart by waiting + /// between the last heartbeat and the FAILED write, on the database's + /// clock); a row crosses the window by waiting; a row of another store, + /// another status, or with `completed_at` unset on the completed clock + /// is not listed. + #[tokio::test] + async fn rows_older_than_are_listed_on_the_named_clock() { + if base_conn().is_none() { + eprintln!("SKIP rows_older_than_are_listed_on_the_named_clock: no PostgreSQL"); + return; + } + let s = scratch().await; + let age = Duration::from_secs(1); + let engine = &s.engine; + let list = move |status: BackupStatus, store: StoreIdentity, clock: StaleClock| async move { + let rows: Vec = engine + .list_backup_rows_older_than(status, &store, clock, age) + .try_collect() + .await + .expect("list"); + rows.into_iter().map(|r| r.backup_name).collect::>() + }; + // `aged`: heartbeat now, then FAILED after the age has passed, so its + // heartbeat is older than the age and its completion is not. + let aged = new_row(ACCOUNT, "Music", "aged"); + let aged_claim = s.engine.claim_backup(&aged).await.expect("claim"); + // `fresh`: FAILED at once; both columns within the age. + let fresh = new_row(ACCOUNT, "Music", "fresh"); + let fresh_claim = s.engine.claim_backup(&fresh).await.expect("claim"); + let failed = StatusFields { + failure_reason: Some("the test's".to_owned()), + completed: true, + ..StatusFields::default() + }; + assert_eq!( + fresh_claim + .set_status(JobStatus::Creating, JobStatus::Failed, &failed) + .await + .unwrap(), + WriteOutcome::Matched + ); + // `open`: FAILED without a completion stamp (`completed: false`). + let open = new_row(ACCOUNT, "Music", "open"); + let open_claim = s.engine.claim_backup(&open).await.expect("claim"); + assert_eq!( + open_claim + .set_status( + JobStatus::Creating, + JobStatus::Failed, + &StatusFields { + failure_reason: Some("the test's".to_owned()), + ..StatusFields::default() + } + ) + .await + .unwrap(), + WriteOutcome::Matched + ); + // `creating`: stays CREATING with its heartbeat ageing. + let creating = new_row(ACCOUNT, "Music", "creating"); + let _creating_claim = s.engine.claim_backup(&creating).await.expect("claim"); + // `elsewhere`: FAILED at once, in another store. + let mut elsewhere = new_row(ACCOUNT, "Music", "elsewhere"); + elsewhere.store.store_id = "store-2".to_owned(); + let elsewhere_claim = s.engine.claim_backup(&elsewhere).await.expect("claim"); + assert_eq!( + elsewhere_claim + .set_status(JobStatus::Creating, JobStatus::Failed, &failed) + .await + .unwrap(), + WriteOutcome::Matched + ); + + // Nothing is older than the age yet, on either clock. + assert_eq!( + list(BackupStatus::Failed, store(), StaleClock::Heartbeat).await, + Vec::::new() + ); + assert_eq!( + list(BackupStatus::Failed, store(), StaleClock::Completed).await, + Vec::::new() + ); + + tokio::time::sleep(age + Duration::from_millis(300)).await; + assert_eq!( + aged_claim + .set_status(JobStatus::Creating, JobStatus::Failed, &failed) + .await + .unwrap(), + WriteOutcome::Matched + ); + // Heartbeat clock: every FAILED row of the store heartbeated before + // the wait (`open` has no completion but has a heartbeat); not the + // CREATING row, not the other store's. + assert_eq!( + list(BackupStatus::Failed, store(), StaleClock::Heartbeat).await, + ["aged", "fresh", "open"] + ); + // Completed clock: `aged` completed just now, `open` never. + assert_eq!( + list(BackupStatus::Failed, store(), StaleClock::Completed).await, + ["fresh"] + ); + assert_eq!( + list(BackupStatus::Creating, store(), StaleClock::Heartbeat).await, + ["creating"] + ); + let mut other = store(); + other.store_id = "store-2".to_owned(); + assert_eq!( + list(BackupStatus::Failed, other, StaleClock::Heartbeat).await, + ["elsewhere"] + ); + // `aged` crosses the completed window by waiting. + tokio::time::sleep(age + Duration::from_millis(300)).await; + assert_eq!( + list(BackupStatus::Failed, store(), StaleClock::Completed).await, + ["aged", "fresh"] + ); + s.cleanup().await; + } + + /// The claim session asks the server for TCP keepalives + /// (`CLAIM_KEEPALIVES`), read back from the session with `SHOW`. + #[tokio::test] + async fn the_claim_session_sets_tcp_keepalives() { + if base_conn().is_none() { + eprintln!("SKIP the_claim_session_sets_tcp_keepalives: no PostgreSQL"); + return; + } + let s = scratch().await; + let names: Vec<&str> = extenddb_storage_postgres::CLAIM_KEEPALIVES + .iter() + .map(|(name, _)| *name) + .collect(); + let values = extenddb_storage_postgres::claim_session_settings( + &s.catalog, + "backup:keepalive-test", + &names, + ) + .await + .expect("open a claim session") + .expect("the lock is free"); + let expected: Vec = extenddb_storage_postgres::CLAIM_KEEPALIVES + .iter() + .map(|(_, value)| value.to_string()) + .collect(); + assert_eq!(values, expected); + // A pooled connection has the server's defaults, not the claim's. + let pooled: String = sqlx::query_scalar("SHOW tcp_keepalives_idle") + .fetch_one(&s.catalog) + .await + .expect("show"); + assert_ne!( + pooled, "30", + "the test server's default is the claim's value" + ); + s.cleanup().await; + } +} + mod store_identity { use std::sync::Arc; diff --git a/crates/storage-postgres/tests/table_snapshot.rs b/crates/storage-postgres/tests/table_snapshot.rs index 9dbc4a928..e0a99d5ab 100644 --- a/crates/storage-postgres/tests/table_snapshot.rs +++ b/crates/storage-postgres/tests/table_snapshot.rs @@ -555,9 +555,18 @@ async fn half_the_pool_holds_snapshots(shared: bool) { .await .expect("an open over the bound is refused at once, not queued") .expect_err("the sixth open is over the bound"); - assert!( - matches!(err, extenddb_storage::error::StorageError::LimitExceeded(_)), - "{err}" + // The one refusal text every job bound answers, naming this one. + assert_eq!( + err.to_string(), + extenddb_storage::error::StorageError::LimitExceeded( + extenddb_storage::backup_catalog::limit_exceeded_message( + extenddb_storage::backup_catalog::JobBound::Counted { + name: extenddb_storage_postgres::SNAPSHOT_SLOT_BOUND, + limit: 5, + } + ) + ) + .to_string() ); let key: Item = [ ("pk".to_owned(), AttributeValue::S("p000".to_owned())), @@ -1077,6 +1086,662 @@ async fn snapshot_marker_and_key_order(shared: bool) { } on_both_layouts!(snapshot_marker_and_key_order); +/// The keyset pages respect the byte budget on both layouts +/// (`snapshot_conformance::run_page_bounds`), and the rows the server reads +/// past each page's cut are those the row bound lets it: with a budget of +/// eight rows the first page asks for `MIN_PAGE_ROWS` (16) and keeps eight, +/// so it reads eight it drops; every later page is sized from the largest +/// row of the page before (the budget divided by it, eight here), so it +/// reads exactly the rows it keeps and drops none. The figure is asserted +/// exactly: a counter that never moves would pass a bound (lesson 25), and a +/// row bound sized from the average again reads the same eight, so the +/// mixed-page case is pinned by `next_page_rows`'s unit test and by +/// `the_row_bound_remembers_large_rows_among_small_ones`. +async fn pages_are_cut_at_the_byte_budget(shared: bool) { + let s = scratch_with(shared).await; + let bounds = snapshot_conformance::run_page_bounds( + &s.hooks, + &s.engine, + ACCOUNT, + "page_bounds", + || async { + s.engine + .process_control_plane_transitions() + .await + .expect("pump"); + }, + ) + .await; + let past_cut = bounds.rows_past_cut; + let first_page = u64::try_from( + extenddb_storage_postgres::MIN_PAGE_ROWS - snapshot_conformance::PAGE_BOUNDS_ROWS, + ) + .unwrap(); + // The figure the memory record cites beside the RSS numbers. + eprintln!( + "rows read past the cut: {past_cut} over {} pages of {} rows ({first_page} expected, all \ + on the first page)", + bounds.pages, + bounds.budget / bounds.row_bytes + ); + assert_eq!( + past_cut, first_page, + "{past_cut} rows read past the cut over {} pages: the first page reads MIN_PAGE_ROWS and \ + drops {first_page}, and every later page is sized to the rows it keeps", + bounds.pages + ); +} +on_both_layouts!(pages_are_cut_at_the_byte_budget); + +/// Every plan node type under `node`, in depth-first order. +fn plan_node_types(node: &serde_json::Value, into: &mut Vec) { + if let Some(t) = node.get("Node Type").and_then(serde_json::Value::as_str) { + into.push(t.to_owned()); + } + for child in node + .get("Plans") + .and_then(serde_json::Value::as_array) + .into_iter() + .flatten() + { + plan_node_types(child, into); + } +} + +/// The index names every scan node under `node` uses. +fn plan_index_names(node: &serde_json::Value, into: &mut Vec) { + if let Some(name) = node.get("Index Name").and_then(serde_json::Value::as_str) { + into.push(name.to_owned()); + } + for child in node + .get("Plans") + .and_then(serde_json::Value::as_array) + .into_iter() + .flatten() + { + plan_index_names(child, into); + } +} + +/// A 100k-row composite-key table (`pk` = `p00000`..`p02000`, 50 rows each, +/// `sk_n` = 1..100000) filled straight into the data table and analyzed, +/// with its description and a data pool; the engine's `put_item` would take +/// minutes for 100k rows. +async fn hundred_k_table(s: &Scratch) -> (extenddb_core::types::TableDescription, PgPool) { + filled_table( + s, + "hundred_k", + "SELECT 'p' || lpad((i / 50)::text, 5, '0'), i, \ + jsonb_build_object('pk', jsonb_build_object('S', 'p' || lpad((i / 50)::text, 5, '0')), \ + 'sk', jsonb_build_object('N', i::text), \ + 'v', jsonb_build_object('S', repeat('x', 100))) \ + FROM generate_series(1, 100000) AS i", + ) + .await +} + +/// One key value bound to a page query's seek: text (numeric keys travel +/// as text) or bytes. +enum Bind { + Text(&'static str), + Bytes(Vec), +} + +/// `EXPLAIN (FORMAT JSON)` of the page query (with `COLLATE "C"` on its text +/// key columns when `collate_c`, as the engine writes it for a table whose +/// columns have that collation) after `after` (`None` for the first page) at +/// row bound `n`: the plan's node types in depth-first order and the index +/// names its scans use, with the plan for the message. +async fn explain_page( + data: &PgPool, + desc: &extenddb_core::types::TableDescription, + collate_c: bool, + after: Option>, + n: i64, +) -> (Vec, Vec, serde_json::Value) { + let budget = i64::try_from(extenddb_storage::snapshot::paging::SNAPSHOT_FETCH_BYTES).unwrap(); + let sql = + extenddb_storage_postgres::page_sql_for(desc, &desc.table_id, collate_c, after.is_some()); + let explain_sql = format!("EXPLAIN (FORMAT JSON) {sql}"); + let mut explain = sqlx::query_scalar::<_, serde_json::Value>(&explain_sql); + let seek = after.is_some(); + for value in after.into_iter().flatten() { + explain = match value { + Bind::Text(t) => explain.bind(t), + Bind::Bytes(b) => explain.bind(b), + }; + } + let plan = explain + .bind(n) + .bind(budget) + .fetch_one(data) + .await + .unwrap_or_else(|e| panic!("explain (after a key: {seek}) at n = {n}: {e}")); + let root = &plan[0]["Plan"]; + let mut types = Vec::new(); + plan_node_types(root, &mut types); + let mut indexes = Vec::new(); + plan_index_names(root, &mut indexes); + (types, indexes, plan) +} + +/// Assert that a page query's plan is `Limit` over one `Index Scan` on the +/// primary key index with no sort: the seek and the order both follow the +/// index, so the page costs the server the rows it reads, up to `n`, each +/// serialized once, and not the rows before them. +fn assert_index_scan( + label: &str, + types: &[String], + indexes: &[String], + plan: &serde_json::Value, + pkey: &str, +) { + assert!( + !types.iter().any(|t| t == "Sort"), + "{label}: the plan sorts: {types:?}\n{plan}" + ); + let scans: Vec<&str> = types + .iter() + .map(String::as_str) + .filter(|t| t.ends_with("Scan") && *t != "Subquery Scan") + .collect(); + assert_eq!( + scans, + vec!["Index Scan"], + "{label}: the relation is not read by one index scan: {types:?}\n{plan}" + ); + assert_eq!( + indexes, + vec![pkey.to_owned()], + "{label}: the scan does not use the primary key index\n{plan}" + ); +} + +/// A key shape the page query is run on: the `CreateTable` input, the data +/// table's key columns, the `SELECT` filling 20,000 rows of it, and a key in +/// its middle to seek past. +struct KeyShape { + label: &'static str, + input: extenddb_core::types::CreateTableInput, + columns: &'static str, + rows: &'static str, + middle: Vec, + /// Whether the text key columns have collation "C", so the engine writes + /// `COLLATE "C"` into the query (every table but the legacy one). + collate_c: bool, + /// Whether the stream reports the table ordered (a multi-part range key + /// has no format order; a legacy table's key column has the database + /// collation). + ordered: bool, +} + +fn key_shape_input(name: &str, range: &[(&str, &str)]) -> extenddb_core::types::CreateTableInput { + let mut key_schema = vec![serde_json::json!({"AttributeName": "pk", "KeyType": "HASH"})]; + let mut defs = vec![serde_json::json!({"AttributeName": "pk", "AttributeType": "S"})]; + for (attr, ty) in range { + key_schema.push(serde_json::json!({"AttributeName": attr, "KeyType": "RANGE"})); + defs.push(serde_json::json!({"AttributeName": attr, "AttributeType": ty})); + } + serde_json::from_value(serde_json::json!({ + "TableName": name, + "KeySchema": key_schema, + "AttributeDefinitions": defs, + "BillingMode": "PAY_PER_REQUEST" + })) + .expect("create input") +} + +/// The six key shapes a data table's primary key can have: hash only; an +/// `S`, `N` or `B` range key; a multi-part range key; and a legacy table +/// whose text key column carries the database collation (recollated after +/// the fill, as `legacy_collation_table_streams_unordered_and_complete` +/// does). +fn key_shapes() -> Vec { + vec![ + KeyShape { + label: "hash only", + input: key_shape_input("hash_only", &[]), + columns: "pk, item_data", + rows: "SELECT 'p' || lpad(i::text, 5, '0'), \ + jsonb_build_object('pk', jsonb_build_object('S', 'p' || lpad(i::text, 5, '0')), \ + 'v', jsonb_build_object('S', repeat('x', 100))) \ + FROM generate_series(1, 20000) AS i", + middle: vec![Bind::Text("p10000")], + collate_c: true, + ordered: true, + }, + KeyShape { + label: "S range key", + input: key_shape_input("range_s", &[("sk", "S")]), + columns: "pk, sk_s, item_data", + rows: "SELECT 'p' || lpad((i / 50)::text, 5, '0'), 'k' || lpad(i::text, 5, '0'), \ + jsonb_build_object('pk', jsonb_build_object('S', 'p' || lpad((i / 50)::text, 5, '0')), \ + 'sk', jsonb_build_object('S', 'k' || lpad(i::text, 5, '0')), \ + 'v', jsonb_build_object('S', repeat('x', 100))) \ + FROM generate_series(1, 20000) AS i", + middle: vec![Bind::Text("p00200"), Bind::Text("k10000")], + collate_c: true, + ordered: true, + }, + KeyShape { + label: "N range key", + input: key_shape_input("range_n", &[("sk", "N")]), + columns: "pk, sk_n, item_data", + rows: "SELECT 'p' || lpad((i / 50)::text, 5, '0'), i, \ + jsonb_build_object('pk', jsonb_build_object('S', 'p' || lpad((i / 50)::text, 5, '0')), \ + 'sk', jsonb_build_object('N', i::text), \ + 'v', jsonb_build_object('S', repeat('x', 100))) \ + FROM generate_series(1, 20000) AS i", + middle: vec![Bind::Text("p00200"), Bind::Text("10000")], + collate_c: true, + ordered: true, + }, + KeyShape { + label: "B range key", + input: key_shape_input("range_b", &[("sk", "B")]), + columns: "pk, sk_b, item_data", + rows: "SELECT 'p' || lpad((i / 50)::text, 5, '0'), decode(lpad(to_hex(i), 8, '0'), 'hex'), \ + jsonb_build_object('pk', jsonb_build_object('S', 'p' || lpad((i / 50)::text, 5, '0')), \ + 'sk', jsonb_build_object('B', encode(decode(lpad(to_hex(i), 8, '0'), 'hex'), 'base64')), \ + 'v', jsonb_build_object('S', repeat('x', 100))) \ + FROM generate_series(1, 20000) AS i", + middle: vec![ + Bind::Text("p00200"), + Bind::Bytes(10000_u32.to_be_bytes().to_vec()), + ], + collate_c: true, + ordered: true, + }, + KeyShape { + label: "multi-part range key (S, N)", + input: key_shape_input("multi_part", &[("sk", "S"), ("sk2", "N")]), + columns: "pk, sk_s, sk2_n, item_data", + rows: "SELECT 'p' || lpad((i / 50)::text, 5, '0'), 'k' || lpad((i / 10)::text, 4, '0'), i, \ + jsonb_build_object('pk', jsonb_build_object('S', 'p' || lpad((i / 50)::text, 5, '0')), \ + 'sk', jsonb_build_object('S', 'k' || lpad((i / 10)::text, 4, '0')), \ + 'sk2', jsonb_build_object('N', i::text), \ + 'v', jsonb_build_object('S', repeat('x', 100))) \ + FROM generate_series(1, 20000) AS i", + middle: vec![ + Bind::Text("p00200"), + Bind::Text("k1000"), + Bind::Text("10000"), + ], + collate_c: true, + ordered: false, + }, + KeyShape { + label: "legacy collation (pk on the database collation)", + input: key_shape_input("legacy", &[("sk", "N")]), + columns: "pk, sk_n, item_data", + rows: "SELECT 'p' || lpad((i / 50)::text, 5, '0'), i, \ + jsonb_build_object('pk', jsonb_build_object('S', 'p' || lpad((i / 50)::text, 5, '0')), \ + 'sk', jsonb_build_object('N', i::text), \ + 'v', jsonb_build_object('S', repeat('x', 100))) \ + FROM generate_series(1, 20000) AS i", + middle: vec![Bind::Text("p00200"), Bind::Text("10000")], + collate_c: false, + ordered: false, + }, + ] +} + +/// On every key shape a table can have, the page query (first page and a +/// page after a key in the table's middle, at the first page's row bound of +/// 16 and at 1000) is `Limit` over one `Index Scan` on the primary key index +/// with no sort, on a 20,000-row table; and the stream over the rows is +/// complete, ordered where the shape has a format order, in 21 round trips +/// (16 rows, then nineteen pages of 1000 and a last of 984). The legacy +/// table carries no `COLLATE "C"` in its query and pages in its index's +/// order. Near the end of a small table the plan differs: +/// `the_last_page_sorts_the_rows_after_the_seek`. +async fn the_page_query_is_an_index_scan_on_every_key_shape(shared: bool) { + let s = scratch_with(shared).await; + for shape in key_shapes() { + let name = shape.input.table_name.clone(); + create_active(&s, shape.input).await; + let desc = s + .engine + .describe_table( + ACCOUNT, + extenddb_core::types::DescribeTableInput { + table_name: name.clone(), + }, + ) + .await + .expect("describe"); + let ddb_table = data_table_name(&desc.table_id); + let data = s.data_pool().await; + sqlx::query(&format!( + "INSERT INTO {ddb_table} ({}) {}", + shape.columns, shape.rows + )) + .execute(&data) + .await + .unwrap_or_else(|e| panic!("{}: fill the table: {e}", shape.label)); + if shape.label.starts_with("legacy") { + sqlx::query(&format!( + "ALTER TABLE {ddb_table} ALTER COLUMN pk TYPE TEXT COLLATE \"default\"" + )) + .execute(&data) + .await + .expect("recollate"); + } + sqlx::query(&format!("ANALYZE {ddb_table}")) + .execute(&data) + .await + .expect("analyze"); + let pkey = format!("{}_pkey", ddb_table.trim_matches('"')); + for n in [ + i64::try_from(extenddb_storage_postgres::MIN_PAGE_ROWS).unwrap(), + 1000, + ] { + let (types, indexes, plan) = explain_page(&data, &desc, shape.collate_c, None, n).await; + assert_index_scan( + &format!("{}, first page at n = {n}", shape.label), + &types, + &indexes, + &plan, + &pkey, + ); + let middle: Vec = shape + .middle + .iter() + .map(|b| match b { + Bind::Text(t) => Bind::Text(t), + Bind::Bytes(v) => Bind::Bytes(v.clone()), + }) + .collect(); + let (types, indexes, plan) = + explain_page(&data, &desc, shape.collate_c, Some(middle), n).await; + assert_index_scan( + &format!("{}, page after a key at n = {n}", shape.label), + &types, + &indexes, + &plan, + &pkey, + ); + } + let sql = + extenddb_storage_postgres::page_sql_for(&desc, &desc.table_id, shape.collate_c, true); + assert_eq!( + sql.contains("COLLATE \"C\""), + shape.collate_c, + "{}: {sql}", + shape.label + ); + data.close().await; + + let fetches_before = extenddb_storage::snapshot::paging::fetches(); + let snapshot = s + .engine + .open_table_snapshot(ACCOUNT, &name) + .await + .expect("open"); + assert_eq!(snapshot.key_ordered, shape.ordered, "{}", shape.label); + let order = extenddb_core::backup_format::KeyOrder::resolve( + &snapshot.description.key_schema, + &snapshot.description.attribute_definitions, + ) + .ok(); + let items: Vec = { + use futures::StreamExt as _; + snapshot + .items + .map(|item| item.expect("item")) + .collect() + .await + }; + assert_eq!(items.len(), 20_000, "{}: every row streams", shape.label); + if shape.ordered { + let order = order.expect("an ordered shape has a format order"); + for pair in items.windows(2) { + assert_eq!( + order.compare(&pair[0], &pair[1]).expect("comparable keys"), + std::cmp::Ordering::Less, + "{}: {:?} is not before {:?}", + shape.label, + pair[0], + pair[1] + ); + } + } + let fetches = extenddb_storage::snapshot::paging::fetches() - fetches_before; + assert_eq!( + fetches, 21, + "{}: a first page of 16, nineteen of 1000 and a last of 984", + shape.label + ); + } +} +on_both_layouts!(the_page_query_is_an_index_scan_on_every_key_shape); + +/// Bytes of item text in a 27 MB row of `scattered_table`: the size of +/// #394's worst-expansion line, so three such rows fill the 64 MiB budget +/// (0, 27, 54 MB before each) and the fourth is the next page's. +const LARGE_ROW_PAD: usize = 27_000_000; + +/// A table of `small` rows of 50-byte text under keys `a0000`.., with a +/// 27 MB row at every `every`th key (`i % every == every - 1`), filled +/// straight into the data table. Key order does not sort items by size, so +/// a table whose large items are a minority is this shape rather than the +/// grouped one (`put-worst`'s `zz` keys after the `a` keys). +async fn scattered_table( + s: &Scratch, + name: &str, + small: usize, + every: usize, +) -> (extenddb_core::types::TableDescription, PgPool) { + filled_table( + s, + name, + &format!( + "SELECT 'a' || lpad(i::text, 4, '0'), i, \ + jsonb_build_object('pk', jsonb_build_object('S', 'a' || lpad(i::text, 4, '0')), \ + 'sk', jsonb_build_object('N', i::text), \ + 'v', jsonb_build_object('S', CASE WHEN i % {every} = {} \ + THEN repeat('x', {LARGE_ROW_PAD}) ELSE repeat('x', 50) END)) \ + FROM generate_series(0, {}) AS i", + every - 1, + small - 1 + ), + ) + .await +} + +/// On a table whose 27 MB rows are scattered among small ones (every 33rd +/// key of 300, nine large rows) the row bound's memory holds: the first +/// page reads 16 small rows, the second asks for the row bound (1000), meets +/// the large rows and reads every row left (284) for the 83 it keeps (up to +/// and including the third large one, whose predecessors hold 54 MB), and +/// every page after it asks for the floor (4) until the table ends, since +/// the memory covers a row bound's worth of rows and the table has fewer: +/// 201 rows read past the cut, all on the second page, over 53 round trips, +/// every row once in key order. With one page of memory the bound returns to +/// 1000 after four small rows and the over-read recurs (306 rows past the +/// cut over 7 pages on this table), which is what the rule corrects; the +/// bound that remains is the second page, and then the page that follows a +/// row bound's worth of small rows, each reading up to `n` rows of up to the +/// line cap: the 201 rows here are that second page's. +async fn the_row_bound_remembers_large_rows_among_small_ones(shared: bool) { + let s = scratch_with(shared).await; + let (_desc, data) = scattered_table(&s, "scattered", 300, 33).await; + data.close().await; + let budget = extenddb_storage::snapshot::paging::SNAPSHOT_FETCH_BYTES; + extenddb_storage::snapshot::paging::reset_page_counters(); + let fetches_before = extenddb_storage::snapshot::paging::fetches(); + let snapshot = s + .engine + .open_table_snapshot(ACCOUNT, "scattered") + .await + .expect("open"); + assert!(snapshot.key_ordered); + let keys: Vec = { + use futures::StreamExt as _; + snapshot + .items + .map(|item| match item.expect("item").get("pk") { + Some(extenddb_core::types::AttributeValue::S(v)) => v.clone(), + other => panic!("pk is {other:?}"), + }) + .collect() + .await + }; + let expected: Vec = (0..300).map(|i| format!("a{i:04}")).collect(); + assert_eq!(keys, expected, "every row once, in key order"); + let fetches = extenddb_storage::snapshot::paging::fetches() - fetches_before; + let past_cut = extenddb_storage::snapshot::paging::rows_past_cut(); + let first = extenddb_storage_postgres::MIN_PAGE_ROWS; + let second_read = 300 - first; + let second_kept = 99 - first; + let rest: usize = 300 - 99; + eprintln!( + "scattered rows: {past_cut} rows read past the cut over {fetches} pages ({} expected, all on \ + the second page)", + second_read - second_kept + ); + assert_eq!( + past_cut, + u64::try_from(second_read - second_kept).unwrap(), + "rows read past the cut: the second page reads the {second_read} rows left for the \ + {second_kept} it keeps, and no later page asks for more than the floor" + ); + // The floor is written as its value: a floor of 2 (the budget divided by + // a 27 MB row) would page a table of such rows two to a page with no page + // reaching its cut, and show here as 103 round trips. + assert_eq!( + fetches, + u64::try_from(2 + rest.div_ceil(4)).unwrap(), + "round trips: two pages, then {rest} rows at the floor of four" + ); + let largest = extenddb_storage::snapshot::paging::max_page_bytes(); + assert!( + largest >= budget && largest < budget + LARGE_ROW_PAD, + "the largest page holds {largest} bytes" + ); +} +on_both_layouts!(the_row_bound_remembers_large_rows_among_small_ones); + +/// A table filled with `rows` straight into its data table and analyzed, as +/// `hundred_k_table` does, with its description and a data pool; `rows` is +/// the `SELECT` producing `(pk, sk_n, item_data)`. +async fn filled_table( + s: &Scratch, + name: &str, + rows: &str, +) -> (extenddb_core::types::TableDescription, PgPool) { + create_active(s, snapshot_conformance::table_input(name)).await; + let desc = s + .engine + .describe_table( + ACCOUNT, + extenddb_core::types::DescribeTableInput { + table_name: name.to_owned(), + }, + ) + .await + .expect("describe"); + let ddb_table = data_table_name(&desc.table_id); + let data = s.data_pool().await; + sqlx::query(&format!( + "INSERT INTO {ddb_table} (pk, sk_n, item_data) {rows}" + )) + .execute(&data) + .await + .expect("fill the table"); + sqlx::query(&format!("ANALYZE {ddb_table}")) + .execute(&data) + .await + .expect("analyze"); + (desc, data) +} + +/// The plan near the end of a table, where the planner's estimate of the +/// rows after the seek is under the row bound, is the planner's choice and +/// not always the index scan: on the 100k-row table a page after the +/// 99,990th row at `n` = 1000 (ten rows left) is still `Limit` over the +/// `Index Scan`, while on a table of 1,060 rows (1,000 under one key prefix, +/// 60 under another) a page after the third +/// of the last 60 at `n` = 810 is a `Bitmap Heap Scan` driven by a `Bitmap +/// Index Scan` on the primary key index and a `Sort` of the 57 rows after +/// the seek under the `Limit`, so that page sorts the rows it reads, with +/// their text (PostgreSQL 16). The cell pins both so the module doc's +/// statement of them stays measured; the seek uses the primary key index in +/// both, and the sorted page returns the rows after the seek in key order. +async fn the_last_page_sorts_the_rows_after_the_seek(shared: bool) { + let s = scratch_with(shared).await; + let (desc, data) = hundred_k_table(&s).await; + let ddb_table = data_table_name(&desc.table_id); + let pkey = vec![format!("{}_pkey", ddb_table.trim_matches('"'))]; + let (types, indexes, plan) = explain_page( + &data, + &desc, + true, + Some(vec![Bind::Text("p01999"), Bind::Text("99990")]), + 1000, + ) + .await; + eprintln!("100k rows, a page after ('p01999', 99990) at n = 1000: {types:?}"); + assert_eq!( + indexes, pkey, + "100k: the seek does not use the primary key index\n{plan}" + ); + assert!( + types.iter().any(|t| t == "Index Scan") && !types.iter().any(|t| t == "Sort"), + "100k rows: the page near the end is not the index scan the module doc says it is \ + (correct the doc if the planner changed): {types:?}\n{plan}" + ); + data.close().await; + + let (desc, data) = filled_table( + &s, + "small", + "SELECT pk, i, jsonb_build_object('pk', jsonb_build_object('S', pk), \ + 'sk', jsonb_build_object('N', i::text), \ + 'v', jsonb_build_object('S', repeat('x', 50))) \ + FROM (SELECT 'a' || lpad(i::text, 4, '0') AS pk, i FROM generate_series(0, 999) AS i \ + UNION ALL \ + SELECT 'zz' || lpad(i::text, 3, '0'), i FROM generate_series(0, 59) AS i) rows", + ) + .await; + let ddb_table = data_table_name(&desc.table_id); + let pkey = vec![format!("{}_pkey", ddb_table.trim_matches('"'))]; + let (types, indexes, plan) = explain_page( + &data, + &desc, + true, + Some(vec![Bind::Text("zz002"), Bind::Text("2")]), + 810, + ) + .await; + eprintln!("1,060 rows, a page after ('zz002', 2) at n = 810: {types:?}"); + assert_eq!( + indexes, pkey, + "1,060: the seek does not use the primary key index\n{plan}" + ); + assert!( + types.iter().any(|t| t == "Sort") && types.iter().any(|t| t == "Bitmap Heap Scan"), + "1,060 rows: the page near the end is not the bitmap scan and Sort the module doc says \ + PostgreSQL 16 chooses there (correct the doc if the planner changed): {types:?}\n{plan}" + ); + // The sorted page's rows: the 57 after the seek, in key order. + let sql = extenddb_storage_postgres::page_sql_for(&desc, &desc.table_id, true, true); + let rows: Vec<(String, String)> = sqlx::query_as(&format!("SELECT c0, c1 FROM ({sql}) q")) + .bind("zz002") + .bind("2") + .bind(810_i64) + .bind(i64::try_from(extenddb_storage::snapshot::paging::SNAPSHOT_FETCH_BYTES).unwrap()) + .fetch_all(&data) + .await + .expect("the last page"); + let expected: Vec<(String, String)> = (3..60) + .map(|i| (format!("zz{i:03}"), i.to_string())) + .collect(); + assert_eq!(rows, expected); + data.close().await; +} +on_both_layouts!(the_last_page_sorts_the_rows_after_the_seek); + /// A hash-only table with no index, for tests that need many of them. fn plain_input(name: &str) -> extenddb_core::types::CreateTableInput { serde_json::from_value(serde_json::json!({ @@ -1253,16 +1918,17 @@ async fn a_restored_table_describes_its_restore(shared: bool) { } on_both_layouts!(a_restored_table_describes_its_restore); -/// The pid of the one backend whose last statement was the stream's `FETCH`. +/// The pid of the one backend whose last statement was the stream's page +/// query (its projection starts with the first key column's alias). async fn stream_backend(s: &Scratch, data: &PgPool) -> i32 { sqlx::query_scalar( "SELECT pid FROM pg_stat_activity WHERE datname = $1 \ - AND state = 'idle in transaction' AND query LIKE 'FETCH%'", + AND state = 'idle in transaction' AND query LIKE 'SELECT page.k0 AS c0%'", ) .bind(s.data_name()) .fetch_one(data) .await - .expect("the stream's backend is idle in its transaction after a FETCH") + .expect("the stream's backend is idle in its transaction after a page query") } /// End the stream's backend from another session, as a server restart or @@ -1436,7 +2102,7 @@ async fn release_wakes_the_active_poller(shared: bool, how: Release) { match snapshot.items.next().await { Some(Ok(_)) => {} Some(Err(_)) => break, - None => panic!("the stream ended without the FETCH error"), + None => panic!("the stream ended without the page query error"), } } kept = Some(snapshot); @@ -1649,7 +2315,7 @@ async fn the_in_catalog_backup_wakes_the_active_poller(shared: bool) { } on_both_layouts!(the_in_catalog_backup_wakes_the_active_poller); -/// A `FETCH` that fails (the backend ended under the stream) ends the stream +/// A page query that fails (the backend ended under the stream) ends the stream /// at the error and frees its permit and connection at once: with the /// errored stream still held a full bound of five opens succeeds. The /// `SnapshotExpired` codes take this branch. @@ -1671,10 +2337,10 @@ async fn a_fetch_error_ends_the_stream_and_frees_its_permit(shared: bool) { match snapshot.items.next().await { Some(Ok(_)) => items += 1, Some(Err(e)) => { - eprintln!("FETCH error after {items} items: {e}"); + eprintln!("page query error after {items} items: {e}"); break; } - None => panic!("the stream ended without reporting the FETCH error"), + None => panic!("the stream ended without reporting the page query error"), } } assert_eq!(items, 5, "the first page, then the error"); diff --git a/crates/storage-sqlite/Cargo.toml b/crates/storage-sqlite/Cargo.toml index 1cea63cec..1a0cc25d9 100644 --- a/crates/storage-sqlite/Cargo.toml +++ b/crates/storage-sqlite/Cargo.toml @@ -42,6 +42,11 @@ libc = { workspace = true } [dev-dependencies] extenddb-storage = { workspace = true, features = ["test-util"] } +# `tests/backup_job_panic.rs` drives the engine's CreateBackup job and +# reconciler against a `SqliteEngine` (the engine depends on `extenddb-storage`, +# not on this crate, so the edge is acyclic). +extenddb-engine = { workspace = true } +extenddb-backup-store = { workspace = true } # The slow-statement test sets sqlx's threshold, a `log` level, and prints # every tracing event of a child process. log = "0.4" diff --git a/crates/storage-sqlite/src/backup.rs b/crates/storage-sqlite/src/backup.rs index 2dfea06a5..aa9a30ec8 100644 --- a/crates/storage-sqlite/src/backup.rs +++ b/crates/storage-sqlite/src/backup.rs @@ -107,6 +107,49 @@ pub(crate) fn backup_server_slot_count(pool_size: u32) -> usize { .clamp(1, BACKUP_SERVER_SLOTS_MAX) } +/// The pool a server needs so that [`backup_server_slot_count`] admits every +/// job `backup.max_concurrent_jobs` admits: `pool_size` when its slot count +/// already meets the jobs, else the smallest pool whose slot count does, the +/// slot cap included (the rule is stated once, on `SqliteConfig::pool_size`). +/// The in-memory pool is not sized by this: it has one connection by nature. +pub(crate) fn pool_size_for_jobs(pool_size: u32, max_concurrent_jobs: usize) -> u32 { + let wanted = max_concurrent_jobs.min(BACKUP_SERVER_SLOTS_MAX); + if backup_server_slot_count(pool_size) >= wanted { + return pool_size; + } + let slots = u32::try_from(wanted).unwrap_or(u32::MAX); + slots.saturating_add(1).saturating_mul(2) +} + +/// The warning a server logs when [`pool_size_for_jobs`] raised its pool: +/// both numbers, the raised pool's slot count, and what the slots admit. +/// With the jobs within the slot cap the slots admit every job the +/// `max_concurrent_jobs` permit admits; over it the slots are capped at +/// [`BACKUP_SERVER_SLOTS_MAX`], and the line says which jobs the slot bound +/// refuses (the limits document's "above 16 jobs a 17th concurrent job is +/// refused by the slot"). +pub(crate) fn pool_raise_message( + pool_size: u32, + for_jobs: u32, + max_concurrent_jobs: usize, +) -> String { + let slots = backup_server_slot_count(for_jobs); + let admits = if slots < max_concurrent_jobs { + format!( + "the slots are capped at {BACKUP_SERVER_SLOTS_MAX}, so jobs {} to \ + {max_concurrent_jobs} are refused by the slot bound", + slots + 1 + ) + } else { + "the slot bound refuses no job the max_concurrent_jobs permit admitted".to_owned() + }; + format!( + "SQLite pool_size {pool_size} raised to {for_jobs} for backup.max_concurrent_jobs = \ + {max_concurrent_jobs}: a pool of {for_jobs} has {slots} server slots, one per running \ + backup job, so {admits}" + ) +} + /// How the slot refusal names its bound, beside the number: the engine's /// `max_concurrent_jobs` refusal carries the same text with its own name, so /// an operator can tell which bound a request met. The shared text reads @@ -3814,6 +3857,119 @@ mod restore_tests { assert!(raised[0].contains("pool_size >= 3"), "{raised:?}"); } + /// A server whose `backup.max_concurrent_jobs` is over the slot count of + /// its configured pool raises the pool, once, with a warning naming both + /// numbers, so the slot count meets the jobs; a pool whose slots already + /// meet them, and an engine opened with no jobs, are left as configured + /// and log nothing; the jobs over the slot cap raise the pool to the cap. + #[tokio::test] + async fn a_pool_raised_for_the_jobs_is_logged_and_its_slots_meet_them() { + use super::pool_size_for_jobs; + assert_eq!(pool_size_for_jobs(10, 4), 10, "the defaults need no raise"); + assert_eq!(pool_size_for_jobs(4, 4), 10); + assert_eq!(pool_size_for_jobs(4, 1), 4); + assert_eq!(pool_size_for_jobs(4, 2), 6); + assert_eq!(pool_size_for_jobs(2, 0), 2); + assert_eq!(pool_size_for_jobs(10, 16), 34); + assert_eq!( + pool_size_for_jobs(10, 1024), + 34, + "the slot cap bounds the raise" + ); + for (pool, jobs) in [(1, 0), (4, 1), (4, 4), (10, 4), (3, 16), (10, 1024)] { + assert!( + backup_server_slot_count(pool_size_for_jobs(pool, jobs)) >= jobs.min(16), + "pool {pool}, jobs {jobs}" + ); + } + + let target = std::env::var_os("CARGO_TARGET_DIR") + .map(std::path::PathBuf::from) + .unwrap_or_else(|| { + std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../../target") + }); + let path = target.join(format!("backup-test-{}.sqlite", uuid::Uuid::new_v4())); + let path = path.to_str().expect("UTF-8 path").to_owned(); + let (guard, logs) = crate::schema::log_capture::capture(); + let raised = SqliteEngine::new_for_server(&path, 4, 4, "us-east-1", 409_600) + .await + .expect("engine"); + assert_eq!( + raised.backup_server_slot_count, 4, + "the slots meet the jobs" + ); + assert!( + raised.backup_server_slots.available_permits() == 4, + "the semaphore has the raised count" + ); + close_file_engine(raised, std::path::Path::new(&path)).await; + let kept = SqliteEngine::new_for_server(&path, 10, 4, "us-east-1", 409_600) + .await + .expect("engine"); + assert_eq!(kept.backup_server_slot_count, 4); + close_file_engine(kept, std::path::Path::new(&path)).await; + let memory = SqliteEngine::new_for_server(":memory:", 10, 4, "us-east-1", 409_600) + .await + .expect("engine"); + assert_eq!( + memory.backup_server_slot_count, 1, + "an in-memory database has one connection and one slot whatever the jobs" + ); + drop(memory); + drop(guard); + let logs = logs.lock().expect("logs").clone(); + let raised: Vec<&String> = logs + .iter() + .filter(|l| l.contains("pool_size") && l.contains("raised")) + .collect(); + assert_eq!(raised.len(), 1, "{logs:#?}"); + assert!(raised[0].starts_with("WARN"), "{raised:?}"); + assert!( + raised[0].contains("pool_size 4 raised to 10 for backup.max_concurrent_jobs = 4"), + "{raised:?}" + ); + assert!(raised[0].contains("has 4 server slots"), "{raised:?}"); + } + + /// The pool-raise warning says what the raised pool's slots admit: with + /// the jobs within the slot cap, every job the permit admits; over the + /// cap (16), the jobs the slot bound refuses, 17 to `max_concurrent_jobs`, + /// as the limits document states it (a line saying the slot refused no + /// admitted job at 20 jobs was wrong: it refuses the 17th). + #[test] + fn the_pool_raise_warning_names_the_jobs_the_slot_cap_refuses() { + use super::{pool_raise_message, pool_size_for_jobs}; + let at_8 = pool_raise_message(10, pool_size_for_jobs(10, 8), 8); + assert!( + at_8.starts_with( + "SQLite pool_size 10 raised to 18 for backup.max_concurrent_jobs = 8: a pool of \ + 18 has 8 server slots, one per running backup job, so " + ), + "{at_8}" + ); + assert!( + at_8.ends_with( + "so the slot bound refuses no job the max_concurrent_jobs permit admitted" + ), + "{at_8}" + ); + let at_20 = pool_raise_message(10, pool_size_for_jobs(10, 20), 20); + assert!( + at_20.starts_with( + "SQLite pool_size 10 raised to 34 for backup.max_concurrent_jobs = 20: a pool of \ + 34 has 16 server slots, one per running backup job, so " + ), + "{at_20}" + ); + assert!( + at_20.ends_with( + "so the slots are capped at 16, so jobs 17 to 20 are refused by the slot bound" + ), + "{at_20}" + ); + assert!(!at_20.contains("refuses no job"), "{at_20}"); + } + /// A flood of CreateBackups from one account on the default pool of 10 /// does not starve another account's reads: at most /// `backup_server_slot_count` backups run at once (observed as CREATING @@ -4170,7 +4326,8 @@ mod restore_tests { use extenddb_storage::BackupCatalog; use extenddb_storage::backup_catalog::{ BackupRow, BackupStatus, DeleteStart, Heartbeat, JobStatus, RestoreTargetKey, - StatusFields, StoreIdentity, StoreKind, TimeBounds, UNOWNED_GENERATION, WriteOutcome, + StaleClock, StatusFields, StoreIdentity, StoreKind, TimeBounds, UNOWNED_GENERATION, + WriteOutcome, }; use futures::TryStreamExt; @@ -4313,6 +4470,7 @@ mod restore_tests { size_bytes: Some(1234), item_count: Some(5), failure_reason: None, + source_table_details_json: None, completed: true, }; assert_eq!( @@ -4980,6 +5138,371 @@ mod restore_tests { ); } + /// `claim_dead_backup` by the slot rule: a CREATING row whose slot no + /// task of this process holds is dead at once, whatever its heartbeat + /// and whatever the stale window (a row the last process left, here + /// inserted with a fresh heartbeat and tested with a window of an + /// hour); a row whose claim a task holds is live and refused; a row + /// whose claim dropped (the job's task ended by any route) is taken + /// over at once. The successor holds the slot: a second taker and a + /// `claim_backup` of the row are refused until the successor's claim + /// drops. The takeover mints a fresh generation and leaves + /// `heartbeat_at` as the dead owner wrote it. The successor's + /// AVAILABLE write carries `source_table_details_json` when `Some`, + /// leaves it when `None`, and stamps `completed_at`. An AVAILABLE row + /// and a missing row are refused, and the slot taken for the test is + /// given back. + #[tokio::test] + async fn claim_dead_backup_takes_over_a_row_whose_slot_no_task_holds() { + let engine = engine().await; + let stale = Duration::from_secs(3600); + let slot_key = |row: &BackupRow| format!("backup:{}", row.backup_id); + + // A row the last process left: inserted directly, no slot held, + // its heartbeat fresh. Dead at once. + let left = row(ACCOUNT, "Music", "left"); + let source_table = serde_json::to_string(&left.source_table_details_json).unwrap(); + sqlx::query(&format!( + "INSERT INTO backup_index (backup_id, account_id, table_name, backup_arn, \ + backup_name, status, store_kind, store_root, store_id, size_bytes, item_count, \ + created_at, source_table_details_json, owner_generation, heartbeat_at) \ + VALUES (?, ?, ?, ?, ?, 'CREATING', ?, ?, ?, 0, 0, ?, ?, 7, {NOW})", + NOW = crate::backup_catalog::NOW + )) + .bind(&left.backup_id) + .bind(&left.account_id) + .bind(&left.table_name) + .bind(&left.backup_arn) + .bind(&left.backup_name) + .bind(left.store.store_kind.as_str()) + .bind(&left.store.store_root) + .bind(&left.store.store_id) + .bind(crate::sqlite_util::format_epoch_seconds(left.created_at)) + .bind(&source_table) + .execute(&engine.pool) + .await + .expect("a row of the last process"); + let before = engine + .read_backup_row(&arn_of(&left)) + .await + .unwrap() + .unwrap(); + assert_eq!(before.owner_generation, 7); + assert!(!engine.job_slots.holds(&slot_key(&left))); + let successor = engine + .claim_dead_backup(&left.key(), stale) + .await + .expect("claim") + .expect("a row with no slot holder is dead at once, fresh heartbeat or not"); + assert_ne!(successor.generation(), 7); + assert_ne!(successor.generation(), UNOWNED_GENERATION); + let after = engine + .read_backup_row(&arn_of(&left)) + .await + .unwrap() + .unwrap(); + assert_eq!(after.owner_generation, successor.generation()); + assert_eq!(after.status, BackupStatus::Creating); + assert_eq!( + after.heartbeat_at, before.heartbeat_at, + "the takeover left the heartbeat as the dead owner wrote it" + ); + assert!( + engine.job_slots.holds(&slot_key(&left)), + "the successor holds the slot" + ); + assert!( + engine + .claim_dead_backup(&left.key(), stale) + .await + .expect("claim") + .is_none(), + "a row another task holds is refused" + ); + assert!( + matches!( + engine.claim_backup(&left).await.err(), + Some(StorageError::AlreadyExists(m)) if m.contains("a live job owns") + ), + "a job's claim of a taken-over row meets the successor's slot" + ); + assert_eq!( + successor.heartbeat().await.expect("heartbeat"), + Heartbeat::Confirmed + ); + // The AVAILABLE write with the manifest's source table: the row + // carries it; a later write with `None` leaves it. + let mut details = left.source_table_details_json.clone(); + details["item_count"] = serde_json::json!(4242); + let fields = StatusFields { + size_bytes: Some(99), + item_count: Some(4242), + failure_reason: None, + source_table_details_json: Some(details.clone()), + completed: true, + }; + assert_eq!( + successor + .set_status(JobStatus::Creating, JobStatus::Available, &fields) + .await + .unwrap(), + WriteOutcome::Matched + ); + let stored = engine + .read_backup_row(&arn_of(&left)) + .await + .unwrap() + .unwrap(); + assert_eq!(stored.status, BackupStatus::Available); + assert_eq!(stored.size_bytes, 99); + assert_eq!(stored.item_count, 4242); + assert_eq!( + stored.source_table_details_json, details, + "the AVAILABLE write carries the source table it was given" + ); + assert_ne!( + stored.source_table_details_json, + left.source_table_details_json + ); + assert!(stored.completed_at.is_some(), "completed_at is stamped"); + assert_eq!( + successor + .set_status( + JobStatus::Available, + JobStatus::Failed, + &StatusFields { + failure_reason: Some("later".to_owned()), + ..StatusFields::default() + } + ) + .await + .unwrap(), + WriteOutcome::Matched + ); + let stored = engine + .read_backup_row(&arn_of(&left)) + .await + .unwrap() + .unwrap(); + assert_eq!( + stored.source_table_details_json, details, + "a write without the member leaves the column" + ); + // Not CREATING any more: refused, and the slot taken for the test + // is given back. + drop(successor); + assert!( + !engine.job_slots.holds(&slot_key(&left)), + "the claim's drop frees the slot" + ); + assert!( + engine + .claim_dead_backup(&left.key(), stale) + .await + .expect("claim") + .is_none(), + "a row that is not CREATING is refused" + ); + assert!( + !engine.job_slots.holds(&slot_key(&left)), + "a refusal leaves no slot behind" + ); + + // A live owner: its claim holds the slot, so it is refused + // whatever the heartbeat; once the claim drops (the task ended), + // the row is dead at once. + let live = row(ACCOUNT, "Music", "live"); + let owner = engine.claim_backup(&live).await.expect("claim"); + sqlx::query( + "UPDATE backup_index SET heartbeat_at = \ + strftime('%Y-%m-%dT%H:%M:%fZ', 'now', '-1 day') WHERE backup_id = ?", + ) + .bind(&live.backup_id) + .execute(&engine.pool) + .await + .expect("age the heartbeat"); + assert!( + engine + .claim_dead_backup(&live.key(), Duration::from_secs(1)) + .await + .expect("claim") + .is_none(), + "a held slot is a live owner, a day-old heartbeat or not" + ); + assert_eq!( + engine + .read_backup_row(&arn_of(&live)) + .await + .unwrap() + .unwrap() + .owner_generation, + owner.generation(), + "a refusal leaves the generation" + ); + drop(owner); + let successor = engine + .claim_dead_backup(&live.key(), stale) + .await + .expect("claim") + .expect("the owner's task ended: dead at once"); + assert_ne!(successor.generation(), UNOWNED_GENERATION); + drop(successor); + + // A missing row: refused, no slot left behind. + let missing = row(ACCOUNT, "Music", "missing"); + assert!( + engine + .claim_dead_backup(&missing.key(), stale) + .await + .expect("claim") + .is_none() + ); + assert!(!engine.job_slots.holds(&slot_key(&missing))); + } + + /// `list_backup_rows_older_than`: the rows of a store with a status + /// whose clock column is older than the age on the database's clock. + /// The two clocks are told apart by a row heartbeated at the start + /// and FAILED after the window passed (as the reconciler fails a dead + /// owner's row): listed on the heartbeat clock at once, not on the + /// completed clock until its own window passes. A FAILED row without + /// `completed_at` is never listed on the completed clock. A CREATING + /// row and another store's row are not listed. The window is crossed + /// by waiting, not by dating rows. + #[tokio::test] + async fn rows_older_than_are_listed_on_the_named_clock() { + let engine = engine().await; + let window = Duration::from_secs(1); + // Names in the listing's order; two rows minted in one millisecond + // share `created_at` and fall to ARN order, so the sets compare + // sorted. + let names = |listed: Vec| -> Vec { + let mut names: Vec = listed.into_iter().map(|r| r.backup_name).collect(); + names.sort(); + names + }; + let older = |status: BackupStatus, clock: StaleClock| { + let engine = engine.clone(); + async move { + let rows: Vec = engine + .list_backup_rows_older_than(status, &store(), clock, window) + .try_collect() + .await + .expect("list"); + names(rows) + } + }; + async fn fail(claim: &dyn extenddb_storage::backup_catalog::JobClaim, completed: bool) { + let fields = StatusFields { + failure_reason: Some("test".to_owned()), + completed, + ..StatusFields::default() + }; + assert_eq!( + claim + .set_status(JobStatus::Creating, JobStatus::Failed, &fields) + .await + .unwrap(), + WriteOutcome::Matched + ); + } + + // All heartbeated now. `late` stays CREATING until the window has + // passed; `unfinished` is FAILED without `completed_at`; + // `creating` stays CREATING; `elsewhere` is another store's; + // `gone` is DELETED now. + let late = row(ACCOUNT, "Music", "late"); + let late_claim = engine.claim_backup(&late).await.expect("claim"); + let unfinished = row(ACCOUNT, "Music", "unfinished"); + let unfinished_claim = engine.claim_backup(&unfinished).await.expect("claim"); + fail(unfinished_claim.as_ref(), false).await; + let creating = row(ACCOUNT, "Music", "creating"); + let _creating_claim = engine.claim_backup(&creating).await.expect("claim"); + let mut elsewhere = row(ACCOUNT, "Music", "elsewhere"); + elsewhere.store.store_id = "store-2".to_owned(); + let elsewhere_claim = engine.claim_backup(&elsewhere).await.expect("claim"); + fail(elsewhere_claim.as_ref(), true).await; + let gone = row(ACCOUNT, "Music", "gone"); + let gone_claim = engine.claim_backup(&gone).await.expect("claim"); + assert_eq!( + gone_claim + .set_status( + JobStatus::Creating, + JobStatus::Available, + &StatusFields::default() + ) + .await + .unwrap(), + WriteOutcome::Matched + ); + assert_eq!( + engine.start_backup_delete(&gone.key()).await.unwrap(), + DeleteStart::Deleting + ); + assert!(engine.set_backup_deleted(&gone.key()).await.unwrap()); + + // Inside the window on every clock: nothing is older. + assert!( + older(BackupStatus::Failed, StaleClock::Heartbeat) + .await + .is_empty() + ); + assert!( + older(BackupStatus::Failed, StaleClock::Completed) + .await + .is_empty() + ); + assert!( + older(BackupStatus::Deleted, StaleClock::Completed) + .await + .is_empty() + ); + + tokio::time::sleep(window + Duration::from_millis(300)).await; + // The reconciler's shape: the owner's last beat is past the + // window, the FAILED write is now. + fail(late_claim.as_ref(), true).await; + assert_eq!( + older(BackupStatus::Failed, StaleClock::Heartbeat).await, + ["late", "unfinished"], + "both heartbeats are past the window; the just-failed row is listed on the \ + heartbeat clock at once" + ); + assert!( + older(BackupStatus::Failed, StaleClock::Completed) + .await + .is_empty(), + "completed just now, or never" + ); + assert_eq!( + older(BackupStatus::Deleted, StaleClock::Completed).await, + ["gone"] + ); + let elsewhere_rows: Vec = engine + .list_backup_rows_older_than( + BackupStatus::Failed, + &elsewhere.store, + StaleClock::Heartbeat, + window, + ) + .try_collect() + .await + .expect("list"); + assert_eq!(names(elsewhere_rows), ["elsewhere"], "scoped to the store"); + + tokio::time::sleep(window + Duration::from_millis(300)).await; + assert_eq!( + older(BackupStatus::Failed, StaleClock::Completed).await, + ["late"], + "the completed window crossed on the database's clock; a row without \ + completed_at is never listed on this clock" + ); + drop(late_claim); + drop(unfinished_claim); + drop(elsewhere_claim); + drop(gone_claim); + } + /// The maintenance claim: taken once, refused to a second holder /// naming the first, heartbeated, released; a holder whose heartbeat /// is older than the stale window is dead, so its claim is not diff --git a/crates/storage-sqlite/src/backup_catalog.rs b/crates/storage-sqlite/src/backup_catalog.rs index a22931da5..b26c36317 100644 --- a/crates/storage-sqlite/src/backup_catalog.rs +++ b/crates/storage-sqlite/src/backup_catalog.rs @@ -8,10 +8,13 @@ //! job's claim is an in-process slot ([`JobSlots`]): a key held in a set for //! the job's life and released by a drop guard, so a task that ends by any //! route, a panic included, frees it. A restart frees every slot at once, -//! which is the proof of death this backend needs: a CREATING in-catalog -//! backup row whose slot no task holds is dead, and every RUNNING restore -//! job record of the stopped process is aged at startup so the reconciler's -//! first cycle finds it ([`SqliteEngine::expire_restore_job_heartbeats`]). +//! which is the proof of death this backend needs: a CREATING row (a +//! `backup_index` row or an in-catalog `backups` row) whose slot no task +//! holds is dead, whatever its heartbeat says, so the reconciler's takeover +//! ([`BackupCatalog::claim_dead_backup`]) reads no stale window here, and +//! every RUNNING restore job record of the stopped process is aged at startup +//! so the reconciler's first cycle finds it +//! ([`SqliteEngine::expire_restore_job_heartbeats`]). //! //! Every write a job makes to its row goes through its claim and carries the //! claim's `owner_generation`, so a row a successor took over matches no @@ -37,8 +40,8 @@ use extenddb_core::backup_format::BackupArn; use extenddb_storage::JobPermit; use extenddb_storage::backup_catalog::{ BackupCatalog, BackupKey, BackupRow, BackupStatus, DeleteStart, Heartbeat, JobClaim, JobRow, - JobStatus, MaintenanceClaim, RestoreJobRow, RestoreKind, RestoreTargetKey, StatusFields, - StoreIdentity, StoreKind, TimeBounds, WriteOutcome, new_generation, + JobStatus, MaintenanceClaim, RestoreJobRow, RestoreKind, RestoreTargetKey, StaleClock, + StatusFields, StoreIdentity, StoreKind, TimeBounds, WriteOutcome, new_generation, }; use extenddb_storage::error::StorageError; use futures::future::BoxFuture; @@ -223,11 +226,20 @@ impl JobClaim for SqliteJobClaim { let _writer = self.engine.write_lock.lock().await; let affected = match &self.record { JobRecord::Backup(key) => { + let source_table = fields + .source_table_details_json + .as_ref() + .map(serde_json::to_string) + .transpose() + .map_err(|e| { + StorageError::Internal(format!("source table details: {e}")) + })?; sqlx::query(&format!( "UPDATE backup_index SET status = ?, \ size_bytes = COALESCE(?, size_bytes), \ item_count = COALESCE(?, item_count), \ failure_reason = COALESCE(?, failure_reason), \ + source_table_details_json = COALESCE(?, source_table_details_json), \ completed_at = CASE WHEN ? THEN {NOW} ELSE completed_at END \ WHERE account_id = ? AND backup_id = ? AND owner_generation = ? \ AND status = ?" @@ -236,6 +248,7 @@ impl JobClaim for SqliteJobClaim { .bind(fields.size_bytes) .bind(fields.item_count) .bind(fields.failure_reason.as_deref()) + .bind(source_table) .bind(fields.completed) .bind(&key.account_id) .bind(&key.backup_id) @@ -743,6 +756,87 @@ impl BackupCatalog for SqliteEngine { }) } + fn claim_dead_backup<'a>( + &'a self, + key: &'a BackupKey, + _stale_after: Duration, + ) -> BoxFuture<'a, Result>, StorageError>> { + Box::pin(async move { + // The slot is the liveness test (module doc): a job of this + // process that holds it is alive whatever its heartbeat says, and + // a row without a holder is dead at once, so the stale window is + // not read. Taking the slot is the test and the exclusion of a + // second taker in one step. + let Some(slot) = JobSlots::take(&self.job_slots, backup_slot_key(&key.backup_id)) + else { + return Ok(None); + }; + let generation = new_generation(); + let _writer = self.write_lock.lock().await; + let mut tx = self + .pool + .begin_with("BEGIN IMMEDIATE") + .await + .map_err(db_err)?; + // `heartbeat_at` stays as the dead owner wrote it (trait doc). + let taken = sqlx::query( + "UPDATE backup_index SET owner_generation = ? \ + WHERE account_id = ? AND backup_id = ? AND status = 'CREATING'", + ) + .bind(generation_column(generation)) + .bind(&key.account_id) + .bind(&key.backup_id) + .execute(&mut *tx) + .await + .map_err(db_err)? + .rows_affected(); + tx.commit().await.map_err(db_err)?; + if taken != 1 { + drop(slot); + return Ok(None); + } + Ok(Some(Box::new(SqliteJobClaim { + engine: self.clone(), + _slot: Some(slot), + _permit: None, + generation, + record: JobRecord::Backup(key.clone()), + }) as Box)) + }) + } + + fn list_backup_rows_older_than<'a>( + &'a self, + status: BackupStatus, + store: &'a StoreIdentity, + clock: StaleClock, + age: Duration, + ) -> BoxStream<'a, Result> { + let column = match clock { + StaleClock::Heartbeat => "heartbeat_at", + StaleClock::Completed => "completed_at", + }; + rows_stream(async move { + // On the database's clock, as `remove_dead_creating_backup_row` + // measures; `julianday(NULL)` is NULL, so a row with + // `completed_at` unset is never older than any age on that clock. + sqlx::query_as(&format!( + "SELECT {ROW_COLUMNS} FROM backup_index \ + WHERE store_kind = ? AND store_root = ? AND store_id = ? AND status = ? \ + AND julianday({column}) < julianday('now') - ? / 86400.0 \ + ORDER BY created_at, backup_arn" + )) + .bind(store.store_kind.as_str()) + .bind(&store.store_root) + .bind(&store.store_id) + .bind(status.as_str()) + .bind(age.as_secs_f64()) + .fetch_all(&self.pool) + .await + .map_err(db_err) + }) + } + fn backup_in_use<'a>( &'a self, key: &'a BackupKey, diff --git a/crates/storage-sqlite/src/config.rs b/crates/storage-sqlite/src/config.rs index b0d947d56..24987988f 100644 --- a/crates/storage-sqlite/src/config.rs +++ b/crates/storage-sqlite/src/config.rs @@ -24,11 +24,17 @@ use serde::Deserialize; /// stream (file-backed or in-memory) holds one. A snapshot or a `CreateBackup` /// that finds no slot free is refused at once with `LimitExceededException`, /// so no backup runs while snapshots hold every slot. With one slot, one held -/// snapshot refuses every `CreateBackup` for as long as it is open. Running -/// backups hold one connection each for their read snapshot plus one shared -/// writer (`n` backups hold `n + 1`), so from `pool_size` 4 up a flood of -/// backups leaves at least half the pool to other requests; with 2 or 3 the -/// one admitted backup holds two of them. +/// snapshot refuses every `CreateBackup` for as long as it is open. A server +/// with `backup.max_concurrent_jobs` over the slot count raises a file-backed +/// `pool_size` to `2 * (max_concurrent_jobs + 1)` (10 at the default of 4, +/// which is the default pool; 34 at most, where the 16-slot cap is reached), +/// logging a warning that names both numbers, so the slot never refuses a job +/// the engine's permit admitted (`backup::pool_size_for_jobs` computes it); +/// an in-memory database has one connection and one slot whatever the jobs. +/// Running backups hold one connection each for their read snapshot plus one +/// shared writer (`n` backups hold `n + 1`), so from `pool_size` 4 up a flood +/// of backups leaves at least half the pool to other requests; with 2 or 3 +/// the one admitted backup holds two of them. #[derive(Debug, Clone, Deserialize)] #[serde(deny_unknown_fields)] pub struct SqliteConfig { diff --git a/crates/storage-sqlite/src/lib.rs b/crates/storage-sqlite/src/lib.rs index 3f2f7cbab..a6d7aa8a9 100644 --- a/crates/storage-sqlite/src/lib.rs +++ b/crates/storage-sqlite/src/lib.rs @@ -256,12 +256,18 @@ fn sqlite_server_components_factory( .await .map_err(BackendError::Catalog)?; } - let mut engine = SqliteEngine::new(&db_path, pool_size, ®ion, MAX_ITEM_SIZE_BYTES) - .await - .map_err(|e| BackendError::ConnectionFailed { - backend: "sqlite".to_owned(), - details: e.to_string(), - })?; + let mut engine = SqliteEngine::new_for_server( + &db_path, + pool_size, + options.max_concurrent_jobs, + ®ion, + MAX_ITEM_SIZE_BYTES, + ) + .await + .map_err(|e| BackendError::ConnectionFailed { + backend: "sqlite".to_owned(), + details: e.to_string(), + })?; if options.beside_server { return components_of(engine, &db_path).await; } diff --git a/crates/storage-sqlite/src/snapshot_engine.rs b/crates/storage-sqlite/src/snapshot_engine.rs index e79fc6da0..4b719b8fb 100644 --- a/crates/storage-sqlite/src/snapshot_engine.rs +++ b/crates/storage-sqlite/src/snapshot_engine.rs @@ -55,12 +55,23 @@ //! LIMIT n`), which `SQLite` serves from the primary key index. Multi-part //! range keys fall back to rowid order and report `key_ordered = false`. //! -//! Memory: one page is `LIMIT paging::DEFAULT_FETCH_BATCH` (1000) rows, all -//! of them held as item text at once. At the default 400 KB item limit one -//! row's text can reach 27.7 MB (the measured worst case in #394's line-cap -//! table), so a page is bounded by rows, not bytes. A byte budget (keyset -//! paging with a byte cutoff) is required before the stream gets a caller; -//! the backup design assigns it to the backup job PR. +//! Memory: a page ends at `paging::fetch_batch()` rows (1000) or after the +//! row that takes its item text to or past `paging::fetch_bytes()` (64 MiB, +//! the budget `extenddb_storage::snapshot::paging` states), whichever comes +//! first, so a page holds at most the budget plus one row of item text. The +//! cut is applied while the page's rows are pulled from the query one at a +//! time (`fetch_page`): the pull stops at the cut and the statement is +//! reset, so the stream reads no row past it and the next page starts after +//! the last row pulled. The driver's worker thread runs the statement ahead +//! of the pull and buffers the rows it has read: that run-ahead is bounded by +//! the pool's `row_buffer_size` (`store.rs`), not by the budget, at five +//! rows (the four the channel holds and the one the worker holds while its +//! `send` waits), 160 MiB at the 32 MiB line cap, and is the one term of a +//! page's memory the budget does not state. A page cut by the budget is short +//! without being the end of the table; the end is a page under the row bound +//! that the budget did not cut. The in-memory copy is a different bound: it +//! holds the whole table decoded, and the slot count, not the budget, bounds +//! how many copies exist at once. use extenddb_core::types::{Item, ScalarAttributeType, TableDescription}; use extenddb_storage::error::StorageError; @@ -266,9 +277,10 @@ fn keyset_stream( buffer: Vec::new().into_iter(), done: false, }; - // Read the page size once so a change to the test knob during a stream - // cannot end it early or late. + // Read the page size and the byte budget once so a change to either + // test knob during a stream cannot end it early or late. let batch = paging::fetch_batch(); + let budget = paging::fetch_bytes(); let limit = i64::try_from(batch).unwrap_or(i64::MAX); Box::pin(futures::stream::unfold( state, @@ -282,8 +294,16 @@ fn keyset_stream( } let tx = state.tx.as_mut()?; paging::record_fetch(); - let page = fetch_page(tx, &state.table, &state.paging, &state.cursor, limit).await; - let (items, next_cursor) = match page { + let page = fetch_page( + tx, + &state.table, + &state.paging, + &state.cursor, + limit, + budget, + ) + .await; + let Page { items, next, cut } = match page { Ok(page) => page, Err(e) => { state.done = true; @@ -292,10 +312,13 @@ fn keyset_stream( return Some((Err(e), state)); } }; - if let Some(next) = next_cursor { + if let Some(next) = next { state.cursor = next; } - if items.len() < batch { + // A short page shows the end only when the budget did not + // cut it: a cut page is short because the pull stopped, not + // because the table did. + if items.len() < batch && !cut { state.done = true; if let Some(tx) = state.tx.take() { let _ = tx.commit().await; @@ -308,127 +331,118 @@ fn keyset_stream( )) } -/// Query one page of up to `limit` rows for the paging mode and cursor. -async fn query_page( - tx: &mut Transaction<'static, Sqlite>, - table: &str, - paging: &Paging, - cursor: &Cursor, - limit: i64, -) -> Result, StorageError> { - match (paging, cursor) { +/// The SQL of one page for the paging mode and cursor: the key columns and +/// `item_data`, in key order, after the cursor when there is one, `LIMIT ?` +/// last. +fn page_sql(table: &str, paging: &Paging, cursor: &Cursor) -> Result { + Ok(match (paging, cursor) { (Paging::Pk, Cursor::Start) => { - sqlx::query(&format!( - "SELECT pk, item_data FROM {table} ORDER BY pk LIMIT ?" - )) - .bind(limit) - .fetch_all(&mut **tx) - .await + format!("SELECT pk, item_data FROM {table} ORDER BY pk LIMIT ?") } - (Paging::Pk, Cursor::Pk(last)) => { - sqlx::query(&format!( - "SELECT pk, item_data FROM {table} WHERE pk > ? ORDER BY pk LIMIT ?" - )) - .bind(last) - .bind(limit) - .fetch_all(&mut **tx) - .await + (Paging::Pk, Cursor::Pk(_)) => { + format!("SELECT pk, item_data FROM {table} WHERE pk > ? ORDER BY pk LIMIT ?") } (Paging::PkSk { sk_col, .. }, Cursor::Start) => { - sqlx::query(&format!( - "SELECT pk, {sk_col}, item_data FROM {table} ORDER BY pk, {sk_col} LIMIT ?" - )) - .bind(limit) - .fetch_all(&mut **tx) - .await - } - (Paging::PkSk { sk_col, .. }, Cursor::PkSkText(pk, sk)) => { - sqlx::query(&format!( - "SELECT pk, {sk_col}, item_data FROM {table} \ - WHERE (pk, {sk_col}) > (?, ?) ORDER BY pk, {sk_col} LIMIT ?" - )) - .bind(pk) - .bind(sk) - .bind(limit) - .fetch_all(&mut **tx) - .await + format!("SELECT pk, {sk_col}, item_data FROM {table} ORDER BY pk, {sk_col} LIMIT ?") } - (Paging::PkSk { sk_col, .. }, Cursor::PkSkBlob(pk, sk)) => { - sqlx::query(&format!( + (Paging::PkSk { sk_col, .. }, Cursor::PkSkText(..) | Cursor::PkSkBlob(..)) => { + format!( "SELECT pk, {sk_col}, item_data FROM {table} \ WHERE (pk, {sk_col}) > (?, ?) ORDER BY pk, {sk_col} LIMIT ?" - )) - .bind(pk) - .bind(sk) - .bind(limit) - .fetch_all(&mut **tx) - .await + ) } (Paging::Rowid, Cursor::Start) => { - sqlx::query(&format!( - "SELECT rowid, item_data FROM {table} ORDER BY rowid LIMIT ?" - )) - .bind(limit) - .fetch_all(&mut **tx) - .await + format!("SELECT rowid, item_data FROM {table} ORDER BY rowid LIMIT ?") } - (Paging::Rowid, Cursor::Rowid(last)) => { - sqlx::query(&format!( - "SELECT rowid, item_data FROM {table} WHERE rowid > ? ORDER BY rowid LIMIT ?" - )) - .bind(last) - .bind(limit) - .fetch_all(&mut **tx) - .await + (Paging::Rowid, Cursor::Rowid(_)) => { + format!("SELECT rowid, item_data FROM {table} WHERE rowid > ? ORDER BY rowid LIMIT ?") } _ => { return Err(StorageError::Internal( "snapshot cursor does not match its paging mode".to_owned(), )); } - } - .map_err(|e| db_err(&e)) + }) +} + +/// The cursor a page's last row leaves for the next page. +fn cursor_after(row: &sqlx::sqlite::SqliteRow, paging: &Paging) -> Result { + Ok(match paging { + Paging::Pk => Cursor::Pk(row.try_get("pk").map_err(|e| db_err(&e))?), + Paging::PkSk { sk_col, sk_type } => { + let pk: String = row.try_get("pk").map_err(|e| db_err(&e))?; + match sk_type { + ScalarAttributeType::B => Cursor::PkSkBlob( + pk, + row.try_get::, _>(*sk_col).map_err(|e| db_err(&e))?, + ), + ScalarAttributeType::S | ScalarAttributeType::N => Cursor::PkSkText( + pk, + row.try_get::(*sk_col).map_err(|e| db_err(&e))?, + ), + } + } + Paging::Rowid => Cursor::Rowid(row.try_get("rowid").map_err(|e| db_err(&e))?), + }) } -/// One page: items and the cursor for the next page. Fewer items than -/// `limit` means the table is exhausted. +/// One page as [`fetch_page`] returns it. +struct Page { + items: Vec, + /// The cursor for the next page; `None` for an empty page. + next: Option, + /// Whether the byte budget ended the page before `limit` rows. + cut: bool, +} + +/// One page: at most `limit` rows, pulled one at a time from the page query +/// and stopped after the row that takes the page's item text to or past +/// `budget` bytes, so a page holds at most the budget plus one row and at +/// least one row, and the stream pulls no row past the cut (the statement is +/// reset when the row stream is dropped, with the rows the driver's worker +/// read ahead, at most the pool's `row_buffer_size` plus the one the worker +/// holds, five; the next page starts at the cursor the last pulled row +/// leaves). Fewer rows than `limit` +/// without a cut means the table is exhausted. async fn fetch_page( tx: &mut Transaction<'static, Sqlite>, table: &str, paging: &Paging, cursor: &Cursor, limit: i64, -) -> Result<(Vec, Option), StorageError> { - let rows = query_page(tx, table, paging, cursor, limit).await?; - let mut items = Vec::with_capacity(rows.len()); + budget: usize, +) -> Result { + use futures::TryStreamExt as _; + let sql = page_sql(table, paging, cursor)?; + let query = sqlx::query(&sql); + let query = match cursor { + Cursor::Start => query, + Cursor::Pk(pk) => query.bind(pk), + Cursor::PkSkText(pk, sk) => query.bind(pk).bind(sk), + Cursor::PkSkBlob(pk, sk) => query.bind(pk).bind(sk.as_slice()), + Cursor::Rowid(rowid) => query.bind(rowid), + } + .bind(limit); + let mut rows = query.fetch(&mut **tx); + let mut items = Vec::new(); + let mut bytes = 0usize; let mut next = None; - for row in &rows { + let mut cut = false; + while let Some(row) = rows.try_next().await.map_err(|e| db_err(&e))? { let data: String = row.try_get("item_data").map_err(|e| db_err(&e))?; + bytes += data.len(); let item: Item = serde_json::from_str(&data) .map_err(|e| StorageError::Internal(format!("Parse item: {e}")))?; items.push(item); + next = Some(cursor_after(&row, paging)?); + if bytes >= budget { + cut = true; + break; + } } - if let Some(last) = rows.last() { - next = Some(match paging { - Paging::Pk => Cursor::Pk(last.try_get("pk").map_err(|e| db_err(&e))?), - Paging::PkSk { sk_col, sk_type } => { - let pk: String = last.try_get("pk").map_err(|e| db_err(&e))?; - match sk_type { - ScalarAttributeType::B => Cursor::PkSkBlob( - pk, - last.try_get::, _>(*sk_col) - .map_err(|e| db_err(&e))?, - ), - ScalarAttributeType::S | ScalarAttributeType::N => Cursor::PkSkText( - pk, - last.try_get::(*sk_col).map_err(|e| db_err(&e))?, - ), - } - } - Paging::Rowid => Cursor::Rowid(last.try_get("rowid").map_err(|e| db_err(&e))?), - }); - } - Ok((items, next)) + drop(rows); + paging::record_page_bytes(bytes); + Ok(Page { items, next, cut }) } #[cfg(test)] @@ -550,6 +564,33 @@ mod tests { }, ) .await; + // The byte budget: pages are cut after the row that reaches it, every + // row streams once across cut pages, and the rows past a cut are + // never read (the pull stops at the cut), which `rows_past_cut` + // reports as zero here. + let bounds = snapshot_conformance::run_page_bounds( + &hooks, + &*engine, + ACCOUNT, + "snap_bounds", + || async { + engine + .process_control_plane_transitions() + .await + .expect("pump"); + }, + ) + .await; + assert_eq!( + bounds.rows_past_cut, 0, + "SQLite pulls rows one at a time and stops at the cut: {bounds:?}" + ); + assert_eq!( + bounds.pages, + (snapshot_conformance::PAGE_BOUNDS_ITEMS as u64) + .div_ceil(snapshot_conformance::PAGE_BOUNDS_ROWS as u64), + "a page is exactly the budget's rows: {bounds:?}" + ); let pool = engine.pool.clone(); snapshot_conformance::run_refusals( &hooks, diff --git a/crates/storage-sqlite/src/store.rs b/crates/storage-sqlite/src/store.rs index e5daac6de..77555c093 100644 --- a/crates/storage-sqlite/src/store.rs +++ b/crates/storage-sqlite/src/store.rs @@ -38,17 +38,24 @@ //! so the invariant stays auditable, is init-time bootstrap (in this file and //! in `bootstrapper.rs`), which runs before the server serves traffic. +use std::str::FromStr; use std::sync::Arc; use std::sync::atomic::AtomicU64; use extenddb_storage::error::StorageError; use sqlx::SqlitePool; -use sqlx::sqlite::SqlitePoolOptions; +use sqlx::sqlite::{SqliteConnectOptions, SqlitePoolOptions}; use tokio::sync::Mutex; use crate::INDEX_PROPAGATION_DELAY_QUERY; use crate::sqlite_util::sqlite_url; +/// Rows the driver's worker buffers ahead of the reading task (see +/// `SqliteEngine::new_for_server`). The rows ahead of the reader are one +/// more than this: the channel holds this many and the worker, blocked on +/// its `send`, holds the next row it has already stepped and copied. +const ROW_BUFFER_ROWS: usize = 4; + /// The SQLite storage engine. Implements `StorageEngine` (all data/metadata/ /// stream/worker/backup traits) over a single SQLite database. #[derive(Clone)] @@ -95,11 +102,30 @@ impl SqliteEngine { /// Open (creating if necessary) the SQLite database and build the engine. /// /// `path_or_url` accepts a filesystem path, `:memory:`, or a `sqlite:` URL. + /// The pool is sized for no backup jobs; a server opens the engine with + /// [`Self::new_for_server`]. pub async fn new( path_or_url: &str, pool_size: u32, region: &str, max_item_size_bytes: usize, + ) -> Result { + Self::new_for_server(path_or_url, pool_size, 0, region, max_item_size_bytes).await + } + + /// [`Self::new`] for a server that runs up to `max_concurrent_jobs` + /// backup and restore jobs at once: a file-backed pool is raised, with a + /// warning naming both numbers, until its server slots admit every job + /// the engine's permit admits (`backup::pool_size_for_jobs`; the rule is + /// stated on `SqliteConfig::pool_size`), so no job the permit admitted + /// is refused by the slot. The in-memory pool has one connection by + /// nature and is not raised. + pub async fn new_for_server( + path_or_url: &str, + pool_size: u32, + max_concurrent_jobs: usize, + region: &str, + max_item_size_bytes: usize, ) -> Result { let url = sqlite_url(path_or_url); // Use the same parsed-location classifier as the serve lock. String @@ -107,6 +133,18 @@ impl SqliteEngine { let in_memory = crate::serve_lock::database_file(path_or_url) .map_err(StorageError::Connection)? .is_none(); + let for_jobs = if in_memory { + pool_size + } else { + crate::backup::pool_size_for_jobs(pool_size, max_concurrent_jobs) + }; + if for_jobs != pool_size { + tracing::warn!( + "{}", + crate::backup::pool_raise_message(pool_size, for_jobs, max_concurrent_jobs) + ); + } + let pool_size = for_jobs; // An in-memory database lives only inside its own connection: a second // connection opens a *separate* empty database. So for an in-memory // location we pin the pool to a single connection that is never @@ -129,13 +167,25 @@ impl SqliteEngine { if in_memory { opts = opts.idle_timeout(None).max_lifetime(None); } + // The driver runs each statement on a worker thread that buffers rows + // ahead of the task reading them (sqlx's `row_buffer_size`, 50 by + // default) over a bounded channel; the worker steps and copies the + // next row before its `send` blocks, so the rows ahead of the reader + // are the buffer plus one. A table snapshot reads rows of up to the + // 32 MiB line cap and stops a page at the byte budget, so a 50-row + // buffer is up to 1.6 GB the page never asked for; ROW_BUFFER_ROWS + // bounds the run-ahead at five rows (160 MiB at the line cap), which + // costs ordinary queries a channel hand-off every four rows. + let connect = SqliteConnectOptions::from_str(&url) + .map_err(|e| StorageError::Connection(e.to_string()))? + .row_buffer_size(ROW_BUFFER_ROWS); let pool = opts .after_connect(|conn, _| { Box::pin( async move { crate::sqlite_util::set_connection_pragmas(conn, true).await }, ) }) - .connect(&url) + .connect_with(connect) .await .map_err(|e| StorageError::Connection(e.to_string()))?; diff --git a/crates/storage-sqlite/tests/backup_job_panic.rs b/crates/storage-sqlite/tests/backup_job_panic.rs new file mode 100644 index 000000000..d366de136 --- /dev/null +++ b/crates/storage-sqlite/tests/backup_job_panic.rs @@ -0,0 +1,338 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! A CreateBackup job whose task panics on SQLite leaves a CREATING row that +//! the reconciler's next cycle resolves to FAILED, in the same process, with +//! no restart (design 4(e): at runtime a row whose in-process slot no task +//! holds is dead; PR 8's acceptance). +//! +//! The engine is built the way the server builds it (the backend's server +//! components factory, its workers spawned through the runtime hooks, so a +//! table reaches ACTIVE as it does under `serve`), the store is a filesystem +//! store whose `put` panics, and the request goes through the engine's +//! `dispatch`. The job's task panics at its first put; the task's drop +//! releases the `Arc` and with it the row's `JobSlot`, so the +//! SQLite catalog's `claim_dead_backup` takes the row over at once, whatever +//! its heartbeat says, and the reconciler writes FAILED (no manifest) and +//! then, since the dead owner's heartbeat is within the window, leaves the +//! prefix for a later cycle. + +use std::os::unix::fs::DirBuilderExt; +use std::path::PathBuf; +use std::sync::Arc; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::time::Duration; + +use extenddb_backup_store::{BackupStore, ByteStream, FilesystemStore, ObjectMeta, StoreError}; +use extenddb_core::backup_format::BackupArn; +use extenddb_core::limits::LimitsConfig; +use extenddb_engine::reconciler::{self, OWNER_DEAD_NO_MANIFEST, ReconcilerState}; +use extenddb_engine::{ + BackupRuntime, ConfiguredStore, OperationContext, check_configured_store, dispatch, +}; +use extenddb_storage::backup_catalog::{BackupStatus, StoreKind}; +use extenddb_storage::server_components::ServerComponentsOptions; +use extenddb_storage::{CancellationToken, WorkerContext}; +use extenddb_storage_sqlite::SqliteConfig; +use futures::future::BoxFuture; +use futures::stream::BoxStream; +use serde_json::json; + +const REGION: &str = "us-east-1"; + +/// A filesystem store whose `put` panics: every other call reaches the real +/// store, so the identity check, the listing and the delete behave as on a +/// server; only the job's first object write dies. +struct PanicOnPut { + inner: FilesystemStore, + puts: AtomicUsize, +} + +impl BackupStore for PanicOnPut { + fn put<'a>( + &'a self, + key: &'a str, + _body: ByteStream, + ) -> BoxFuture<'a, Result> { + self.puts.fetch_add(1, Ordering::SeqCst); + Box::pin(async move { panic!("injected panic in put of {key}") }) + } + + fn put_if_absent<'a>( + &'a self, + key: &'a str, + body: ByteStream, + ) -> BoxFuture<'a, Result, StoreError>> { + self.inner.put_if_absent(key, body) + } + + fn get<'a>(&'a self, key: &'a str) -> BoxFuture<'a, Result> { + self.inner.get(key) + } + + fn head<'a>(&'a self, key: &'a str) -> BoxFuture<'a, Result, StoreError>> { + self.inner.head(key) + } + + fn list<'a>(&'a self, prefix: &'a str) -> BoxStream<'a, Result> { + self.inner.list(prefix) + } + + fn delete_prefix<'a>(&'a self, prefix: &'a str) -> BoxFuture<'a, Result> { + self.inner.delete_prefix(prefix) + } + + fn validate(&self) -> BoxFuture<'_, Result<(), StoreError>> { + self.inner.validate() + } +} + +/// A scratch directory under the cargo target directory, removed on drop. +struct Scratch(PathBuf); + +impl Drop for Scratch { + fn drop(&mut self) { + let _ = std::fs::remove_dir_all(&self.0); + } +} + +async fn call( + ctx: &OperationContext, + operation: &str, + body: serde_json::Value, +) -> serde_json::Value { + dispatch(operation, body, ctx, "127.0.0.1:0") + .await + .unwrap_or_else(|e| panic!("{operation}: {e:?}")) + .body +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_job_task_that_panics_leaves_a_row_the_next_cycle_resolves_without_a_restart() { + let scratch = Scratch( + PathBuf::from(env!("CARGO_TARGET_TMPDIR")) + .join(format!("backup-job-panic-{}", uuid::Uuid::new_v4())), + ); + std::fs::create_dir_all(&scratch.0).expect("scratch directory"); + let db_path = scratch.0.join("catalog.sqlite"); + let root = scratch.0.join("backups"); + std::fs::DirBuilder::new() + .mode(0o700) + .create(&root) + .expect("root"); + + // The engine as the server builds it: a dev bootstrap of a fresh file + // (the migrations, the default account), the workers spawned. + let config = SqliteConfig { + path: db_path.to_str().expect("UTF-8 path").to_owned(), + pool_size: 4, + }; + let components = (extenddb_storage_sqlite::backend().server_components)(&config, REGION, { + let mut options = ServerComponentsOptions::default(); + options.bootstrap_if_uninitialized = true; + options.release = "test"; + options.max_concurrent_jobs = 4; + options + }) + .await + .unwrap_or_else(|e| panic!("server components: {e:?}")); + let storage = components.engine; + let shutdown = CancellationToken::new(); + let (_layer, reload_handle) = + tracing_subscriber::reload::Layer::new(tracing_subscriber::EnvFilter::new("info")); + let worker_ctx = WorkerContext { + metrics: Arc::new(extenddb_core::metrics::MetricsCollector::new()), + catalog_store: components.catalog_store.clone(), + reload_handle, + config_log_level: "info".to_owned(), + shutdown: shutdown.clone(), + }; + let hooks = components + .runtime_hooks + .expect("the SQLite backend has runtime hooks"); + let workers = hooks.spawn_workers(&worker_ctx).await; + let account_id: String = { + let pool = sqlx::SqlitePool::connect(&format!("sqlite://{}", db_path.display())) + .await + .expect("a second connection to the catalog"); + sqlx::query("INSERT OR REPLACE INTO settings (key, value) VALUES ('control_plane_delay_seconds', '0.05')") + .execute(&pool) + .await + .expect("setting"); + let id = sqlx::query_scalar("SELECT value FROM settings WHERE key = 'default_account_id'") + .fetch_one(&pool) + .await + .expect("the bootstrapped account"); + pool.close().await; + id + }; + + // The store: a healthy filesystem store (its identity file created by + // the check the server runs at startup) whose `put` panics. + let store = Arc::new(PanicOnPut { + inner: FilesystemStore::open(&root) + .await + .expect("filesystem store"), + puts: AtomicUsize::new(0), + }); + let configured = ConfiguredStore::checking( + StoreKind::Filesystem, + root.to_str().expect("UTF-8 path").to_owned(), + store.clone(), + None, + ); + let outcome = check_configured_store(&configured, &*storage, "sqlite").await; + assert!(outcome.is_ok(), "the store's first check: {outcome:?}"); + configured.record_check(outcome); + let stale_job_after = Duration::from_secs(300); + let runtime = Arc::new( + BackupRuntime::new(4) + .with_stale_job_after(stale_job_after) + .with_store(configured) + .with_source("sqlite", "test") + .with_shutdown(shutdown.child_token()), + ); + let ctx = OperationContext { + storage: storage.clone(), + backup: runtime.clone(), + limits: Arc::new(LimitsConfig::default()), + region: Arc::from(REGION), + account_id: Arc::from(account_id.as_str()), + import_paths: Arc::from(Vec::new()), + export_paths: Arc::from(Vec::new()), + authorized_key_info: Default::default(), + auth_cache: extenddb_auth::AuthCacheRegistry::empty(), + table_key_info_lookup: None, + }; + + call( + &ctx, + "CreateTable", + json!({ + "TableName": "Music", + "KeySchema": [{"AttributeName": "pk", "KeyType": "HASH"}], + "AttributeDefinitions": [{"AttributeName": "pk", "AttributeType": "S"}], + "BillingMode": "PAY_PER_REQUEST" + }), + ) + .await; + let deadline = std::time::Instant::now() + Duration::from_secs(30); + loop { + let desc = call(&ctx, "DescribeTable", json!({"TableName": "Music"})).await; + if desc["Table"]["TableStatus"] == "ACTIVE" { + break; + } + assert!( + std::time::Instant::now() < deadline, + "Music did not become ACTIVE: {desc}" + ); + tokio::time::sleep(Duration::from_millis(50)).await; + } + call( + &ctx, + "PutItem", + json!({"TableName": "Music", "Item": {"pk": {"S": "a"}, "n": {"N": "1"}}}), + ) + .await; + + // The request answers CREATING; the job's task then panics at its first + // put and ends. + let details = call( + &ctx, + "CreateBackup", + json!({"TableName": "Music", "BackupName": "nightly"}), + ) + .await; + assert_eq!( + details["BackupDetails"]["BackupStatus"], "CREATING", + "{details}" + ); + let arn_text = details["BackupDetails"]["BackupArn"] + .as_str() + .expect("an ARN") + .to_owned(); + let arn = BackupArn::parse(&arn_text).expect("a backup ARN"); + let deadline = std::time::Instant::now() + Duration::from_secs(30); + while runtime.running_jobs() > 0 { + assert!( + std::time::Instant::now() < deadline, + "the job task did not end" + ); + tokio::time::sleep(Duration::from_millis(20)).await; + } + assert_eq!( + store.puts.load(Ordering::SeqCst), + 1, + "the task died at its first put" + ); + let row = storage + .read_backup_row(&arn) + .await + .expect("read the row") + .expect("the row exists"); + assert_eq!( + row.status, + BackupStatus::Creating, + "the panicking task wrote nothing" + ); + let heartbeat_before = row.heartbeat_at; + assert_eq!( + runtime.free_slots(), + 4, + "the task's drop returned the job permit" + ); + + // One reconciler cycle in the same process: the slot no task holds makes + // the row dead at once, and without a manifest it is FAILED. + let mut state = ReconcilerState::with_shutdown(shutdown.child_token()); + let report = reconciler::cycle(&*storage, &runtime, stale_job_after, "sqlite", &mut state) + .await + .expect("a reconciler cycle"); + assert_eq!(report.store_healthy, Some(true), "{report:?}"); + assert_eq!( + report.creating_resolved, + vec![(arn_text.clone(), BackupStatus::Failed)], + "{report:?}" + ); + let row = storage + .read_backup_row(&arn) + .await + .expect("read the row") + .expect("the row exists"); + assert_eq!(row.status, BackupStatus::Failed); + assert_eq!(row.failure_reason.as_deref(), Some(OWNER_DEAD_NO_MANIFEST)); + assert!( + row.completed_at.is_some(), + "the FAILED write stamps completed_at" + ); + assert_eq!( + row.heartbeat_at, heartbeat_before, + "the takeover left the heartbeat" + ); + // The dead owner's heartbeat is within the window, so the prefix (empty + // here: the put never landed) is the FAILED-prefix duty's once it is + // stale, not this cycle's. + assert!(report.failed_prefixes_deleted.is_empty(), "{report:?}"); + let described = dispatch( + "DescribeBackup", + json!({"BackupArn": arn_text}), + &ctx, + "127.0.0.1:0", + ) + .await; + match described { + Err(extenddb_core::error::DynamoDbError::BackupNotFoundException(_)) => {} + Err(other) => panic!("DescribeBackup of the FAILED row: {other:?}"), + Ok(answer) => panic!("DescribeBackup of the FAILED row answered {}", answer.body), + } + // A second cycle has nothing left to resolve. + let report = reconciler::cycle(&*storage, &runtime, stale_job_after, "sqlite", &mut state) + .await + .expect("a second cycle"); + assert!(report.creating_resolved.is_empty(), "{report:?}"); + + shutdown.cancel(); + for worker in workers { + let _ = tokio::time::timeout(Duration::from_secs(5), worker).await; + } +} diff --git a/crates/storage/src/backup_catalog.rs b/crates/storage/src/backup_catalog.rs index 8079298f5..01f7222d3 100644 --- a/crates/storage/src/backup_catalog.rs +++ b/crates/storage/src/backup_catalog.rs @@ -537,9 +537,24 @@ pub struct StatusFields { pub size_bytes: Option, pub item_count: Option, pub failure_reason: Option, + /// The manifest's `source_table` object, which the AVAILABLE write of a + /// backup row carries so DescribeBackup reports the counts the manifest + /// has; a restore job record has no such column and ignores it. + pub source_table_details_json: Option, pub completed: bool, } +/// The column an age in [`BackupCatalog::list_backup_rows_older_than`] is +/// measured on, against the database's own `now()`. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum StaleClock { + /// `heartbeat_at`: the owner's last sign of life. + Heartbeat, + /// `completed_at`: when the row reached its terminal status. A row with + /// `completed_at` unset is never older than any age on this clock. + Completed, +} + /// What a conditional write did. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum WriteOutcome { @@ -760,6 +775,52 @@ pub trait BackupCatalog: Send + Sync { stale_after: Duration, ) -> BoxFuture<'a, Result>; + /// Take over a CREATING row whose owner is dead, under a fresh + /// generation, and return the claim; the reconciler's way onto a row a + /// job left behind. Dead is `heartbeat_at` older than `stale_after` on + /// the database's clock (on PostgreSQL, once the owner's claim session + /// has ended as well: the advisory lock is tried first, so a stopped + /// owner whose session lives keeps its row), except on SQLite, where a + /// CREATING row whose in-process slot no task holds is dead at once (a + /// restart frees every slot, a job that panics drops its slot) and + /// `stale_after` is not read. + /// `heartbeat_at` is left as the dead owner wrote it, so the stale + /// window of the FAILED-prefix duty counts from the owner's last beat + /// and not from the takeover. `None` when the row is not CREATING, is + /// gone, has a live owner, or is already claimed by another task; the + /// old generation's writes through its [`JobClaim`] find + /// [`WriteOutcome::NotMatched`] from then on. + fn claim_dead_backup<'a>( + &'a self, + key: &'a BackupKey, + stale_after: Duration, + ) -> BoxFuture<'a, Result>, StorageError>>; + + /// The rows of `store` with `status` whose `clock` column is older than + /// `age` on the database's clock: the reconciler's listing for the + /// FAILED-prefix delay (`Failed`, [`StaleClock::Heartbeat`]) and the + /// tombstone retention (`Failed` or `Deleted`, [`StaleClock::Completed`]). + /// The comparison is the backend's, in its own query, so the engine never + /// compares a row's timestamp with a clock of its own. + fn list_backup_rows_older_than<'a>( + &'a self, + status: BackupStatus, + store: &'a StoreIdentity, + clock: StaleClock, + age: Duration, + ) -> BoxStream<'a, Result>; + + /// Remove the restore registrations a backend keeps beside its rows that + /// name a target no longer a CREATING restore target of that backup, and + /// return the ARNs of the backups touched. Only Cassandra keeps such a + /// map (`active_restores`, maintained by lightweight transactions whose + /// removal step can be lost with its owner); every other backend derives + /// the in-use check from the target rows themselves and has nothing to + /// sweep, which is what the default body says. + fn sweep_restore_registrations(&self) -> BoxFuture<'_, Result, StorageError>> { + Box::pin(async { Ok(Vec::new()) }) + } + /// The name of a CREATING restore target registered from the backup, if /// any: the rule under which DeleteBackup and a second restore of the /// backup are refused with `BackupInUseException`. Reads the target's diff --git a/crates/storage/src/snapshot.rs b/crates/storage/src/snapshot.rs index 10f14d790..0f692e5f9 100644 --- a/crates/storage/src/snapshot.rs +++ b/crates/storage/src/snapshot.rs @@ -65,8 +65,12 @@ //! of each backend's `snapshot_engine.rs` are the record of what the //! snapshots do. -use extenddb_core::backup_format::KeyOrder; -use extenddb_core::types::{IndexStatus, Item, TableDescription}; +use extenddb_core::backup_format::{KeyOrder, SourceTableSchema}; +use extenddb_core::types::{ + BillingMode, GsiInput, IndexStatus, Item, LsiInput, ProvisionedThroughput, + ProvisionedThroughputDescription, TableDescription, TimeToLiveDescription, + VectorIndexSpecification, +}; use futures::future::BoxFuture; use futures::stream::BoxStream; @@ -89,26 +93,37 @@ pub use extenddb_core::backup_format::SnapshotDescriptor as SnapshotInfo; /// so the check can set it and observe its effect without a per-backend /// setter that a test could forget to call. pub mod paging { - /// Rows or documents a backend fetches per round trip. Measured on + /// Rows or documents a backend fetches per round trip, at most. Measured on /// localhost `PostgreSQL` 16 with 318-byte items: 1000 and 10000 read a /// million rows in the same time; 1000 holds less memory. Over a 1 ms /// network link the larger fetch would save about a second per million /// rows, which is why the size is a constant and not a configuration knob. - /// - /// The bound is rows, not bytes. `PostgreSQL`'s `FETCH` and `SQLite`'s - /// `LIMIT` each hold up to this many rows of item text in memory at - /// once, and at the default 400 KB item limit one row's text can reach - /// 27.7 MB (the measured worst case in #394's line-cap table), so one - /// fetch is bounded by rows, not bytes. A byte budget (keyset paging with a byte cutoff) is - /// required before the stream gets a caller; the backup design assigns - /// it to the backup job PR. `MongoDB`'s batches are bounded by the - /// server's 16 MiB reply limit as well as by this count. pub const DEFAULT_FETCH_BATCH: usize = 1000; + /// Bytes of item text a page holds, at most, before the page is cut: a + /// page ends at [`DEFAULT_FETCH_BATCH`] rows or after the row that takes + /// its running total of item text to or past this budget, whichever + /// comes first, so a page holds at most the budget plus one row and at + /// least one row. At the default 400 KB item limit one row's text can + /// reach 27.7 MB (the measured worst case in #394's line-cap table), so a + /// row bound alone would let a page of such rows hold gigabytes; the + /// budget is the term of the backup job's memory formula the stream + /// contributes. `PostgreSQL` applies the cut in its page query (a running + /// sum over the key order, so the server never returns the rows past the + /// cut), `SQLite` while it pulls the page's rows, and `MongoDB` is bounded + /// by the server's 16 MiB reply instead. + pub const SNAPSHOT_FETCH_BYTES: usize = 64 * 1024 * 1024; + #[cfg(feature = "test-hooks")] static OVERRIDE: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0); #[cfg(feature = "test-hooks")] + static BYTES_OVERRIDE: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0); + #[cfg(feature = "test-hooks")] static FETCHES: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0); + #[cfg(feature = "test-hooks")] + static MAX_PAGE_BYTES: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0); + #[cfg(feature = "test-hooks")] + static ROWS_PAST_CUT: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0); /// The page size a stream is built with. Backends read it once when they /// open a stream, so a change to the knob during a stream cannot end it @@ -128,12 +143,29 @@ pub mod paging { } } + /// The byte budget a stream is built with ([`SNAPSHOT_FETCH_BYTES`], or + /// the `test-hooks` override). Read once per open, like [`fetch_batch`]. + #[must_use] + pub fn fetch_bytes() -> usize { + #[cfg(feature = "test-hooks")] + { + match BYTES_OVERRIDE.load(std::sync::atomic::Ordering::Relaxed) { + 0 => SNAPSHOT_FETCH_BYTES, + n => n, + } + } + #[cfg(not(feature = "test-hooks"))] + { + SNAPSHOT_FETCH_BYTES + } + } + /// Record one round trip to the backend for a page of items. A no-op /// without `test-hooks`. /// /// The counter is trusted: the conformance check reads it to confirm /// the backend paged at the knob's size, so a backend must call this - /// exactly once per round trip (one `FETCH`, one page query, one + /// exactly once per round trip (one page query, one /// `find` or `getMore`), never per item. A backend that ignored the /// knob and called it per item would pass the paging assertions with /// the whole table read in one trip. `MongoDB`'s driver hides `getMore` @@ -145,6 +177,27 @@ pub mod paging { FETCHES.fetch_add(1, std::sync::atomic::Ordering::Relaxed); } + /// Record the bytes of item text one page returned, so the conformance + /// check can read the largest page (`max_page_bytes`). Called once per + /// page by the backends that apply the byte cut; a no-op without + /// `test-hooks`. + pub fn record_page_bytes(bytes: usize) { + #[cfg(feature = "test-hooks")] + MAX_PAGE_BYTES.fetch_max(bytes, std::sync::atomic::Ordering::Relaxed); + #[cfg(not(feature = "test-hooks"))] + let _ = bytes; + } + + /// Record rows the database read for a page and dropped at the byte cut + /// (`PostgreSQL`'s outer `WHERE`; at most the page's row bound less one). + /// A no-op without `test-hooks`. + pub fn record_rows_past_cut(n: u64) { + #[cfg(feature = "test-hooks")] + ROWS_PAST_CUT.fetch_add(n, std::sync::atomic::Ordering::Relaxed); + #[cfg(not(feature = "test-hooks"))] + let _ = n; + } + /// Set the page size for every stream opened afterwards; `0` restores the /// default. #[cfg(feature = "test-hooks")] @@ -152,6 +205,21 @@ pub mod paging { OVERRIDE.store(rows, std::sync::atomic::Ordering::Relaxed); } + /// Set the byte budget for every stream opened afterwards; `0` restores + /// [`SNAPSHOT_FETCH_BYTES`]. + #[cfg(feature = "test-hooks")] + pub fn set_fetch_bytes(bytes: usize) { + BYTES_OVERRIDE.store(bytes, std::sync::atomic::Ordering::Relaxed); + } + + /// Zero [`max_page_bytes`] and [`rows_past_cut`], so a check reads only + /// the pages it drives afterwards. + #[cfg(feature = "test-hooks")] + pub fn reset_page_counters() { + MAX_PAGE_BYTES.store(0, std::sync::atomic::Ordering::Relaxed); + ROWS_PAST_CUT.store(0, std::sync::atomic::Ordering::Relaxed); + } + /// Round trips recorded by every stream in this process so far. #[cfg(feature = "test-hooks")] #[must_use] @@ -159,6 +227,22 @@ pub mod paging { FETCHES.load(std::sync::atomic::Ordering::Relaxed) } + /// The largest page (bytes of item text) recorded since the last + /// [`reset_page_counters`]. + #[cfg(feature = "test-hooks")] + #[must_use] + pub fn max_page_bytes() -> usize { + MAX_PAGE_BYTES.load(std::sync::atomic::Ordering::Relaxed) + } + + /// Rows read past a page's cut, summed since the last + /// [`reset_page_counters`]. + #[cfg(feature = "test-hooks")] + #[must_use] + pub fn rows_past_cut() -> u64 { + ROWS_PAST_CUT.load(std::sync::atomic::Ordering::Relaxed) + } + /// The two points in `open_table_snapshot` at which every backend calls /// [`open_pause`]. Under `test-hooks` the conformance check holds an open /// at one of them while it commits writes and an `UpdateTable`, to check @@ -177,9 +261,8 @@ pub mod paging { /// layout the pause is before the `LOCK TABLE`, in the window where a /// `DeleteTable` can still drop the data table. AfterDefinition, - /// The data snapshot is pinned; the item read (`PostgreSQL`'s `DECLARE - /// ... CURSOR`, `MongoDB`'s `find`, `SQLite`'s first page) has not - /// started. + /// The data snapshot is pinned; the item read (`PostgreSQL`'s and + /// `SQLite`'s first page, `MongoDB`'s `find`) has not started. BeforeItems, } @@ -252,6 +335,116 @@ pub fn key_schema_is_orderable(description: &TableDescription) -> bool { KeyOrder::resolve(&description.key_schema, &description.attribute_definitions).is_ok() } +/// The manifest's `source_table` object for a table, built from the +/// description a snapshot returned and the table's TTL description, which is +/// not part of a [`TableSnapshot`] and is read beside the open +/// (`MetadataEngine::describe_ttl`). One derivation for every backend, so the +/// row a CreateBackup inserts and the manifest its job writes carry the same +/// object; the legacy format's `BackupTableDefinition` is a different shape +/// and stays its own. +/// +/// `billing_mode` is the summary's, PROVISIONED when the description has no +/// summary; `provisioned_throughput` and each GSI's are written only for a +/// PROVISIONED table (an on-demand table's description echoes zeros, which +/// no CreateTable accepts). `vector_indexes` is `None` when the description +/// carries none (a backend without vector support) and the list of +/// specifications otherwise, empty included, so a reader can tell the two +/// apart. `item_count` and `table_size_bytes` are the description's, which +/// are the stats pass's figures: the job replaces `item_count` with the +/// stream's count before it writes the manifest. +/// +/// # Errors +/// +/// [`StorageError::Internal`] when the description's `TableClassSummary` has +/// no `TableClass` string: no in-tree backend writes such a summary, and +/// dropping the class silently would make a backup that restores as a +/// different table than it describes. +pub fn source_table_schema( + description: &TableDescription, + ttl: Option, +) -> Result { + let billing_mode = description + .billing_mode_summary + .as_ref() + .map_or(BillingMode::Provisioned, |s| s.billing_mode); + let provisioned = billing_mode == BillingMode::Provisioned; + let throughput = |d: &ProvisionedThroughputDescription| { + provisioned.then_some(ProvisionedThroughput { + read_capacity_units: d.read_capacity_units, + write_capacity_units: d.write_capacity_units, + }) + }; + let table_class = match &description.table_class_summary { + None => None, + Some(summary) => Some( + summary + .get("TableClass") + .and_then(serde_json::Value::as_str) + .ok_or_else(|| { + StorageError::Internal(format!( + "table {} has a TableClassSummary without a TableClass: {summary}", + description.table_name + )) + })? + .to_owned(), + ), + }; + let global_secondary_indexes = description + .global_secondary_indexes + .iter() + .flatten() + .map(|g| GsiInput { + index_name: g.index_name.clone(), + key_schema: g.key_schema.clone(), + projection: g.projection.clone(), + provisioned_throughput: g.provisioned_throughput.as_ref().and_then(throughput), + }) + .collect(); + let local_secondary_indexes = description + .local_secondary_indexes + .iter() + .flatten() + .map(|l| LsiInput { + index_name: l.index_name.clone(), + key_schema: l.key_schema.clone(), + projection: l.projection.clone(), + }) + .collect(); + let vector_indexes = description.vector_indexes.as_ref().map(|indexes| { + indexes + .iter() + .map(|v| VectorIndexSpecification { + index_name: v.index_name.clone(), + dimensions: v.dimensions, + distance_function: v.distance_function, + vector_attribute: v.vector_attribute.clone(), + search_schema: v.search_schema.clone(), + projection: v.projection.clone(), + }) + .collect() + }); + Ok(SourceTableSchema { + table_name: description.table_name.clone(), + table_id: description.table_id.clone(), + table_arn: description.table_arn.clone(), + table_creation_date_time: description.creation_date_time, + key_schema: description.key_schema.clone(), + attribute_definitions: description.attribute_definitions.clone(), + billing_mode, + provisioned_throughput: throughput(&description.provisioned_throughput), + on_demand_throughput: description.on_demand_throughput.clone(), + table_class, + sse_description: description.sse_description.clone(), + global_secondary_indexes, + local_secondary_indexes, + vector_indexes, + stream_specification: description.stream_specification.clone(), + time_to_live: ttl, + item_count: description.item_count, + table_size_bytes: description.table_size_bytes, + }) +} + /// A consistent view of one table: its definition, how the snapshot was /// taken, and every item as of the same instant. pub struct TableSnapshot { @@ -396,8 +589,199 @@ pub trait SnapshotEngine: Send + Sync { #[cfg(test)] mod tests { use super::*; + use extenddb_core::backup_format::ExtenddbManifest; use extenddb_core::types::GsiDescription; + /// A description as the storage layer reports it: PROVISIONED, one GSI + /// (with throughput), one LSI, one vector index, a stream, a table class + /// and SSE; the table of the manifest fixture, so the schema can be + /// spliced into it. + fn described() -> TableDescription { + serde_json::from_value(serde_json::json!({ + "TableName": "Music", + "TableId": "12345a12-abcd-123a-ab12-1234abc12345", + "TableArn": "arn:aws:dynamodb:us-east-1:123456789012:table/Music", + "TableStatus": "ACTIVE", + "CreationDateTime": 1757400000.0, + "TableSizeBytes": 123456, + "ItemCount": 100000, + "KeySchema": [ + {"AttributeName": "Artist", "KeyType": "HASH"}, + {"AttributeName": "SongTitle", "KeyType": "RANGE"} + ], + "AttributeDefinitions": [ + {"AttributeName": "Artist", "AttributeType": "S"}, + {"AttributeName": "SongTitle", "AttributeType": "S"}, + {"AttributeName": "Genre", "AttributeType": "S"} + ], + "ProvisionedThroughput": { + "ReadCapacityUnits": 5, "WriteCapacityUnits": 5, "NumberOfDecreasesToday": 0 + }, + "BillingModeSummary": {"BillingMode": "PROVISIONED"}, + "GlobalSecondaryIndexes": [{ + "IndexName": "GenreIndex", + "KeySchema": [{"AttributeName": "Genre", "KeyType": "HASH"}], + "Projection": {"ProjectionType": "INCLUDE", "NonKeyAttributes": ["Year"]}, + "IndexStatus": "ACTIVE", + "ProvisionedThroughput": { + "ReadCapacityUnits": 1, "WriteCapacityUnits": 1, "NumberOfDecreasesToday": 0 + }, + "IndexSizeBytes": 10, "ItemCount": 3, + "IndexArn": "arn:aws:dynamodb:us-east-1:123456789012:table/Music/index/GenreIndex" + }], + "LocalSecondaryIndexes": [{ + "IndexName": "TitleIndex", + "KeySchema": [ + {"AttributeName": "Artist", "KeyType": "HASH"}, + {"AttributeName": "SongTitle", "KeyType": "RANGE"} + ], + "Projection": {"ProjectionType": "KEYS_ONLY"}, + "IndexSizeBytes": 10, "ItemCount": 3, + "IndexArn": "arn:aws:dynamodb:us-east-1:123456789012:table/Music/index/TitleIndex" + }], + "VectorIndexes": [{ + "IndexName": "embeddings", + "VectorAttribute": {"AttributeName": "vec"}, + "Dimensions": 1024, + "DistanceFunction": "COSINE", + "IndexStatus": "ACTIVE", + "IndexSizeBytes": 0, "ItemCount": 0, + "IndexArn": "arn:aws:dynamodb:us-east-1:123456789012:table/Music/index/embeddings" + }], + "StreamSpecification": {"StreamEnabled": true, "StreamViewType": "NEW_AND_OLD_IMAGES"}, + "DeletionProtectionEnabled": false, + "SSEDescription": { + "Status": "ENABLED", "SSEType": "KMS", + "KMSMasterKeyArn": "arn:aws:kms:us-east-1:123456789012:key/abcd" + }, + "TableClassSummary": {"TableClass": "STANDARD"}, + "OnDemandThroughput": {"MaxReadRequestUnits": 100, "MaxWriteRequestUnits": 50} + })) + .expect("a table description") + } + + fn ttl() -> TimeToLiveDescription { + serde_json::from_value(serde_json::json!({ + "TimeToLiveStatus": "ENABLED", + "AttributeName": "expires_at" + })) + .expect("a TTL description") + } + + /// The fixture manifest with `source_table` replaced by `schema`, parsed + /// back through the reader's own checks. + fn manifest_with(schema: &SourceTableSchema) -> Result { + let mut manifest: serde_json::Value = serde_json::from_str(include_str!( + "../../core/testdata/extenddb-manifest/v1.json" + )) + .expect("fixture"); + manifest["source_table"] = serde_json::to_value(schema).expect("serialize"); + let bytes = serde_json::to_vec(&manifest).expect("bytes"); + ExtenddbManifest::from_json_bytes(&bytes).map_err(|e| e.to_string()) + } + + /// The schema built from a full description is the fixture's own + /// `source_table`, member for member, and reads back through the + /// manifest's checks. + #[test] + fn a_described_table_becomes_the_fixtures_source_table() { + let schema = source_table_schema(&described(), Some(ttl())).expect("schema"); + let fixture: serde_json::Value = serde_json::from_str(include_str!( + "../../core/testdata/extenddb-manifest/v1.json" + )) + .expect("fixture"); + assert_eq!( + serde_json::to_value(&schema).expect("serialize"), + fixture["source_table"] + ); + let manifest = manifest_with(&schema).expect("the manifest reads back"); + assert_eq!(manifest.source_table, schema); + assert_eq!( + manifest.source_table.vector_indexes.as_ref().map(Vec::len), + Some(1), + "the vector index specification is carried" + ); + } + + /// The members the reader requires even when absent (`vector_indexes`, + /// `stream_specification`, `time_to_live`) are written as explicit + /// nulls, so the document reads back; an on-demand table writes no + /// throughput for itself or its GSIs. + #[test] + fn absent_members_are_written_as_nulls_and_on_demand_drops_throughput() { + let mut description = described(); + description.vector_indexes = None; + description.stream_specification = None; + description.table_class_summary = None; + description.billing_mode_summary = Some(extenddb_core::types::BillingModeSummary { + billing_mode: BillingMode::PayPerRequest, + last_update_to_pay_per_request_date_time: None, + }); + let schema = source_table_schema(&description, None).expect("schema"); + let json = serde_json::to_value(&schema).expect("serialize"); + for member in [ + "vector_indexes", + "stream_specification", + "time_to_live", + "table_class", + ] { + assert_eq!(json[member], serde_json::Value::Null, "{member}"); + assert!( + json.as_object().unwrap().contains_key(member), + "{member} must be present as null, not missing" + ); + } + assert_eq!(schema.billing_mode, BillingMode::PayPerRequest); + assert_eq!(schema.provisioned_throughput, None); + assert_eq!( + schema.global_secondary_indexes[0].provisioned_throughput, + None + ); + assert_eq!( + json["global_secondary_indexes"].as_array().map(Vec::len), + Some(1) + ); + assert_eq!( + json["local_secondary_indexes"].as_array().map(Vec::len), + Some(1) + ); + manifest_with(&schema).expect("the manifest reads back"); + } + + /// A description without a billing-mode summary is PROVISIONED with the + /// description's throughput; a table without vector support writes + /// `vector_indexes: null`, one with support and no indexes `[]`. + #[test] + fn billing_defaults_to_provisioned_and_vector_support_is_told_from_no_indexes() { + let mut description = described(); + description.billing_mode_summary = None; + description.vector_indexes = Some(vec![]); + let schema = source_table_schema(&description, None).expect("schema"); + assert_eq!(schema.billing_mode, BillingMode::Provisioned); + assert_eq!( + schema.provisioned_throughput, + Some(ProvisionedThroughput { + read_capacity_units: 5, + write_capacity_units: 5 + }) + ); + assert_eq!(schema.vector_indexes, Some(vec![])); + description.vector_indexes = None; + let schema = source_table_schema(&description, None).expect("schema"); + assert_eq!(schema.vector_indexes, None); + } + + #[test] + fn a_table_class_summary_without_a_class_is_an_error() { + let mut description = described(); + description.table_class_summary = Some(serde_json::json!({"Other": 1})); + let err = source_table_schema(&description, None).expect_err("refused"); + assert!( + matches!(&err, StorageError::Internal(m) if m.contains("TableClassSummary")), + "{err}" + ); + } + #[test] fn deleting_indexes_are_dropped_and_others_kept() { let gsi = |name: &str, status: &str| -> GsiDescription { diff --git a/crates/storage/src/snapshot_conformance.rs b/crates/storage/src/snapshot_conformance.rs index 04b58c9b2..7429836e7 100644 --- a/crates/storage/src/snapshot_conformance.rs +++ b/crates/storage/src/snapshot_conformance.rs @@ -375,11 +375,39 @@ where /// Drain `stream` into `into`, each item within [`DRAIN_TIMEOUT`] (see /// [`next_within`]); `stage` names the drain in the panic. async fn drain_within(stream: &mut S, stage: &str, into: &mut Vec) +where + S: futures::Stream> + Unpin, +{ + drain_at_most(stream, stage, into, usize::MAX).await; +} + +/// [`drain_within`] with a bound on the count: a stream that has yielded +/// more than `at_most` items is repeating rows (a seek that is inclusive +/// instead of exclusive yields the row it was seeked past on every page, and +/// a cursor that never advances yields the first page forever), and the +/// drain panics at once naming `stage` and the first key yielded twice, so +/// the check fails in seconds instead of running until another test's wait +/// on the hooks lock gives out. +async fn drain_at_most(stream: &mut S, stage: &str, into: &mut Vec, at_most: usize) where S: futures::Stream> + Unpin, { while let Some(next) = next_within(stream, stage, into.len()).await { into.push(next.unwrap_or_else(|e| panic!("{stage}: snapshot stream item: {e}"))); + if into.len() > at_most { + let mut seen = std::collections::HashSet::new(); + let repeated = into + .iter() + .map(key_id) + .find(|id| !seen.insert(id.clone())) + .unwrap_or_else(|| "(no key yielded twice)".to_owned()); + panic!( + "{stage}: the stream yielded {} items of a table of {at_most}; the first key \ + yielded twice is {repeated} (the stream repeats rows: an inclusive seek or a \ + cursor that does not advance)", + into.len() + ); + } } } @@ -566,6 +594,26 @@ impl Drop for PageSize { } } +/// Sets the shared snapshot byte budget ([`paging::set_fetch_bytes`]) and +/// restores the default when dropped, as [`PageSize`] does for the row +/// bound; hold [`hooks_lock`] while one is alive. +pub struct ByteBudget(()); + +impl ByteBudget { + /// Set the byte budget to `bytes` until the returned value is dropped. + #[must_use] + pub fn set(bytes: usize) -> Self { + paging::set_fetch_bytes(bytes); + Self(()) + } +} + +impl Drop for ByteBudget { + fn drop(&mut self) { + paging::set_fetch_bytes(0); + } +} + /// What the check expects of the backend under test. #[derive(Debug, Clone, Copy)] pub struct Expectations { @@ -1465,6 +1513,206 @@ async fn key_shapes( } } +/// Items [`run_page_bounds`] fills its table with: more than one row bound's +/// worth at the budget it sets, and ten past a multiple of the rows a page +/// keeps, so the last page is one the cut shortens (a backend that ends the +/// stream on a short page without asking whether the cut shortened it loses +/// those rows). +pub const PAGE_BOUNDS_ITEMS: usize = 610; + +/// Rows of item text one page of [`run_page_bounds`]'s second pass holds at +/// the budget it sets: the budget is this many rows' text. +pub const PAGE_BOUNDS_ROWS: usize = 8; + +/// What [`run_page_bounds`] measured, for a backend's own assertions beside +/// it. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct PageBounds { + /// Bytes of item text of one row of the table (every row is the same + /// size). + pub row_bytes: usize, + /// The byte budget the second pass ran under: [`PAGE_BOUNDS_ROWS`] rows. + pub budget: usize, + /// Round trips the second pass made. + pub pages: u64, + /// Rows the backend recorded as read past a cut during the second pass + /// ([`paging::rows_past_cut`]; a backend that cuts in its query records + /// them, one that cuts while pulling rows records none). + pub rows_past_cut: u64, +} + +fn page_bounds_item(i: usize) -> Item { + // Fixed-width keys and a fixed padding, so every row's text is the same + // length and the budget can be set to a whole number of rows. + [ + ("pk".to_owned(), s(format!("p{i:05}"))), + ("sk".to_owned(), n(100_000 + i)), + ("v".to_owned(), s("x".repeat(1000))), + ] + .into_iter() + .collect() +} + +/// Check that the stream pages by bytes: no page holds more than the budget +/// plus one row, and the cut does not lose rows. `hooks` is the caller's +/// [`hooks_lock`] guard; `table` must not exist; `pump` runs while the table +/// becomes `ACTIVE`. +/// +/// The table is [`PAGE_BOUNDS_ITEMS`] rows of one size. A first pass at a +/// budget of one byte reads one row per page, which measures the row's text +/// size as the backend counts it (`row_bytes`) and shows the budget took +/// effect (as many round trips as rows). The second pass sets the budget to +/// [`PAGE_BOUNDS_ROWS`] rows and the row bound to its default, and asserts: +/// every item once (completeness across cut pages, the last one included; +/// both passes stop and fail as soon as the stream has yielded more items +/// than the table holds, naming the key yielded twice, so a stream that +/// repeats rows fails in seconds rather than running until the hooks lock's +/// wait gives out); +/// more round trips than the row bound alone would make (the budget, not the +/// row bound, paged the table: a counter beside the knob, since the stream's +/// contents alone cannot show the knob took effect); the largest page at least the +/// budget (a page was cut) and under the budget plus one row (the rule: a +/// page ends after the row that takes it to or past the budget, so a page +/// cut one row late shows as exactly the budget plus one row); and, when the +/// stream is `key_ordered`, ascending order. +/// +/// # Panics +/// +/// On any violation, with the figures. +pub async fn run_page_bounds( + hooks: &HooksGuard, + engine: &E, + account: &str, + table: &str, + mut pump: F, +) -> PageBounds +where + E: SnapshotEngine + TableEngine + DataEngine + Send + Sync, + F: FnMut() -> Fut, + Fut: std::future::Future, +{ + let _ = hooks; + engine + .create_table(account, table_input(table)) + .await + .expect("create the page-bounds table"); + wait_active(engine, account, table, &mut pump).await; + let ki = key_info(engine, account, table).await; + for i in 0..PAGE_BOUNDS_ITEMS { + put(engine, &ki, page_bounds_item(i)).await; + } + let expected: BTreeSet = (0..PAGE_BOUNDS_ITEMS) + .map(|i| key_id(&page_bounds_item(i))) + .collect(); + + // First pass: one row per page. + let row_bytes = { + let _budget = ByteBudget::set(1); + let _rows = PageSize::set(0); + paging::reset_page_counters(); + let fetches_before = paging::fetches(); + let mut snapshot = engine + .open_table_snapshot(account, table) + .await + .expect("open at a one-byte budget"); + let mut seen = Vec::new(); + drain_at_most( + &mut snapshot.items, + "page bounds, one-byte budget", + &mut seen, + PAGE_BOUNDS_ITEMS, + ) + .await; + drop(snapshot); + assert_eq!(seen.len(), PAGE_BOUNDS_ITEMS, "one-byte budget: items read"); + let fetches = paging::fetches() - fetches_before; + assert!( + fetches >= PAGE_BOUNDS_ITEMS as u64, + "one-byte budget: {fetches} round trips for {PAGE_BOUNDS_ITEMS} rows; the budget \ + did not cut the pages to one row" + ); + let row_bytes = paging::max_page_bytes(); + assert!( + row_bytes > 1000, + "one row of item text measured as {row_bytes} bytes" + ); + row_bytes + }; + + // Second pass: a budget of a whole number of rows, the row bound at its + // default. + let budget = row_bytes * PAGE_BOUNDS_ROWS; + let _budget = ByteBudget::set(budget); + let _rows = PageSize::set(0); + paging::reset_page_counters(); + let fetches_before = paging::fetches(); + let mut snapshot = engine + .open_table_snapshot(account, table) + .await + .expect("open at the row-multiple budget"); + let key_ordered = snapshot.key_ordered; + let order = extenddb_core::backup_format::KeyOrder::resolve( + &snapshot.description.key_schema, + &snapshot.description.attribute_definitions, + ) + .expect("the page-bounds table has an orderable key"); + let mut seen = Vec::new(); + drain_at_most( + &mut snapshot.items, + "page bounds, row-multiple budget", + &mut seen, + PAGE_BOUNDS_ITEMS, + ) + .await; + drop(snapshot); + let pages = paging::fetches() - fetches_before; + + let mut got = BTreeSet::new(); + for item in &seen { + let id = key_id(item); + assert!(got.insert(id.clone()), "page bounds: duplicate item {id}"); + } + assert_eq!( + got, expected, + "page bounds: the stream did not yield every row exactly once (a cut page lost or \ + repeated rows)" + ); + let row_bound_pages = PAGE_BOUNDS_ITEMS.div_ceil(paging::fetch_batch()) as u64; + assert!( + pages > row_bound_pages, + "page bounds: {pages} round trips, no more than the row bound alone would make \ + ({row_bound_pages}); the byte budget did not page the table" + ); + let largest = paging::max_page_bytes(); + assert!( + largest >= budget, + "page bounds: the largest page holds {largest} bytes, under the budget of {budget}; no \ + page was cut by bytes" + ); + assert!( + largest < budget + row_bytes, + "page bounds: the largest page holds {largest} bytes, the budget plus a row or more \ + ({budget} + {row_bytes}); a page is cut one row late" + ); + if key_ordered { + for pair in seen.windows(2) { + assert_eq!( + order.compare(&pair[0], &pair[1]).expect("comparable keys"), + std::cmp::Ordering::Less, + "page bounds: {} is not before {}", + key_id(&pair[0]), + key_id(&pair[1]) + ); + } + } + PageBounds { + row_bytes, + budget, + pages, + rows_past_cut: paging::rows_past_cut(), + } +} + /// Checks that a snapshot cannot be opened on a missing or non-`ACTIVE` /// table. `hooks` is the caller's [`hooks_lock`] guard. `table` must not /// exist; `set_updating` flips that table's catalog diff --git a/devtools/run-tests b/devtools/run-tests index 0c46e71be..0af98bf7d 100755 --- a/devtools/run-tests +++ b/devtools/run-tests @@ -38,6 +38,13 @@ # --parallel[=N] run pytest in parallel (default: 1/3 of CPU cores) # --config PATH config file path (default: extenddb.toml in repo root) # +# Environment (--pytest against extenddb): +# EXTENDDB_TEST_BACKUP_STORE where the server writes its backups: filesystem, +# s3 or none; the backup tests derive their +# expected outcome from it, so it is required +# (the Rust integration suite's backup tests read +# it too and fail when it is unset). +# # REQ-TEST-001, D-1, D-2, D-3, D-4, D-5 set -euo pipefail @@ -98,6 +105,15 @@ Options: --filter PATTERN filter for cargo test / pytest -k --parallel[=N] run pytest in parallel (default: 1/3 of CPU cores) --config PATH config file path + +Environment (--extenddb with an integration suite): + EXTENDDB_TEST_ENDPOINT the server, https://localhost: (required) + EXTENDDB_ADMIN_PASSWORD the password from 'extenddb init' (required) + EXTENDDB_TEST_BACKUP_STORE where the server writes its backups: filesystem, + s3 or none (required for --pytest; the Rust + integration suite's backup tests read it too) +Environment (--real-dynamodb): EXTENDDB_TEST_ENDPOINT and EXTENDDB_ADMIN_PASSWORD + unset; AWS credentials from the SDK's chain or AWS_ACCESS_KEY_ID. EOF exit 1 } @@ -186,6 +202,25 @@ validate_local_target() { echo " This is the password from '${prefix_lower} init'." exit 1 fi + + # EXTENDDB_TEST_BACKUP_STORE must say where the server writes its backups. + # The backup tests derive their expected outcome from it (the store-backed + # create backs up vector tables and is asynchronous; the in-catalog create + # refuses them), so a run cannot leave it to be guessed. + if $RUN_PYTEST; then + case "${EXTENDDB_TEST_BACKUP_STORE:-}" in + filesystem|s3|none) ;; + *) + echo "error: --${prefix_lower} --pytest requires EXTENDDB_TEST_BACKUP_STORE to be set." + echo " Set it to the server's [backup.store]: 'filesystem' or 's3' for a configured" + echo " store (a file-backed SQLite server without the section has the defaulted" + echo " filesystem root), 'none' for a server without one (kind = \"disabled\", or" + echo " PostgreSQL, MongoDB and Cassandra without the section)." + echo " Got: '${EXTENDDB_TEST_BACKUP_STORE:-}'" + exit 1 + ;; + esac + fi } validate_real_dynamodb() { @@ -243,22 +278,42 @@ fi echo "" # --- Health check (local targets only) --- +# With EXTENDDB_TEST_BACKUP_STORE at filesystem or s3 the check also waits for +# the backup store component to report healthy: /health answers 200 while the +# component is still "checking", and the backup suites assert the component +# is healthy at their start, so what is waited for here is what they assert. health_check() { local endpoint="$1" local timeout=30 local interval=1 local elapsed=0 + local want_store=false + case "${EXTENDDB_TEST_BACKUP_STORE:-}" in + filesystem|s3) want_store=true ;; + esac echo "=== Health check: ${endpoint}/health ===" while [[ $elapsed -lt $timeout ]]; do - if curl -skf -o /dev/null "${endpoint}/health" 2>/dev/null; then - echo " ✓ Server is healthy (${elapsed}s)" - return 0 + local body + if body=$(curl -skf "${endpoint}/health" 2>/dev/null) && [[ -n "$body" ]]; then + if ! $want_store; then + echo " ✓ Server is healthy (${elapsed}s)" + return 0 + fi + if printf '%s' "$body" | python3 -c 'import json, sys +h = json.load(sys.stdin) +sys.exit(0 if h.get("components", {}).get("backup_store", {}).get("status") == "healthy" else 1)' 2>/dev/null; then + echo " ✓ Server and its backup store are healthy (${elapsed}s)" + return 0 + fi fi sleep "$interval" elapsed=$((elapsed + interval)) done echo " ✗ Server health check failed after ${timeout}s" + if $want_store; then + echo " The backup store component did not report healthy; last: $(curl -sk "${endpoint}/health" 2>/dev/null)" + fi echo " Is the server running? Check: curl -sk ${endpoint}/health" return 1 } @@ -467,7 +522,10 @@ if $RUN_PYTEST; then fi echo " AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID:-}" # CLI lifecycle tests manage their own server; exclude from main suite - PYTEST_ARGS=(python3 -m pytest tests/ -v --ignore=tests/python --ignore=tests/test_cli_lifecycle.py --ignore=tests/test_cli_container_readiness.py --ignore=tests/test_cli_migrate_concurrency.py --ignore=tests/test_cli_vector_catalog_migration.py --ignore=tests/test_gsi_async_queue.py --ignore=tests/test_backup_store_config.py) + # -rP: the summary repeats the passed tests that printed something (the + # backup cells print which branch ran and which store the server reports), + # which pytest's capture would otherwise keep out of the log. + PYTEST_ARGS=(python3 -m pytest tests/ -v -rP --ignore=tests/python --ignore=tests/test_cli_lifecycle.py --ignore=tests/test_cli_container_readiness.py --ignore=tests/test_cli_migrate_concurrency.py --ignore=tests/test_cli_vector_catalog_migration.py --ignore=tests/test_gsi_async_queue.py --ignore=tests/test_backup_store_config.py --ignore=tests/test_backup_job.py) # Parallel execution (--parallel): distribute by file so module/class-scoped # fixtures stay within a single worker. if [[ -n "$PARALLEL" ]] && python3 -c "import xdist" 2>/dev/null; then diff --git a/docs/design/15-backup-restore-bench-pr8.md b/docs/design/15-backup-restore-bench-pr8.md new file mode 100644 index 000000000..2fb548fa4 --- /dev/null +++ b/docs/design/15-backup-restore-bench-pr8.md @@ -0,0 +1,479 @@ +# PR 8 memory and duration record (`bench-rss.sh`, 2026-10-31 and 2026-11-01) + +This is the record the design cites (section 7, revision item 29 of +`15-backup-restore.md`). The scripts (`bench-rss.sh`, `kill-points.py`) and +the raw samples (`/proc//status` every 200 ms per run, the servers' +logs) are kept beside the PR outside the repository, on the measuring host; +the figures below are what they produced. The runs of 2026-11-01, after the +PostgreSQL page's row bound changed, are in the section at the end and in the +sentences they correct; the runs of the same day after the row bound gained +its memory are in the last section. + +What was measured: the release binary (`cargo build --release`, the default +PostgreSQL features; `--no-default-features --features sqlite`; `--features +mongodb-test-hooks`) running `serve --foreground` with a filesystem backup +store (SQLite: the defaulted root beside the database file), sampled from +`/proc//status` every 200 ms (`VmRSS` and `VmHWM`). Each table is loaded +through the wire (`kill-points.py create-table`: BatchWriteItem from 16 +threads, items of a 200-byte `S` pad under a composite key with two GSIs), +then the server is restarted so the process's high-water mark covers the +backup alone; the idle figure is read right before `CreateBackup`, the peak +is `VmHWM` once `DescribeBackup` reports `AVAILABLE`. Samples: +`rss//{host.txt,results.txt,.json,.samples}` with `` the +backend for the record's runs and `sqlite-itemchannel1`, +`sqlite-itemchannel-swapped`, `sqlite-arena1`, `sqlite-worst24`, +`sqlite-rowbuffer4` for the comparison runs below; the series of 2026-11-01 +under `sqlite-worst-{default,arena1}/` and `postgres/`. + +Host (`host.txt`): 16 cores, 30.3 GiB, Linux 5.10.270 (amzn2int); PostgreSQL +16.15 in a container on loopback (55432); the SQLite the binary bundles +(libsqlite3-sys 0.30.1, SQLite 3.46; the host's 3.7.17 is unused); MongoDB +7.0.43 single-node replica set in a container (27117). What was not held +constant, stated before any figure here is read across runs: the host ran +other work during every run (the PostgreSQL runs +beside two 1M-item loads and a wire suite; the SQLite and MongoDB runs beside +the Cassandra integration suite and the kill-point runs), so the durations +are upper bounds and comparable only within one back-to-back pair; the peaks +are the process's own and do not depend on the host's load. Everything is on +loopback: nothing here says what a network database or a remote S3 store does +to the duration; the upload window of an S3 store (`upload_part_size_bytes * +max_concurrent_uploads`) is not in these figures, which use the filesystem +store's one write buffer. The MongoDB binary was built before the pipeline's +last change (the sink's `BrokenPipe` answer after a stop, which touches no +allocation); the PostgreSQL and SQLite figures of the record are from binaries +of the PR's first build (`ITEM_CHANNEL = 4`); the PostgreSQL page changed +after these runs (the row bound from the largest row, the rows parsed as they +arrive), so the PostgreSQL worst-shape row below is the first build's and the +section of 2026-11-01 carries what changed. + +## The bound the acceptance asserts: peak(1M) − peak(100k) < 112 MiB + +The bound the script asserts is 64 + 32 + 16 MiB (the page budget, the line +cap, the channels). MongoDB's "1M" run is 500k: its 100k load took 361 s +and the 500k load 1,604 s through the wire on this host, so a 1M load would +have run for close to an hour; the 500k run is recorded in its place and the +difference is against it. + +| backend | run | items | idle RSS | peak (VmHWM) | peak − idle | AVAILABLE after | backup size | +|---|---|---|---|---|---|---|---| +| PostgreSQL | 100k | 100,000 | 24.6 MiB | 35.9 MiB | 11.3 MiB | 1.85 s | 847,689 B | +| PostgreSQL | 1M | 1,000,000 | 24.8 MiB | 45.4 MiB | 20.6 MiB | 36.1 s | 6,452,803 B | +| PostgreSQL | 1M − 100k | | | **+9.5 MiB** | | | PASS (bound 112) | +| SQLite | 100k | 100,000 | 20.4 MiB | 65.4 MiB | 45.0 MiB | 1.44 s | 847,689 B | +| SQLite | 1M | 1,000,000 | 22.8 MiB | 73.3 MiB | 50.5 MiB | 12.45 s | 6,452,803 B | +| SQLite | 1M − 100k | | | **+7.9 MiB** | | | PASS (bound 112) | +| MongoDB | 100k | 100,000 | 26.8 MiB | 36.4 MiB | 9.6 MiB | 1.85 s | 962,185 B | +| MongoDB | 500k (in the 1M row's place) | 500,000 | 25.7 MiB | 41.2 MiB | 15.5 MiB | 8.81 s | 4,869,108 B | +| MongoDB | 500k − 100k | | | **+4.8 MiB** | | | PASS (bound 112) | + +The page budget is never reached by 200-byte rows (a page is `n` rows, at +most `DEFAULT_FETCH_BATCH` = 1000, about 250 KB of text), and the data files +roll at 64 MiB of uncompressed text without the file being held (the encoder +streams 1 MiB chunks into the put), so the peak over idle is a few tens of +MiB on every backend and moves by under 10 MiB from 100k to 1M rows. SQLite's +45 MiB over idle at 100k is the backend's own: the pool's connections each +hold SQLite's page cache, which a backup's reads fill; it is bounded by the +pool and does not grow with the table (the 1M run adds 7.9 MiB to it); the +constant is `PRAGMA cache_size = -32000` +(`crates/storage-sqlite/src/sqlite_util.rs`), 31 MiB per connection, so the +pool's bound is 10 × 31 MiB at the default pool and 34 × 31 MiB at the raised +one, and the 45 MiB is about one and a half connections' caches filled, not +the pool's. The duration scales +with the items: 36 s for 1M on PostgreSQL under a loaded host (the server's +log line says 36.1 s of job time; 1.9 s for 100k), 12.5 s on SQLite, 8.8 s +for 500k on MongoDB. + +Rows read past the cut on PostgreSQL (the `test-hooks` counter +`paging::rows_past_cut`, `crates/storage-postgres/tests/table_snapshot.rs::pages_are_cut_at_the_byte_budget`, +run with `--nocapture`): on the first build **602 rows over 77 pages of 8 +rows** with `n` = 16 for every page (the row bound from the average row, whose +floor was 16; 8 per page where the budget cut each page in half), on both +layouts; after the paging change **8 rows over 77 pages**, all on the first page (the +row bound is then the budget divided by the largest row, 8, so every later +page reads exactly the rows it keeps), asserted exactly. The SQLite stream +pulls no row past its cut (its `rows_past_cut` is 0 and asserted so by +`snapshot_engine.rs`'s test); the driver's worker reads ahead of it, which +the worst-expansion runs below measure. + +## The worst-expansion run + +`put-worst`: 1,000 small rows (`a0000`.., a 50-byte pad) and then 12 items +of #394's worst-expansion shape (`zz000`..: a list of 2,262 `NS` sets of the +90 values `1.0E-130`..`9.9E-130`; 409,429 bytes as the service sizes an +item; 27.7 MB as a data-file line and as stored item text, every number +normalized to its 133-character decimal form; about 34 MB decoded, 203,580 +`String`s of 133 bytes with their headers). The page the cut gives on such +rows is three rows (the rows whose predecessors' bytes are under 64 MiB: 0, +27.7, 55.4 MB), 83 MB of text, so 12 rows are four pages. MongoDB stores an +item as one BSON document and refuses the shape at 28.9 MB (`BSONObjectTooLarge`, +the 16 MiB cap; `rss-mongodb-worst.out`'s first attempt), so its run holds +12 items of 1,100 sets (13.5 MB lines, `--sets 1100`): there every row is a +page of its own (the server's 16 MiB reply bound) and the shape is half the +size of the other two backends'. + +| backend | build | worst rows | idle RSS | peak (VmHWM) | peak − idle | AVAILABLE after | backup size | +|---|---|---|---|---|---|---|---| +| PostgreSQL | `ITEM_CHANNEL = 4` | 12 | 24.5 MiB | **904.5 MiB** | 880 MiB | 18.2 s | 3,793,555 B | +| SQLite | `ITEM_CHANNEL = 4`, the 15:34 build (before the sink's `BrokenPipe` answer) | 12 | 21.1 MiB | **782.7 MiB** | 762 MiB | 8.16 s | 3,793,555 B | +| SQLite | `ITEM_CHANNEL = 4`, the first build (with the `BrokenPipe` answer) | 12 | 20.7 MiB | 602.6 MiB | 582 MiB | 8.97 s | 3,793,555 B | +| SQLite | `ITEM_CHANNEL = 1` | 12 | 20.9 MiB | 555.4 MiB | 535 MiB | 8.13 s | 3,793,555 B | +| SQLite | `ITEM_CHANNEL = 1`, back to back with the row above it | 12 | 20.4 MiB | 552.2 MiB | 532 MiB | 7.95 s | 3,793,555 B | +| SQLite | `ITEM_CHANNEL = 4`, `MALLOC_ARENA_MAX=1` | 12 | 22.0 MiB | 529.5 MiB | 507 MiB | 8.76 s | 3,793,555 B | +| SQLite | `ITEM_CHANNEL = 4` | **24** | 20.3 MiB | **919.4 MiB** | 899 MiB | 15.7 s | 7,567,821 B | +| MongoDB | `ITEM_CHANNEL = 4`, 1,100 sets | 12 | 25.0 MiB | 498.7 MiB | 474 MiB | 4.7 s | 1,851,153 B | + +The driver's run-ahead on SQLite, measured back to back on one 24-row table +(`rss/sqlite-rowbuffer4/results.txt`; the final tree sets `row_buffer_size` +to four on the pool, `crates/storage-sqlite/src/store.rs`): + +| SQLite build | worst rows | idle RSS | peak (VmHWM) | peak − idle | AVAILABLE after | +|---|---|---|---|---|---| +| `row_buffer_size` 4 (final tree) | 24 | 20.2 MiB | 716.7 MiB | 696 MiB | 15.9 s | +| `row_buffer_size` 50 (sqlx's default) | 24 | 22.5 MiB | 866.2 MiB | 844 MiB | 15.7 s | +| `row_buffer_size` 4, the 1M-row table | 1,000,000 small | 22.3 MiB | 67.3 MiB | 45.0 MiB | 11.43 s | +| `row_buffer_size` 50, the 1M-row table | 1,000,000 small | 21.1 MiB | 67.6 MiB | 46.5 MiB | 11.42 s | + +The samples show the shape of it: with 50 the RSS reaches 850 MiB within a +second of the start (the worker has read every one of the 24 rows while the +stream parses the first page's three), falls to 280 MiB when the first page's +cut drops the statement, and climbs again on the second; with four it climbs +with the pipeline (236, 292, 339, 424 MiB and on) to its peak. A 1M-row +backup takes the same 11.4 s with either, so the pool keeps four. + +The terms, by the pipeline's parts, for a row of `r` = 27.7 MB of text whose +decoded `Item` is `d` ≈ 34 MB: + +- the page's text: on the first build PostgreSQL fetched the page whole + (`fetch_all`, 83 MB as the driver's rows) and copied each row's text out + as it extracted it, about one page plus one row; after the paging change it pulls + the rows from the connection one at a time and parses each as it arrives, + so one row, as SQLite, which pulls the rows one at a time; +- the page's decoded rows (up to 3 × `d` = 102 MB) in the stream's buffer + while the pipeline takes them one by one; +- one row's parse: the text, the deserializer's intermediate value and the + item, `r + 2d` ≈ 96 MB, for the row being parsed; +- the pipeline's items: the feeder's, the item channel's (`ITEM_CHANNEL`) + and the encoder's, (2 + `ITEM_CHANNEL`) × `d`: 204 MB at four, 102 MB at + one; +- the encoder's line buffer (one line, `r`), its gzip window, the chunk + sink's 1 MiB and the chunk channel's four 1 MiB chunks; +- on SQLite, the driver's run-ahead (below). + +At the instant the stream fetches a new page while the pipeline is full the +terms sum to about 400 MB with a four-item channel and about 300 MB with a +one-item channel. The high-water marks are 1.4 to 2.2 times that, and the +difference is the allocator's: the row's 203,580 short strings are allocated +on the tokio worker that parses the row and freed on the blocking thread +that encodes it, and glibc's malloc keeps that memory in its arenas between +pages. Evidence: two builds of the same pipeline code (the table's two 12-row +rows, the 15:34 build and the first build) peaked at 762 and 582 MiB over +idle on one shape, two runs of one first-build binary at 12 rows peaked at +574 and 805 MiB (the terms do not vary; the allocator's state does), and +`MALLOC_ARENA_MAX=1` took the same run to 507 MiB. The 24-row run on SQLite +(899 MiB over idle against 582 to 762 for 12 rows) adds one term the 12-row +runs hid: sqlx's SQLite driver runs each statement on a worker thread that +buffers rows ahead of the reading task (`row_buffer_size`, 50 by default), +so while the stream parses its page's three rows the worker has already read +the rows after them, up to 50 or the table's end, whichever comes first (12 +rows in the first runs, 24 in this one, 50 × the line cap, 1.6 GB, on a +large table of such rows); the page's cut drops the statement and frees them. +`rows_past_cut` is 0 because it counts the rows the stream pulls, not the +rows the driver read. The final tree sets the +pool's `row_buffer_size` to four (the table above): the run-ahead is then at +most five rows (the four the channel holds and the one the worker has stepped +and copied while its `send` waits; 160 MiB at the line cap) and the SQLite +peak is the pipeline's terms and the allocator's share. + +The item channel. The proposal was a one-item channel to save the +three items the four-item channel holds. Measured back to back on SQLite on +one loaded 1M-row table, each build once in each order +(`rss/sqlite-itemchannel1/results.txt`, `rss/sqlite-itemchannel-swapped/results.txt`): + +| order | `ITEM_CHANNEL = 4` | `ITEM_CHANNEL = 1` | +|---|---|---| +| one-item build first (18:22) | 12.45 s | 21.85 s | +| four-item build first (18:33) | 12.24 s | 21.41 s | + +A one-item channel costs a 1M-row backup 75% of its duration (every item is +a hand-off between the feeder task and the encoder's blocking thread) and +saves about 50 MiB of the worst-shape peak (582 against 532 MiB over idle, +back to back), so `ITEM_CHANNEL` stays at four and `writer.rs` says why. + +The terms do not grow with the number of small rows (the 1M run) and grow +with the row (every term but the budget and the chunks is a multiple of `r` +or `d`), which the line cap (32 MiB) and `max_item_size_bytes` bound; the +process's peak, with glibc's default arenas, grows with the number of large +rows a job reads, and does not under `MALLOC_ARENA_MAX=1` (the series of +2026-11-01 below). Section 7 of the design states these terms with the figures; +revision item 29 carries the correction (the earlier text, "the budget plus +one row of text, plus its decoded form", named the first term alone). + +## The incompressible run + +`put-random`: 256 items each holding a 256 KiB `B` attribute of +`os.urandom` bytes (64 MiB in all; 85 MB as base64 text). + +| backend | idle RSS | peak (VmHWM) | peak − idle | AVAILABLE after | backup size | +|---|---|---|---|---|---| +| PostgreSQL | 24.6 MiB | 145.1 MiB | 120.5 MiB | 5.9 s | 68,016,482 B | +| SQLite | 22.7 MiB | 97.5 MiB | 74.8 MiB | 4.3 s | 68,016,702 B | +| MongoDB | 25.6 MiB | 115.8 MiB | 90.2 MiB | 4.9 s | 68,016,522 B | + +The backup's size (68 MB) is the data's: gzip shrank nothing. The peak over +idle is one page of text (the 85 MB fits one page: 256 rows of 342 KB, which +PostgreSQL holds whole and SQLite a row at a time) plus the decoded items +(the 64 MiB of bytes) plus the channels; the encoder did not hold the file +(a 68 MB file through a 4 MiB chunk window), which is what this run proves. + +## Durations for the record (the design's RTO baseline, PR 11) + +CreateBackup to AVAILABLE, the `DescribeBackup` poll's figure, a composite +key with two GSIs, 200-byte items, a filesystem store on the same disk, the +host loaded as said above: + +| backend | 100k items | 1M items | 500k items | +|---|---|---|---| +| PostgreSQL 16.15 (loopback, separate data and catalog databases) | 1.85 s | 36.1 s | | +| SQLite 3.46 (file) | 1.44 s | 12.45 s (12.24 and 12.45 s in the comparison runs, no GSIs) | | +| MongoDB 7.0 (loopback replica set) | 1.85 s | not run | 8.81 s | + +The worst shape (12 rows of 27.7 MB, 332 MB of text): 18.2 s on PostgreSQL +on the first build and 16.5 s after the paging change (the section of 2026-11-01), +8.0 to 9.0 s on SQLite; the incompressible 64 MiB: 5.9, 4.3 and 4.9 s. The +loads themselves (the wire, 16 threads of BatchWriteItem): 100k in 217 s on +SQLite with two GSIs and 1M in 6,531 s; 1M in 463 s without GSIs; MongoDB +100k in 361 s and 500k in 1,604 s. + +## 2026-11-01: the allocator series and the PostgreSQL page's cost + +The worst shape (1,000 small rows then N rows of 27.7 MB) in two series of +six runs, each series on one SQLite release build, each run a fresh +deployment, `bench-rss.sh sqlite worst` with `WORST_COUNT` 12, 24, 48, first +with glibc's default arenas and then with `MALLOC_ARENA_MAX=1` in the +server's environment. Series 1: the build before the feeder's line +measurement was added (it measures no line at the default item limit, so the +pipeline's allocation is the same), a quiet host, samples under +`rss-series1/rss/`. Series 2: the final build, with the kill-point runs of the +same day running beside it on the host, samples under +`rss/sqlite-worst-{default,arena1}/`; each binary's md5 is recorded beside +its samples: + +| allocator | worst rows | series 1, quiet host: idle / peak / over idle, AVAILABLE after | series 2, final build, loaded host: idle / peak / over idle, AVAILABLE after | +|---|---|---|---| +| default arenas | 12 | 24.0 / 489.6 / **466 MiB**, 8.74 s | 22.7 / 729.3 / **707 MiB**, 9.15 s | +| default arenas | 24 | 23.7 / 611.4 / **588 MiB**, 18.07 s | 20.6 / 720.4 / **700 MiB**, 17.89 s | +| default arenas | 48 | 22.4 / 782.1 / **760 MiB**, 36.56 s | 23.2 / 1304.5 / **1,281 MiB**, 35.96 s | +| `MALLOC_ARENA_MAX=1` | 12 | 22.1 / 503.3 / **481 MiB**, 9.57 s | 24.0 / 476.2 / **452 MiB**, 9.99 s | +| `MALLOC_ARENA_MAX=1` | 24 | 23.5 / 475.6 / **452 MiB**, 19.55 s | 20.6 / 500.8 / **480 MiB**, 19.32 s | +| `MALLOC_ARENA_MAX=1` | 48 | 22.0 / 501.5 / **480 MiB**, 39.45 s | 22.2 / 477.6 / **455 MiB**, 39.67 s | + +The terms of the section above at this row (`r` = 26.4 MiB, `d` ≈ 32.4 MiB: +the page's three decoded rows 97 MiB, one row's parse 91 MiB, the pipeline's +six items 195 MiB, the line buffer 26 MiB, the chunks 5 MiB, the five-row +run-ahead 132 MiB) sum to 545 MiB on SQLite. With the default arenas the peak +over idle is 0.86 to 1.39 of that sum in the first series and 1.28 to 2.35 in the +second, grows with the rows in both, and differs by 240 to 520 MiB between +the two series at the same count (runs of the first build on a loaded host +reached 1,096 MiB at 48 rows, and its 12-row runs of one binary spread over +574 to 805 MiB); under one arena it is 0.83 to 0.88 of the sum in both series and +flat from 12 to 48 rows. So the pipeline's terms do not grow with the table, +the growth and the spread under the default arenas are the allocator's +retention of the strings freed on the encoder's thread (how much it keeps +depends on the host's state at the time, which is why the host's load moves +it), and the bound the design states (section 7) is the terms at the line +cap times 1.2 per job under `MALLOC_ARENA_MAX=1`, which the admin guide and +the deployment guide's service unit now carry; with the default arenas the +design records the growth and names 2.35 of the terms at 48 rows as the +largest measured. The duration grows with the rows as it should (the 48-row +table is 1.3 GB of text) and is the same under either setting. + +The PostgreSQL page (the probe `pg_adapt_probe.py`, kept with the scripts, +run twice with the first build's and the changed row-bound rule on a +1,000-small-then-60-worst table, `EXPLAIN (ANALYZE, BUFFERS)` of page 3): + +| row bound rule | page 2 (`n` = 1000, reads 984 small + 16 worst, keeps 987) | page 3: `n` asked | rows read / past the cut | temp written (8 KiB blocks) | page 3 time | pages 4, 5 (`n`, time) | +|---|---|---|---|---|---|---| +| the first build: budget / average row, floor 16 | 4.6 s | 810 | 57 / 54 | 757,844 (5.9 GB) | 18.2 s (43.7 s on a loaded host) | 16, 4.4 s each (13 rows past the cut) | +| the change: budget / largest row, floor 4 | 14.0 s | 4 | 4 / 1 | 26,564 (208 MiB) | 1.13 s | 4, 1.2 s each (1 row past the cut) | + +Page 2 is the over-read of this shape (it asks for the rows the small rows +justified and meets the large ones: up to `n` rows of up to the line cap); +its two timings differ by the host, not the rule, since both runs issue the +same query. On a table whose large rows are scattered among small ones that +page is not the only one: with the bound sized from the previous page alone +it recurs every other page, which the last section measures and the row +bound's memory corrects. The PostgreSQL release build with the changed page under +`bench-rss.sh postgres 100k 1m worst` with `GSIS=0` (one deployment, the +runs in that order, samples under `postgres/`): 100k, idle +23.7 MiB, peak 30.4 MiB, 1.87 s; 1M, idle 22.6 MiB, peak 34.4 MiB, 13.27 s +(peak(1M) − peak(100k) = +4.0 MiB, bound 112, PASS; the record's 36.1 s was +under two 1M loads and a wire suite); the worst shape at 12 rows, peak +790.2 MiB (`VmHWM`) after 16.5 s, against the first build's 904.5 MiB and 18.2 s, +with the page-text term gone and each large-row page reading one row past +its cut rather than thirteen (the idle before this run, 122.7 MiB, is the +restarted process's startup work on a table of 27.7 MB rows, not the 1M +load: on a fresh deployment holding only the worst table the restarted +process idled at 138 and 164 MiB in two later runs, climbing from 79 to +168 MiB within two seconds of the listener answering and before +`CreateBackup`, so the over-idle figure, 667 MiB, is not comparable with +the record's 880; the `VmHWM` is). The backup sizes of the `GSIS=0` tables (366,631 and 3,017,864 +bytes) are smaller than the record's because the items carry no GSI +attributes; the backup reads the base table either way. The plan near the end of a table is the planner's: on +the 100k-row test table the index scan holds ten rows from the end, and on a +1,060-row table a page after the third of the last 60 at `n` = 810 is a +bitmap scan of the primary key index and a `Sort` of the 57 rows after the +seek (PostgreSQL 16; `the_last_page_sorts_the_rows_after_the_seek` pins +both). + +## 2026-11-01, later: the row bound's memory, the server-side walk that was measured and not taken, the allocator on PostgreSQL + +The shape the sections above do not cover is a table whose large rows are +scattered among small ones, which is what a table whose large items are a +minority looks like (key order does not sort items by size; `put-worst` +groups its `zz` keys after the `a` keys). With the row bound sized from the +previous page alone, a page that holds a 27 MB row sets the next bound to 4, +four small rows set the one after back to 1000, and that page meets large +rows again: the cycle repeats every other page for the whole table. + +The SQL probe (`pg_memory.py`, kept with the scripts; the window-form page +query as `page_sql` writes it, a complete walk of a 1,000-row table with 30 +rows of 27 MB text at every 33rd key, PostgreSQL 16.15, `work_mem` 4 MB, +one plain run per rule, the server's `temp_bytes` for that run): + +| row bound rule | pages | rows the inner `LIMIT` read for 1,000 kept | `temp_bytes` | wall | +|---|---|---|---|---| +| the largest row of the previous page | 22 (`n` 16, 1000, 4, 1000, 4, …) | 5,555 | 13.38 GB | 71.5 s | +| the largest row of the last four pages | 50 (16, 1000, 4, 4, 4, 4, 1000, …) | 5,555 | 13.38 GB | 64.6 s | +| the largest row of the last pages holding at least 1,000 rows | 228 (16, 1000, then 4 to the end) | 1,901 | 3.89 GB | 22.4 s | + +Four pages of memory read the same rows as one: the four pages after a +large row hold 16 small rows, so the bound returns to 1000 as before and +the 1000-row page recurs every fifth page instead of every other. A memory +in rows holds the bound at 4 for a row bound's worth of rows after the last +large one, so on this table it never returns to 1000, and the one over-read +left is the first page that meets the large rows (984 rows read for 83 +kept). That is the rule the tree carries (`RowBoundMemory`, +`snapshot_engine.rs`): the bound that remains is the second page, and then +the page that follows a row bound's worth of small rows, each reading up to +`n` rows of up to the line cap, and such a page recurs at most once per row +bound's worth of small rows (on a table whose large rows are rarer than one +per thousand keys, at every large row). `the_row_bound_remembers_large_rows_among_small_ones` +pins the figure on a 300-row table with nine such rows: 201 rows read past +the cut, all on the second page, over 53 round trips (306 over 7 with one +page of memory; 103 round trips with a floor of 2). + +The release binary (`pg-interleave.sh`, kept with the scripts; one +deployment with a filesystem store, two tables of 20,000 small rows loaded +through the wire with 30 worst-expansion items, at keys `a0032`, `a0065`, +… `a0989` in one and `zz000`.. in the other, the server restarted before +each backup, the data database's `temp_bytes` during it, both binaries on +the same tables back to back; host load 3.6 to 8.6): + +| table | row bound rule (binary) | AVAILABLE after | data database `temp_bytes` | server `VmHWM` | +|---|---|---|---|---| +| interleaved | the previous page (`80f2d287`) | 72.0 s | 9.21 GB | 905 MiB | +| interleaved | the memory (`e020f82e`, the tree) | 54.6 s | 3.18 GB | 900 MiB | +| grouped | the previous page | 39.8 s | 2.85 GB | 1,143 MiB | +| grouped | the memory | 38.9 s | 2.85 GB | 1,066 MiB | + +The interleaved table's remaining 16 s and 3.2 GB over the grouped one are +the second page, whose `LIMIT 1000` reads keys 17 to 1,016, which hold all +30 large rows (3 kept, 27 past the cut), for the 83 it keeps; the grouped +table is unchanged (its one 1000-row page meets small rows). A second pair on the same host +earlier that hour: 77.4 s and 9.21 GB for the previous-page rule on the +interleaved table, 38.9 s and 2.85 GB grouped. The `VmHWM` figures are +under glibc's default arenas and vary by about 200 MiB between runs; they +are reported, not rested on. + +Measured and not taken: a page query that stops at the budget on the server, +a recursive walk of the primary key one index probe per row +(`WITH RECURSIVE … LATERAL … ORDER BY LIMIT 1`, the running total +carried from step to step, the walk ending after the row that reaches the +budget or at `n` steps), so the server serializes exactly the rows the page +returns and reads none past the cut. On the same two tables and binaries it +took the interleaved table in 38.8 s with 2.51 GB of temp and the grouped +one in 38.0 s with 2.51 GB (the three rows of each large-row page spilled +three times: the work table, the CTE's materialization, the sort), and the +first-page and middle-page plans on the six key shapes were the two index +probes with no other scan of the relation. Its cost is one index probe per +row, which a table of small rows pays on every row: the 1M-row backup +(`bench-rss.sh postgres 100k 1m worst`, `GSIS=0`, the two binaries back to +back on one loaded table) took 23.07 s against 11.85 s for the window form +(+95%; the 100k run 1.64 s; the worst shape at 12 rows 13.4 s and 658 MiB +`VmHWM` against 17.2 s and 553 MiB). The rule for the choice was a 1M-row +duration under 21 s (within 60% of 13 s), so the window form with the +memory stays; the walk's query, tests and figures are kept with the scripts +(`option1/`). A walk in chunks of a few rows per step would bound the +over-read at the chunk less one and cost a probe per chunk; it was not +measured. + +The 1M-row backup with the memory rule, the same back-to-back method: 13.69 s +against 12.06 s for the previous-page rule (`VmHWM` 36.7 against 36.3 MiB; +peak(1M) − peak(100k) = +4.1 MiB, bound 112, PASS); the rule changes no +page of a table with no large row (every page is the row bound either way), +and the tree's binary carries that day's other changes to the job's +validator and feeder as well, so the difference is the host's and theirs, +not the page's. + +What the memory costs where the previous-page rule had no over-read: after +a page holding a row at the line cap the bound stays at the floor for a row +bound's worth of rows, 250 pages of four, whatever those rows hold, so a +table whose large rows are rarer than one per row bound pays 250 round +trips per large row where one page of memory pages it at the row bound (a +1000-row page holding one 27 MB row and 999 small ones is under the budget +and kept whole). Measured on a 20,000-row table with a 27 MB row at every +1,100th key (18 large rows, loaded through the wire with `kill-points.py +put-worst --small 20000 --count 18 --interleave 1100`; the PostgreSQL +release build against a one-line build with one page of memory, the data +table's `idx_scan` counting the page queries; `pg-rare.sh`, kept with the +scripts): 2,585 page queries against 38, 25.6 and 24.9 s in two runs +against 25.1 s, 1.06 against 1.09 GB of temp, the same 5,821,368-byte +backup; the SQL mirror (`pg_probe.py`) walks the same shape in 2,508 pages +against 38 with 0 rows past the cut under either rule. On loopback the +extra round trips cost about 0.3 ms each and disappear under the large rows' +own serialization (about 1.3 s per 27 MB row); on a 1M-row table with 900 +such rows they are 125,000 round trips, about 40 s on loopback and about two +minutes on a 1 ms link, against the 20 minutes the large rows themselves +take. A bound that doubles after each page without a large row (4, 8, 16, +…) would reach the row bound after eight small pages instead of 250 with +the same rows read past the cut on the three shapes the mirror walked; it +was not taken, the cost above being bounded and the rule stated. +The worst shape at 12 rows under the default arenas: 787.8 MiB `VmHWM` in +17.7 s against 750.1 MiB in 17.5 s (the idle before the first of the two, +241.6 MiB, is the restarted process's startup work on the table of 27.7 MB +rows, the caveat of the previous section). + +The allocator on PostgreSQL (`MALLOC_ARENA_MAX=1 bench-rss.sh postgres +worst`, `GSIS=0`, the tree's binary, a fresh deployment): the worst shape +at 12 rows peaked at **339.9 MiB `VmHWM`** in 18.7 s (idle 144.2 MiB, the +startup work above; another measurement of the same shape on a build of the +same day: 346.3 MiB, 208 MiB over an idle of 138.0 MiB, 22.4 s), against +750 to 788 MiB under the default arenas. The terms of the section above on +PostgreSQL at this row are 413 MiB (545 MiB less the five-row run-ahead, +which PostgreSQL has not), so the one-arena peak is 0.82 of the terms here, +0.84 in the other measurement, under the 1.2 the design states, which was +measured on SQLite alone until now. + +What one arena costs the request path. On the SQLite release build with a +50k-item composite-key table, one boto3 client per thread, three +alternating repeats, 20 s per operation (`sqlite-ab.sh`, +`workload.py`, kept with the scripts; host load 2.4 to 5.2): GetItem at +8 threads 624, 621, 616 req/s against 605, 588, 598 under one arena +(−3.8%); Query of a 50-item partition at 8 threads 414, 407, 406 against +398, 391, 395 (−3.5%); Scan without Limit at 4 threads 47.7k, 46.7k, 46.0k +items/s against 44.2k, 42.7k, 43.9k (−6.9%); Scan with Limit 1000 over the +table 46.1k, 42.9k, 43.6k against 40.7k, 40.4k, 39.3k (−9.2%). On the +PostgreSQL release build (`pg-ab.sh`, the same workload and method, the +tree's binary, host load 2.6 to 3.8): GetItem 600, 631, 606 against 617, +623, 617 (+1.1%); Query 398, 413, 407 against 402, 401, 408 (−0.6%); Scan +46.5k, 46.2k, 47.3k against 46.1k, 45.8k, 45.0k (−2.1%); paged Scan 45.0k, +44.3k, 45.9k against 44.3k, 44.1k, 44.1k (−2.1%). glibc with one arena +serializes every allocation of the worker threads on one lock; SQLite's +request path runs in the process (the JSON parse and render per item and +the database's own work), where PostgreSQL's spends its time on the +database's round trip, which is why the setting shows on SQLite and is +within the spread between repeats on PostgreSQL but for the scans. A Python +client bounds these rates; the server's own share of the loss may be larger +under a faster client. The deployment guide's service unit carries the line +commented out, with the condition (a host that backs up tables of large +items) and this cost beside it. diff --git a/docs/design/15-backup-restore.md b/docs/design/15-backup-restore.md index 5e9759fc0..05164e9e3 100644 --- a/docs/design/15-backup-restore.md +++ b/docs/design/15-backup-restore.md @@ -77,7 +77,7 @@ Three store facts that rules of sections 4 and 5 rest on, and the one method a l ### 3. Storage trait changes -`SnapshotEngine` and `TableSnapshot` are `crates/storage/src/snapshot.rs` and the four `snapshot_engine.rs` files (PR 5, #395); the module docs there carry the per-backend mechanism, `snapshot.rs`'s trait docs state what `description` is and which errors end a stream, and `docs/troubleshooting.md` carries the operator side of the PostgreSQL DeleteTable change. The fetch size of every backend's stream is one shared constant, `extenddb_storage::snapshot::paging::DEFAULT_FETCH_BATCH` (1000), with a `test-hooks` override and a fetch counter the conformance check reads. +`SnapshotEngine` and `TableSnapshot` are `crates/storage/src/snapshot.rs` and the four `snapshot_engine.rs` files (PR 5, #395); the module docs there carry the per-backend mechanism, `snapshot.rs`'s trait docs state what `description` is and which errors end a stream, and `docs/troubleshooting.md` carries the operator side of the PostgreSQL DeleteTable change. The fetch size of every backend's stream is one shared constant, `extenddb_storage::snapshot::paging::DEFAULT_FETCH_BATCH` (1000), with a `test-hooks` override and a fetch counter the conformance check reads; beside it PR 8 adds the byte budget `SNAPSHOT_FETCH_BYTES` (64 MiB): on PostgreSQL and SQLite a page ends at the row bound or after the row that takes its item text to or past the budget, whichever comes first, so a page holds at least one row and at most the budget plus one row, with the cut applied in the page query on PostgreSQL (a keyset page in the snapshot's transaction, three layers with the cut in the outer one; the server reads up to `n - 1` rows past the cut, PR 8's section states the bound) and while the rows are pulled on SQLite (PR 8's section states the page and what it costs the server); a MongoDB batch is under the budget by the server's 16 MiB reply bound and applies no cut. - Cassandra: `open_table_snapshot` returns `Unsupported`. Cassandra has no `SnapshotEngine`, so the engine routes CreateBackup there to the legacy create (`LegacyBackupReader::create`, the #384 in-catalog path) whether or not `[backup.store]` is configured, the path every backend takes while no store is configured (section 7); the off-host path is never taken there. That stays until PR 14, by which time either a token-range `SnapshotEngine` with a documented non-point-in-time `SnapshotInfo.kind` (`cassandra_token_scan`) has been accepted as a design change to this section, or Cassandra backups are documented as unavailable. RFC-0003 (backend acceptance criteria) has no backup criterion today; the criterion this trait implies ("a backend that cannot fix an instant across a table returns `Unsupported` rather than a best-effort stream") is to be proposed there in a separate docs change, not in PR 5. @@ -98,7 +98,7 @@ The invariants and state machines every later PR honours, by lettered part; the (d) Identities. The ARN encodes partition, region, account, table, and backup id, and the key prefix is derived from account, table and id deterministically by `BackupArn::store_prefix`, under the configured store's prefix, and no row stores it, so restore, sync, purge and the reconciler derive it the same way and `describe_backup(arn)` works from the store alone after a sync. The engine, the writer and sync all parse and format ARNs through one implementation, `BackupArn` in `extenddb_core::backup_format` (#394); the engine keeps #384's region handling: it resolves a backup under the ARN as written and under the ARN re-spelled in its own region, and refuses a partition other than its own. `backup_creation_date_time` and `created_at` are the id's instant, so the ARN, the manifest, the API's reported time and the ListBackups cursor agree by construction. One row is addressed by `BackupKey` (`account_id`, `backup_id`, `backup_arn`): PostgreSQL, SQLite and MongoDB key `backup_index` by (`account_id`, `backup_id`), Cassandra keys `backups_by_arn` by (`account_id`, `backup_arn`); no method takes a bare backup id, and the engine treats a row whose `account_id`, `table_name` or `backup_arn` (after that re-spelling) differs from the request's as absent. One target is addressed by `RestoreTargetKey` (`account_id`, `table_name`, `table_id`), the table id guarding against a table re-created under the name. -(e) The claim. There is no single-instance lease on PostgreSQL or MongoDB, so each job holds a claim that both excludes a second owner and lets a successor detect a dead one. Every backend sets `owner_generation` (a fresh random value) at claim time, and every catalog write by the job is `WHERE owner_generation = $gen`, so a job that lost its claim cannot publish over a successor's work. No row ever exists unowned: on every backend the claim precedes the row's insert or is the insert itself (5.1 step 2); what the claim is per backend is PR 6's section. Who is dead: on SQLite every CREATING row at startup, and at runtime a row whose in-process slot no task holds; on PostgreSQL, MongoDB and Cassandra a row whose `heartbeat_at` is older than `stale_job_after_secs` (default 300), the one liveness rule the three share. On PostgreSQL the advisory lock is the claim's proof against a second owner, not the liveness test: a free lock shows that the owner's catalog session ended, not that its process stopped (a session a managed server dropped leaves a process whose data writes carry no generation), and the owner itself aborts at its next heartbeat inside the stale window, so the reconciler reads the heartbeat alone and never probes the lock. Staleness is always measured on the database's clock, never on an application host's. The reconciler takes the same claim as a worker, with a fresh `owner_generation` that fences the dead job's generation out, leaves `heartbeat_at` untouched (a refreshed heartbeat would restart the stale window the prefix deletion waits for), writes AVAILABLE or FAILED at once, both conditioned on `status = CREATING`, and never continues the upload. A job whose conditional write matches no row, or whose conditional write errors with an unknown outcome, performs no further store or table operation, deletes of a prefix or of a restore target included (5.1 step 5, 5.3 step 7). A FAILED `restore_jobs` row whose target table is still CREATING (the FAILED write landed and the target's removal did not, whether the job's own or the reconciler's) is the FAILED-target duty: the reconciler lists it once the row's `completed_at` (or, where none was stamped, its `heartbeat_at`) is older than `stale_job_after_secs`, so a writer still on its own failure path finishes it, and removes the target through `remove_restore_target` with no second FAILED write; PR 6's reconciler runs it for every kind of record, since on this rule the kind does not matter (PR 8 adds the `target_not_empty` exception). On Cassandra the removal is itself two writes, a lightweight transaction claiming the target CREATING to DELETING and then the drop, so the duty also lists a FAILED row whose target is DELETING with its registration still present, on the same staleness rule, and `remove_restore_target` resumes the removal from the drop; PostgreSQL and SQLite remove in one transaction and MongoDB deletes the document first, so neither leaves that state. +(e) The claim. There is no single-instance lease on PostgreSQL or MongoDB, so each job holds a claim that both excludes a second owner and lets a successor detect a dead one. Every backend sets `owner_generation` (a fresh random value) at claim time, and every catalog write by the job is `WHERE owner_generation = $gen`, so a job that lost its claim cannot publish over a successor's work. No row ever exists unowned: on every backend the claim precedes the row's insert or is the insert itself (5.1 step 2); what the claim is per backend is PR 6's section. Who is dead: on SQLite every CREATING row at startup, and at runtime a row whose in-process slot no task holds; on PostgreSQL, MongoDB and Cassandra a row whose `heartbeat_at` is older than `stale_job_after_secs` (default 300), the rule MongoDB and Cassandra share; PostgreSQL adds the lock. On PostgreSQL the advisory lock is the claim's proof against a second owner and not a liveness test on its own: a free lock shows that the owner's catalog session ended, not that its process stopped (a session a managed server dropped leaves a process whose data writes carry no generation), and the owner itself aborts at its next heartbeat inside the stale window, so the reconciler takes over only a row whose heartbeat is stale **and** whose lock is free: a held lock with a stale heartbeat is a process that is stopped, not dead (SIGSTOP, a paused VM), and its row stays with it until its session ends or it resumes and finishes; a host that vanished releases the lock when the server's keepalives give up (about a minute with `CLAIM_KEEPALIVES`), so the takeover waits for the later of the two. Staleness is always measured on the database's clock, never on an application host's: the engine holds no clock, and every age test is a predicate of the backend's own (`BackupCatalog::claim_dead_backup(key, stale_after)`, `list_backup_rows_older_than(status, store, clock, age)` on the heartbeat or the completion column, PR 8's section). The reconciler takes over a dead row through `claim_dead_backup`, the same claim as a worker's with a fresh `owner_generation` that fences the dead job's generation out (PostgreSQL's lock and SQLite's slot refuse a second taker outright; on MongoDB and Cassandra the generation fence orders two takers, since the successor is a live owner to a later taker only once it heartbeats, and the reconciler writes its one status before any beat, so two instances' reconcilers can both take over one dead row within that gap and exactly one resolution lands, the other's write finding no row; Cassandra's conditional write also carries the heartbeat it read, so a beat landing between the read and the write refuses the takeover), leaves `heartbeat_at` untouched (a refreshed heartbeat would restart the stale window the prefix deletion waits for), writes AVAILABLE or FAILED at once, both conditioned on `status = CREATING`, and never continues the upload. A job whose conditional write matches no row, or whose conditional write's outcome stays unknown after the re-read, performs no further store or table operation, deletes of a prefix or of a restore target included (5.1 step 5, 5.3 step 7); each unknown outcome is followed by a re-read; the write is sent once more at most; after the second write a re-read that shows the row in the target status counts as matched and anything else as matching no row. A FAILED `restore_jobs` row whose target table is still CREATING (the FAILED write landed and the target's removal did not, whether the job's own or the reconciler's) is the FAILED-target duty: the reconciler lists it once the row's `completed_at` (or, where none was stamped, its `heartbeat_at`) is older than `stale_job_after_secs`, so a writer still on its own failure path finishes it, and removes the target through `remove_restore_target` with no second FAILED write; PR 6's reconciler runs it for every kind of record, since on this rule the kind does not matter (PR 8 adds the `target_not_empty` exception). On Cassandra the removal is itself two writes, a lightweight transaction claiming the target CREATING to DELETING and then the drop, so the duty also lists a FAILED row whose target is DELETING with its registration still present, on the same staleness rule, and `remove_restore_target` resumes the removal from the drop; PostgreSQL and SQLite remove in one transaction and MongoDB deletes the document first, so neither leaves that state. (f) The maintenance claim. `backup purge`, `backup remove-target` and `backup sync` with `--move-store`, `--forget-store`, `--forget-backup`, `--prune-missing` or `--adopt-store` change rows or bytes that jobs on any instance may be using, and a job permit (5.1 step 1) is per instance, so these commands take a catalog-level maintenance claim first (PR 10's section: the claim per backend and the SQLite serve lock). Every backup and restore job checks the claim after its own claim and row exist (5.1 step 2, 5.3 step 4; this release's in-catalog create and restore check before their row, since the backends commit it and detach, revision item 28) and refuses while it is held; while a claim is held the reconciler runs only the duties that resolve the states the command waits on (DELETING rows, rows whose owner is dead, the FAILED-target duty) and skips every other duty; the command waits until no CREATING or FAILED backup row of the configured store has a live owner (a heartbeat within `stale_job_after_secs`; a FAILED row's heartbeat may still be refreshed by a job that has not seen its claim lost) and no RUNNING `restore_jobs` row remains, a DELETING row not being waited on (nothing heartbeats one; the command finishes it as it finishes every row), and refuses and releases the claim if they have not gone within `2 * stale_job_after_secs`, which it prints; a second maintenance command while one holds the claim fails at once. Because the job checks after its row exists and the command checks rows after its claim exists, one of the two always sees the other. @@ -116,12 +116,12 @@ Per operation, the contract: the request checks in their order, the error each r #### 5.1 CreateBackup -1. Validate the request as #384 does and in its order, every check before the permit: `TableName` resolved (a table ARN is accepted), the table exists and is ACTIVE for the account (`TableNotFoundException`, "Table not found: ", otherwise, whether or not a permit is free), the server's region and the table name fit a backup ARN (`check_backup_arn_fits`, each refusal naming its cause: a `server.region` outside `[a-z0-9-]` is a ValidationException naming the setting, since no backup ARN can be formed for any table under it; a name over 255 bytes or made only of dots, the store-prefix rules, is one naming the rule, applied before the ARN is formed so the limit is named whatever the name's length). #384's `refuse_backup_of_vector_table` is not among the off-host path's checks (the manifest records vector indexes, section 1); the engine keeps the refusal in front of `LegacyBackupReader::create`, the path of every backend without `[backup.store]` and of Cassandra always (section 7), and the tests that change are PR 8's and PR 9a's sections. A second CreateBackup on a table that already has a CREATING backup is allowed and takes its own snapshot; the `BackupInUseException` the service models on CreateBackup is not raised. Then reserve a job slot: `max_concurrent_jobs` is a per-instance semaphore the request `try_acquire`s (the maintenance claim of section 4 is the cross-instance exclusion), and the permit lives with the job; if none is free return `LimitExceededException` before opening anything, because a queued snapshot would hold a PostgreSQL transaction or a MongoDB snapshot session while it waited; RestoreTableFromBackup and the legacy create take a permit from the same semaphore and return the same error (5.3 step 1, section 3). The default is 4 (section 7), so one running restore does not turn every other backup operation on the instance into `LimitExceededException`. -2. Open the table snapshot (`SnapshotEngine::open_table_snapshot`) in the request. The pin happens during the request, so the backup holds every write acknowledged before the request was received, which is stricter than the service's one-minute window. Mint the backup id from the millisecond the open returned, just after the pin (`format_backup_id`), and the owner generation, then, through `BackupCatalog::claim_backup` (section 3), insert the catalog row with status CREATING, `created_at` = the id's instant, `owner_generation`, `heartbeat_at` = the database's `now()`, and `source_table_details_json` taken from the snapshot's description; then check the maintenance claim (section 4; this release's in-catalog create checks it before its row, revision item 28): if it is held, set the row FAILED with reason `maintenance`, release the snapshot, the lock and the permit, and return `LimitExceededException`; otherwise return `BackupDetails` with BackupStatus CREATING. Everything from the open to the hand-off in step 3 runs in a detached task whose result the request awaits, so a client that disconnects mid-request cannot leave a row whose owner was never spawned; if the row insert fails the snapshot is dropped, the lock released, the permit returned, and nothing was written. +1. Validate the request as #384 does and in its order, every check before the permit: `TableName` resolved (a table ARN is accepted), the table exists and is ACTIVE for the account (`TableNotFoundException`, "Table not found: ", otherwise, whether or not a permit is free), the server's region and the table name fit a backup ARN (`check_backup_arn_fits`, each refusal naming its cause: a `server.region` outside `[a-z0-9-]` is a ValidationException naming the setting, since no backup ARN can be formed for any table under it; a name over 255 bytes or made only of dots, the store-prefix rules, is one naming the rule, applied before the ARN is formed so the limit is named whatever the name's length). #384's `refuse_backup_of_vector_table` is not among the off-host path's checks (the manifest records vector indexes, section 1); the engine keeps the refusal in front of `LegacyBackupReader::create`, the path of every backend without `[backup.store]` and of Cassandra always (section 7), and the tests that change are PR 8's and PR 9a's sections. A second CreateBackup on a table that already has a CREATING backup is allowed and takes its own snapshot; the `BackupInUseException` the service models on CreateBackup is not raised. Then, on the store-backed path, the store must be healthy (section 7: a store that is unhealthy or whose first check is pending refuses with the ValidationException naming its kind and the category, before the permit); then reserve a job slot: `max_concurrent_jobs` is a per-instance semaphore the request `try_acquire`s (the maintenance claim of section 4 is the cross-instance exclusion), and the permit lives with the job; if none is free return `LimitExceededException` before opening anything, because a queued snapshot would hold a PostgreSQL transaction or a MongoDB snapshot session while it waited; RestoreTableFromBackup and the legacy create take a permit from the same semaphore and return the same error (5.3 step 1, section 3). The default is 4 (section 7), so one running restore does not turn every other backup operation on the instance into `LimitExceededException`. +2. Open the table snapshot (`SnapshotEngine::open_table_snapshot`) in the request. The pin happens during the request, so the backup holds every write acknowledged before the request was received, which is stricter than the service's one-minute window. Mint the backup id from the millisecond the open returned, just after the pin (`format_backup_id`), and the owner generation, then, through `BackupCatalog::claim_backup` (section 3), insert the catalog row with status CREATING, `created_at` = the id's instant, `owner_generation`, `heartbeat_at` = the database's `now()`, and `source_table_details_json` the manifest's `source_table` object built from the snapshot's description and the table's TTL, which is read beside the open and not under the snapshot (`extenddb_storage::snapshot::source_table_schema`; its `item_count` is the description's figure, the stats pass's, until the AVAILABLE write replaces the object with the manifest's, whose count is the stream's, so DescribeBackup's `ItemCount` is the table's count as last counted while CREATING and the backup's exact count once AVAILABLE); then check the maintenance claim (section 4; this release's in-catalog create checks it before its row, revision item 28): if it is held, set the row FAILED with reason `maintenance`, release the snapshot, the lock and the permit, and return `LimitExceededException`, or if it cannot be read, set the row FAILED with the read's error as its reason and answer that error; otherwise return `BackupDetails` with BackupStatus CREATING. Everything from the open to the hand-off in step 3 runs in a detached task whose result the request awaits, so a client that disconnects mid-request cannot leave a row whose owner was never spawned; if the row insert fails the snapshot is dropped, the lock released, the permit returned, and nothing was written. 3. The claim: section 4(e) states the rules; what the claim is on each backend, the claim connection's settings and the hand-off to the worker are PR 6's and PR 8's sections. -4. The job streams items from the snapshot into gzip data files through `DataFileEncoder`; a file closes once its `uncompressed_bytes` reaches `data_file_target_bytes` (default 64 MiB), so a file exceeds the knob by less than one line, at most `MAX_LINE_BYTES` (32 MiB, section 1), which is also the knob's minimum. Every item is checked, before it is encoded, with the item validator PR 8 adds to `extenddb-core` (`extenddb_core::validation::validate_restorable_item`, with this instance's `max_item_size_bytes`; the restore binds the same function, 5.3 step 5), so no backup that no restore can load is reported AVAILABLE; that is a regression from main for a table that already holds a value #618 let in (Drawbacks), and the reason reaches the operator only (`backup list --failed` and the log); the client sees CREATING and then BackupNotFound. The heartbeat is written on a timer (`heartbeat_interval_secs`, default 15) for as long as the job task is alive, independent of upload progress; before each put the job checks that its latest heartbeat confirmed the claim within `heartbeat_interval_secs` and starts no put otherwise; a heartbeat that does not confirm the claim, or that has not returned within `heartbeat_interval_secs`, aborts the job without further store writes, deletes included. `stale_job_after_secs` must exceed several heartbeat periods; startup refuses a value under three times the period the jobs run at, which until the knob reaches the jobs is the larger of `heartbeat_interval_secs` and the 15 s constant `extenddb_storage::backup_catalog::JOB_HEARTBEAT_INTERVAL` every backend's heartbeat loop reads (so 45 s at the least), which is what makes that detection precede a reconciler's deletion. After the last file: write `manifest-files.json`, `manifest-summary.json`, `extenddb-manifest.sha256` (the order #394's `BackupDocuments::build` doc gives); re-check the claim; then write `extenddb-manifest.json` and update the row to AVAILABLE with size and count (`JobClaim::set_status(CREATING, AVAILABLE)`, conditional on the generation and on `status = CREATING`). An update whose outcome is unknown is followed by a re-read of the row, not treated as a failure; one that matches zero rows ends the job with no store access. Store writes are not fenced by the generation; the re-check narrows the window in which a fenced-out job can still write a manifest, and the FAILED tombstone (section 4) closes what remains: sync never adopts a prefix whose row is FAILED, and the reconciler never deletes a prefix while the row's `heartbeat_at` is younger than `stale_job_after_secs` on the catalog database's clock, so a late manifest from a fenced-out job lands under a tombstone that sync will not adopt. -5. On any failure, including `SnapshotExpired`: set the row FAILED with the reason, conditional on the generation and on `status = CREATING`, so a job whose AVAILABLE update committed with its response lost cannot turn a backup a client has seen AVAILABLE into a deleted one. Only if that write reports exactly one row matched does the job stop its uploads, await its in-flight puts and `delete_prefix` the partial backup (which on S3 also aborts any upload still open under it); a write that matches zero rows, or any error on the claim connection, ends the job with no further store access of any kind, because the holder of the claim owns the prefix. If the delete fails the reconciler retries it (section 4). A backup never resumes from a cursor position, so a lost snapshot means a new CreateBackup. DescribeBackup returns BackupNotFound; ListBackups omits it. -6. Shutdown: the snapshot lives in the process and dies with it, so a cancelled job can never complete. On cancellation the worker runs step 5 with reason `shutdown`, and the server's shutdown awaits each job's failure path within the drain window before the runtime stops: runtime shutdown sends no abort for a put still in flight (section 2), so step 5's `delete_prefix` is what sweeps its upload. If the catalog is unreachable it leaves the row CREATING for the reconciler; if only the store is unreachable the row is FAILED and the reconciler deletes the prefix later. Crash: on startup and every reconciler cycle, a CREATING row is resolved when its owner is dead, by the test and the claim of section 4(e); the prefix of a FAILED row is deleted only once `heartbeat_at` is older than `stale_job_after_secs`, the delay step 4 relies on. A restarted process cannot finish a CREATING backup; AVAILABLE from an existing manifest or FAILED are the only two outcomes. +4. The job streams items from the snapshot into gzip data files through `DataFileEncoder`; a file closes once its `uncompressed_bytes` reaches `data_file_target_bytes` (default 64 MiB), so a file exceeds the knob by less than one line, at most `MAX_LINE_BYTES` (32 MiB, section 1), which is also the knob's minimum. Every item is checked, before it is encoded, with the item validator PR 8 adds to `extenddb-core` (`extenddb_core::validation::validate_restorable_item`, with this instance's `max_item_size_bytes`, hash and range key sizes and attribute name size (`RestorableLimits`); the restore binds the same function, 5.3 step 5), so no backup that no restore can load is reported AVAILABLE. An item over the default item limit has its data file line measured against the line cap through a counting writer that holds no line, and one over the cap (only possible above the default item limit) is refused through the same count-and-log path as a #618 item (`RestorableItemReason::LineTooLong`, whose reason names the cap, the line's own length being unknown once the count stops); that is a regression from main for a table that already holds a value #618 let in (Drawbacks), and the reason reaches the operator only (`backup list --failed` and the log); the client sees CREATING and then BackupNotFound. The heartbeat is written on a timer (`heartbeat_interval_secs`, default 15) for as long as the job task is alive, independent of upload progress; before each put the job checks that no loss has been raised and that its latest heartbeat confirmed the claim within two periods (one for the timer, one for the beat to return: a beat sent at the tick returns up to one period later) and starts no put otherwise; the claim's insert stands for the first confirmed beat (the job takes the instant before `claim_backup` and the heartbeat's watch starts from it), so the maintenance check between the insert and the job's start counts against the two periods and a first put that comes later than two periods after the insert is refused as a lost claim, the row left to the reconciler; a heartbeat that does not confirm the claim, or that has not returned within one period, aborts the job without further store writes, deletes included, and without a catalog write either, the claim connection being the one that failed (the row is left to the reconciler). `stale_job_after_secs` must exceed several heartbeat periods; startup refuses a value under three times the period the jobs run at, the larger of `heartbeat_interval_secs` and the 15 s constant `extenddb_storage::backup_catalog::JOB_HEARTBEAT_INTERVAL` (so 45 s at the least): the store-backed job heartbeats at that larger value, the in-catalog paths at the constant until PR 14 removes them; two periods stay inside three, so a job whose claim connection has died stops its puts before a reconciler's deletion; on MongoDB and Cassandra a process stopped across the window (SIGSTOP) resumes the put it had in flight after the takeover, which the FAILED tombstone and the FAILED-prefix retry of step 5 and section 4 cover; on PostgreSQL its claim session holds the advisory lock while it is stopped, so no takeover happens and the job resumes and completes under its own generation (4(e)). After the last file: write `manifest-files.json`, `manifest-summary.json`, `extenddb-manifest.sha256` (the order #394's `BackupDocuments::build` doc gives); re-check the claim; then write `extenddb-manifest.json` and update the row to AVAILABLE with size and count (`JobClaim::set_status(CREATING, AVAILABLE)`, conditional on the generation and on `status = CREATING`). An update whose outcome is unknown is followed by a re-read of the row, not treated as a failure (4(e): one more write when the row is still CREATING under the job's generation); one that matches zero rows ends the job with no store access. Store writes are not fenced by the generation; the re-check narrows the window in which a fenced-out job can still write a manifest, and the FAILED tombstone (section 4) closes what remains: sync never adopts a prefix whose row is FAILED, and the reconciler never deletes a prefix while the row's `heartbeat_at` is younger than `stale_job_after_secs` on the catalog database's clock, so a late manifest from a fenced-out job lands under a tombstone that sync will not adopt. +5. On any failure, including `SnapshotExpired`: set the row FAILED with the reason, conditional on the generation and on `status = CREATING`, so a job whose AVAILABLE update committed with its response lost cannot turn a backup a client has seen AVAILABLE into a deleted one. The pipeline ends its puts before the FAILED write (the failing put has ended, and a stopped encoder ends the body of the one running, which the store then fails or completes; nothing is started after the failure) and the feeder stops reading at once when the uploader fails, so the write does not wait for the snapshot stream's next page, and there is no put in flight at the write; only if that write reports exactly one row matched does the job `delete_prefix` the partial backup (which on S3 also aborts any upload still open under it); a write that matches zero rows, or any error on the claim connection, ends the job with no further store access of any kind, because the holder of the claim owns the prefix. If the delete fails the reconciler retries it (section 4). A backup never resumes from a cursor position, so a lost snapshot means a new CreateBackup. DescribeBackup returns BackupNotFound; ListBackups omits it. +6. Shutdown: the snapshot lives in the process and dies with it; a job cancelled before its manifest put never completes; one cancelled after it completes AVAILABLE, since the manifest is the commit record. On cancellation the worker runs step 5 with reason `shutdown`, and the server's shutdown awaits each job's failure path within the drain window before the runtime stops: runtime shutdown sends no abort for a put still in flight (section 2), so step 5's `delete_prefix` is what sweeps its upload. If the catalog is unreachable it leaves the row CREATING for the reconciler; if only the store is unreachable the row is FAILED and the reconciler deletes the prefix later. Crash: on startup and every reconciler cycle, a CREATING row is resolved when its owner is dead, by the test and the claim of section 4(e); the prefix of a FAILED row is deleted only once `heartbeat_at` is older than `stale_job_after_secs`, the delay step 4 relies on; on a row the reconciler itself just failed that window is counted from the dead owner's last beat, so the FAILED-prefix duty of the same cycle, which runs after the dead-owner duty, deletes it as soon as that beat is stale. A restarted process cannot finish a CREATING backup; AVAILABLE from an existing manifest or FAILED are the only two outcomes. #### 5.2 DescribeBackup, ListBackups, DeleteBackup @@ -159,8 +159,8 @@ Export gains the store as a target and uses `open_table_snapshot`, which fixes t ```toml [backup] data_file_target_bytes = 67108864 # uncompressed bytes per data file; a file closes once it reaches this, so it exceeds it by less than one line (5.1 step 4); minimum 32 MiB, the line cap -heartbeat_interval_secs = 15 # job heartbeat timer (5.1 step 4); #384's restore heartbeat interval -stale_job_after_secs = 300 # heartbeat age after which a PostgreSQL, MongoDB or Cassandra job is dead (4(e)), and the delay before a reconciler deletes a dead job's prefix on every backend; at least three times the period the jobs run at, max(heartbeat_interval_secs, JOB_HEARTBEAT_INTERVAL), so 45 at the least until the knob reaches the jobs (5.1 step 4); a new value (#384's restore grace is 60 s) +heartbeat_interval_secs = 15 # the store-backed job's heartbeat timer, floored at JOB_HEARTBEAT_INTERVAL (15 s), which the in-catalog paths keep (5.1 step 4); #384's restore heartbeat interval +stale_job_after_secs = 300 # heartbeat age after which a PostgreSQL, MongoDB or Cassandra job is dead (4(e)), and the delay before a reconciler deletes a dead job's prefix on every backend; at least three times the period the jobs run at, max(heartbeat_interval_secs, JOB_HEARTBEAT_INTERVAL), so 45 at the least (5.1 step 4); a new value (#384's restore grace is 60 s) failed_row_retention_secs = 604800 # FAILED tombstones kept for the operator, 7 days reconcile_interval_secs = 60 # reconciler cycle (section 4) max_concurrent_jobs = 4 # backup and restore jobs running at once on this instance, legacy creates and restores included; a further CreateBackup or RestoreTableFromBackup is LimitExceeded, the text naming the bound it met; at most 1024; on PostgreSQL, pool_size and catalog_pool_size are raised to at least 2 * max_concurrent_jobs + 10 (below) @@ -171,9 +171,9 @@ spool_path = "" # restore download area for an S3 store; d On SQLite (file-backed) a missing `[backup]` section is equivalent to `kind = "filesystem"` with `path` = `backups/` beside the database file; a `validate` failure of that defaulted root is a logged warning that leaves the store unconfigured, not a startup error, so an upgrade never stops a server that never configured backups, while an explicitly configured root that fails `validate` is still a startup error. The warning holds only while the catalog has no backup rows for the defaulted root (`read_store_ids`); once it has, the root has been used and is handled as an explicitly configured one (revision item 28). On PostgreSQL and MongoDB a missing `[backup.store]` leaves the off-host path unconfigured and startup logs that; `kind = "disabled"` leaves it unconfigured on purpose on any backend. Without a configured store, CreateBackup on every backend uses the legacy in-catalog create (`LegacyBackupReader::create`, section 3) and logs a deprecation warning naming `[backup.store]` on each call, until PR 14 removes that path; after it CreateBackup without a configured store returns a ValidationException naming `[backup.store]`. Legacy backups stay readable, listable, restorable and deletable either way. `sqlite-memory` (any `:memory:` database path) has no database file and so no default root: CreateBackup there takes the off-host path only with `[backup.store]` set and the legacy create without it. Cassandra has no `SnapshotEngine`, so it uses the legacy create even with a store configured (section 3); a configured store serves restores of off-host backups into Cassandra and nothing else there. -Memory per running job, which operators size hosts from: a backup holds the upload window (`upload_part_size_bytes * max_concurrent_uploads` on an S3 store, one write buffer on a filesystem store), the read path, the gzip window and the manifest's data-file entries until the manifest is written; with the byte budget PR 8 adds to the PostgreSQL and SQLite streams (`SNAPSHOT_FETCH_BYTES`, a constant beside `DEFAULT_FETCH_BATCH`, not a `[backup]` key in v1) the read path holds at most one page, the budget plus one row of text, plus its decoded form, and neither term grows with `data_file_target_bytes` or with the table, except on `sqlite-memory`, whose open copies the whole table (section 3). A restore holds one line's decode plus one insert batch plus the parsed manifest for the whole job; neither grows with the table. Each running backup also holds one connection from the PostgreSQL data pool for its duration, and on the separate layout one catalog-pool connection while its snapshot opens (section 3); the backend admits at most half the pool (after the raise below) as open snapshots and refuses more with `LimitExceeded`; a running restore holds one data connection and has no slot of its own after PR 6; and each PostgreSQL job holds one dedicated claim connection outside the pool. `pool_size` sizes both pools, and the rule is `pool_size >= max(MIN_POOL_SIZE, 2 * max_concurrent_jobs + 10)`: twice the jobs, so the half-pool bound admits every job `max_concurrent_jobs` allows, plus ten for requests; 18 at the defaults, under the default `pool_size` of 20. It applies whether or not `[backup.store]` is configured, because legacy restores are jobs too. Startup raises a smaller `pool_size` to that value with a warning naming both numbers, as it already raises a `pool_size` under `MIN_POOL_SIZE` (10) today. +Memory per running job, which operators size hosts from. A backup's terms, for a row of `r` bytes of item text whose decoded `Item` is `d` bytes (`d` is about 1.2 × `r` for a row of many short strings, #394's worst shape, where every `N` is a 133-character string): the read path holds the page's decoded rows, `d` each, while they wait for the pipeline, at most the rows of the budget PR 8 adds to the PostgreSQL and SQLite streams (`SNAPSHOT_FETCH_BYTES`, 64 MiB, a constant beside `DEFAULT_FETCH_BATCH`, not a `[backup]` key in v1) plus one row, and one row's parse, `r + 2d`, since the deserializer builds an intermediate value before the item; PostgreSQL pulls the page's rows from the connection one at a time and parses each as it arrives, so it holds no page text beyond the row being parsed, and SQLite pulls a row at a time with the driver's worker at most five rows ahead of the pull, `r` each (`row_buffer_size` on the pool, four, plus the row the worker holds while its `send` waits; 160 MiB at the line cap; the default of 50 made the run-ahead, not the page, the largest term on a table of large rows). The pipeline holds the feeder's item, the item channel's items (`ITEM_CHANNEL`, four) and the encoder's item, `d` each; the encoder's line buffer (one line, at most 32 MiB, the line cap) and gzip window; the feeder's measurement of an item over the default `max_item_size_bytes` holds no line (it counts the line's bytes through a writer that stops at the cap), so the terms are the same on an instance that raises the limit, at the largest stored row; the chunk sink's 1 MiB buffer and the chunk channel (four 1 MiB chunks); the upload window (`upload_part_size_bytes * max_concurrent_uploads` on an S3 store, one write buffer on a filesystem store); and the manifest's data-file entries until the manifest is written. No term grows with `data_file_target_bytes` or with the table, except on `sqlite-memory`, whose open copies the whole table (section 3); every term but the budget and the chunks grows with the row, which `max_item_size_bytes` bounds (the line cap bounds the line the job writes and names the item it refuses; the row is read and decoded before the feeder measures it). The sizing rule, per running job, is the terms at the line cap times the allocator's factor, times `max_concurrent_jobs`; the rule is stated at the default `max_item_size_bytes`, where no stored item's line exceeds the cap (27.7 MB is the largest line a 400 KB item expands to, `MAX_LINE_BYTES`'s doc); on an instance that raises it the page's decoded rows, one row's parse and the pipeline's items are at the largest stored row, which only `max_item_size_bytes` bounds (about 68 times the item's size as a line, for #394's worst shape). The terms at the line cap (`r` = 32 MiB, `d` = 1.2 `r`): the page's rows, 3 `d` = 115 MiB; one row's parse, `r` + 2 `d` = 109 MiB; the pipeline's six items, 6 `d` = 230 MiB; the line buffer, 32 MiB; the chunks, 5 MiB; on SQLite the run-ahead, 5 `r` = 160 MiB: about 490 MiB on PostgreSQL and 650 MiB on SQLite, an upper bound, since the run-ahead and a full pipeline are not both at their largest at one instant. The allocator's factor is glibc's: the row's 200,000 short strings are allocated on the tokio worker that parses the row and freed on the blocking thread that encodes it, and with glibc's default per-thread arenas that memory stays in the arenas between pages, so the process's high-water mark grows with the number of large rows the job reads and varies by about 200 MiB between identical runs; under `MALLOC_ARENA_MAX=1` (one arena, set in the server's environment on a host that backs up tables of large items; the admin guide's `max_concurrent_jobs` row carries it and the deployment guide's service unit carries the line commented out with that condition, since one arena serializes the worker threads' allocations on one lock and costs the request path 3.5 to 9.2% on SQLite and about 2% on PostgreSQL (the scans, 2.1%), measured on GetItem, Query and Scan with a Python client, the record) the peak is the terms' and does not grow with the table. Measured on PR 8's 16-core host at 27.7 MB rows (`r` = 26.4 MiB, three rows to a page; the terms' sum 545 MiB on SQLite), 1,000 small rows then 12, 24 and 48 such rows, each series six runs back to back on one SQLite binary (`docs/design/15-backup-restore-bench-pr8.md`; revision item 29): with the default arenas the peak over idle was 466, 588 and 760 MiB on a quiet host and 707, 700 and 1,281 MiB on the final build with the kill-point runs beside it (0.86 to 2.35 of the terms, growing with the rows and varying by hundreds of MiB between series; runs of the PR's first build on a loaded host reached 1,096 MiB at 48 rows; no plateau was measured), and under `MALLOC_ARENA_MAX=1` 481, 452 and 480 MiB and then 452, 480 and 455 MiB (0.83 to 0.88 of the terms, flat from 12 to 48 rows in both series; the SQLite ratios are of the peak over idle), and on PostgreSQL at 12 rows 340 MiB `VmHWM` under one arena against 750 to 788 MiB under the default arenas (0.82 of the terms at that row on PostgreSQL, 413 MiB, which has no run-ahead; that ratio is of the `VmHWM`, the process's whole high-water mark, and about 0.47 over its idle of 144 MiB); so the bound an operator can hold the process to is the terms at the line cap times 1.2 per job under `MALLOC_ARENA_MAX=1`, and under the default arenas a figure that grows with the large rows a job reads, 2.35 of the terms at 48 rows being the largest measured. A one-item channel saved about 50 MiB of the peak and cost a 1M-row backup 75% of its duration, so the channel stays at four. With 200-byte rows every term that grows with the row is under 1 MiB, the chunk channel and the sink hold their 5 MiB whatever the row, and the high-water mark of a 1M-row backup is within 10 MiB of a 100k-row backup's on PostgreSQL and SQLite (the bound the acceptance asserts is 112 MiB: the budget, the line cap and the channels) and the 500k-row backup's within 5 MiB on MongoDB, where the 500k run stands in for the 1M run (its load alone took 27 minutes through the wire). A restore holds one line's decode plus one insert batch plus the parsed manifest for the whole job; neither grows with the table. Each running backup also holds one connection from the PostgreSQL data pool for its duration, and on the separate layout one catalog-pool connection while its snapshot opens (section 3); the backend admits at most half the pool (after the raise below) as open snapshots and refuses more with `LimitExceeded`; a running restore holds one data connection and has no slot of its own after PR 6; and each PostgreSQL job holds one dedicated claim connection outside the pool. `pool_size` sizes both pools, and the rule is `pool_size >= max(MIN_POOL_SIZE, 2 * max_concurrent_jobs + 10)`: twice the jobs, so the half-pool bound admits every job `max_concurrent_jobs` allows, plus ten for requests; 18 at the defaults, under the default `pool_size` of 20. It applies whether or not `[backup.store]` is configured, because legacy restores are jobs too. Startup raises a smaller `pool_size` to that value with a warning naming both numbers, as it already raises a `pool_size` under `MIN_POOL_SIZE` (10) today. On SQLite the snapshot slot (`backup_server_slot_count`, `pool_size / 2 - 1` capped at 16, shared by the in-catalog create and the snapshot open) stays the backend's admission bound, as PostgreSQL's half-pool slot does, and a server raises `[storage.sqlite] pool_size` to `2 * (max_concurrent_jobs + 1)` (at most 34) with a warning naming both numbers so the slot refuses no job the permit admitted; 10 at the defaults, the default pool; above 16 jobs the slot refuses the seventeenth. -A configuration error that needs no request to the store (a part size, prefix or key-length problem, an insecure endpoint, a root that fails the owner-only rule or whose ancestors others can write to, a filesystem root that is or contains the database directory, a spool rule) is a startup error naming the path or bucket, except for the defaulted SQLite root above while the catalog has no rows for it, where it and a failure of the root's identity check are a warning that leaves the store unconfigured. A filesystem root's `validate` also writes a probe file, a request to the store: an I/O failure there or in resolving the root is a store condition that degrades the store (`failed validation`), as the same condition one reconciler cycle later does, not a startup error. A filesystem store's `validate` and identity check run before the listener, bounded by a timeout, so a mount that stops answering after the root is opened leaves the server starting with the store `checking` (a root that does not answer at startup holds the server before the listener, as the database directory would); an S3 store's are requests to the endpoint and run after the listener is up, in the reconciler's first cycle, with the store `checking` until the first result reports, so a stalled endpoint never holds the data plane (revision item 28). Every later check runs under the same bound (5 s for a filesystem root; 30 s for a bucket, under the SDK's per-attempt timeout so no retry outlives the check) and against the shutdown token: a store that has stopped answering costs a reconciler cycle its bound and nothing more, the cycle's other duties run, the check is recorded as a transport failure, and neither the drain at shutdown nor the process's exit waits on it (a filesystem root's abandoned call stays parked on a thread until the mount answers, no later check starts while it is, and the binary ends its runtime with a 5 s bound). A transport failure (no region, no credentials, an unreachable endpoint, a check that did not finish within its bound), a missing or denied bucket found after the listener, and a failure of section 4's identity check or manifest probe, do not stop startup, because a store problem must not take the data plane out of rotation (section 9): the failure is logged with its cause, the store component is reported unhealthy with the problem's category as its reason, the operations that touch the store refuse with a ValidationException that names the store's kind and the category and not its root (RestoreTableFromBackup and DeleteBackup of an off-host backup, and CreateBackup once it writes to the store in PR 8; Describe and List, legacy backups and this release's in-catalog CreateBackup are unaffected), the reconciler acts on no row of the store (its legacy duty, which touches no store, still runs) and resumes no restore, and the check runs again each reconciler cycle until it passes. +A configuration error that needs no request to the store (a part size, prefix or key-length problem, an insecure endpoint, a root that fails the owner-only rule or whose ancestors others can write to, a filesystem root that is or contains the database directory, a spool rule) is a startup error naming the path or bucket, except for the defaulted SQLite root above while the catalog has no rows for it, where it and a failure of the root's identity check are a warning that leaves the store unconfigured. A filesystem root's `validate` also writes a probe file, a request to the store: an I/O failure there or in resolving the root is a store condition that degrades the store (`failed validation`), as the same condition one reconciler cycle later does, not a startup error. A filesystem store's `validate` and identity check run before the listener, bounded by a timeout, so a mount that stops answering after the root is opened leaves the server starting with the store `checking` (a root that does not answer at startup holds the server before the listener, as the database directory would); an S3 store's are requests to the endpoint and run after the listener is up, in the reconciler's first cycle, with the store `checking` until the first result reports, so a stalled endpoint never holds the data plane (revision item 28). Every later check runs under the same bound (5 s for a filesystem root; 30 s for a bucket, under the SDK's per-attempt timeout so no retry outlives the check) and against the shutdown token: a store that has stopped answering costs a reconciler cycle its bound and nothing more, the cycle's other duties run, the check is recorded as a transport failure, and neither the drain at shutdown nor the process's exit waits on it (a filesystem root's abandoned call stays parked on a thread until the mount answers, no later check starts while it is, and the binary ends its runtime with a 5 s bound). A transport failure (no region, no credentials, an unreachable endpoint, a check that did not finish within its bound), a missing or denied bucket found after the listener, and a failure of section 4's identity check or manifest probe, do not stop startup, because a store problem must not take the data plane out of rotation (section 9): the failure is logged with its cause, the store component is reported unhealthy with the problem's category as its reason, the operations that touch the store refuse with a ValidationException that names the store's kind and the category and not its root (CreateBackup on PostgreSQL, SQLite and MongoDB, and RestoreTableFromBackup and DeleteBackup of an off-host backup; Describe and List, legacy backups and the in-catalog CreateBackup are unaffected), the reconciler acts on no row of the store (its legacy duty, which touches no store, still runs) and resumes no restore, and the check runs again each reconciler cycle until it passes. Authority: the `[backup]` struct in `crates/config` (PR 6 adds it with the job keys; PR 7 adds `store` and its validation) and `BackupStoreConfig` in `crates/backup-store`; defaults above are the ones they encode. @@ -187,7 +187,7 @@ Authority: the `[backup]` struct in `crates/config` (PR 6 adds it with the job k ### 9. Observability -Metrics: `backup_jobs_total{outcome=succeeded|failed|cancelled}`, `backup_job_duration_seconds`, `backup_bytes_uploaded_total`, `restore_jobs_total{outcome}`, `restore_job_duration_seconds`, `backup_snapshot_age_seconds` (gauge while a job runs), `backup_store_errors_total{op}`. Logs: one info line at job start and completion with ARN, item count, bytes, duration; one error line per failure with the reason. The `/health` readiness work from the operations report should include store reachability as a component, not as a hard failure (a backup store outage should not take the data plane out of rotation). `/health` is a static 200 today; the component model is new work in the configuration PR, and a transport failure of the store, or a failure of its identity check or manifest probe (section 4), marks the component unhealthy rather than stopping the server (section 7). Metric names are variants of the closed `MetricName` enum in `crates/core/src/metrics`; the existing `WorkerLastSuccess`, `WorkerCycleLatency` and `WorkerErrorCount` are emitted by the backup worker and the reconciler like the other workers. +Metrics: `backup_jobs_total{outcome=succeeded|failed|cancelled}`, `backup_job_duration_seconds`, `backup_bytes_uploaded_total`, `restore_jobs_total{outcome}`, `restore_job_duration_seconds`, `backup_snapshot_age_seconds` (gauge while a job runs), `backup_store_errors_total{op}`. Logs: one info line at job start and completion with ARN, item count, bytes, duration; one error line per failure with the reason. The `/health` readiness work from the operations report should include store reachability as a component, not as a hard failure (a backup store outage should not take the data plane out of rotation). `/health` is a static 200 today; the component model is new work in the configuration PR, and a transport failure of the store, or a failure of its identity check or manifest probe (section 4), marks the component unhealthy rather than stopping the server (section 7). Metric names are variants of the closed `MetricName` enum in `crates/core/src/metrics`, and `/metrics` emits them under those names as JSON (`render_metrics`, `docs/design/06-component-server.md`: the endpoint is not Prometheus exposition format; the deployment guide tells operators to convert); the snake-case spellings above are the names a converter would give them, not a rendering the server does. PR 8 adds `BackupJobs` (one count per job end, `operation` = the outcome, `succeeded`, `failed` or `cancelled`), `BackupJobDuration` (the job's duration in microseconds, like every duration the collector records, with the same `operation`), `BackupBytesUploaded` (bytes put to the store), `BackupSnapshotAge` (seconds since the snapshot instant, set at each confirmed heartbeat) and `BackupStoreErrors` (`operation` = the store call, `put` or `delete_prefix`), with `QuerySource::BackupJob`; `BackupBytesUploaded` is recorded for a job that ended AVAILABLE, and `WorkerErrorCount` under `BackupJob` for a FAILED job and not for one the shutdown cancelled (which records `BackupJobs` with `cancelled` and neither worker metric); the restore variants are PR 9a's. The existing `WorkerLastSuccess`, `WorkerCycleLatency` and `WorkerErrorCount` are emitted by the reconciler like the other workers and by the backup job at its end under `BackupJob`. ### 10. Documentation @@ -205,15 +205,19 @@ Which document and which row each PR changes; what each row says is in the ownin | differences doc: restore-interruption row | PR 8 | | differences doc: Import and Export rows | PR 13 | | differences doc: new row for CreateBackup of a table holding an `N` value #618 let in | PR 8 | -| limits doc: backup rows | PR 7 (the knobs) and PR 8 (the line cap and file size) | +| differences doc: where-backups-live and backup-creation rows (the store-backed, asynchronous create) | PR 8 | +| limits doc: backup rows | PR 7 (the knobs) and PR 8 (the line cap and file size, the SQLite pool raise, the backup-contents row, the "Backup store" row rewritten: CreateBackup writes to the store and is refused while the store is unhealthy) | | differences and limits docs: the override rows stay | PR 9a confirms; no edit | | admin guide: database names | PR 11 | +| admin guide: the `[backup]` rows (`data_file_target_bytes`, `heartbeat_interval_secs`, `stale_job_after_secs`, `failed_row_retention_secs`, `reconcile_interval_secs`, `max_concurrent_jobs` with the memory figures) and the `[backup.store]` section (the store-backed create, the job's two log lines, the unhealthy-store refusal) | PR 8 | +| deployment guide: the service unit's `MALLOC_ARENA_MAX=1` line | PR 8 | +| `docs/design/README.md` row 15b (the measurement record) | PR 8 | | usage guide: "backups not supported" line | PR 8 | -| upgrade manual: stop pre-PR-6 servers before `extenddb migrate`; a restore in flight at the stop; the rollback paragraph | PR 6 | +| upgrade manual: stop pre-PR-6 servers before `extenddb migrate`; a restore in flight at the stop; the rollback paragraph | PR 6 (and PR 8 for the store-backed backups a restored catalog no longer lists) | | upgrade manual: CreateBackup without `[backup.store]`, `kind = "disabled"`, the SQLite default root, shared store roots and `--adopt-store` | PR 7 (and PR 10 for `--adopt-store`) | | upgrade manual: the SQLite serve lock; `migrate-legacy` and its refusals | PR 10 | | upgrade manual: PR 14's migration refusal, `--force-drop-legacy-backups`, CreateBackup without a store after PR 14 | PR 14 | -| upgrade manual: the #618 regression on the off-host path and the way out | PR 8 | +| upgrade manual: the #618 regression on the off-host path and the way out; the `[backup.store]` paragraph's asynchronous create and the restore of a store-backed backup deferred to PR 9a | PR 8 | | `docs/troubleshooting.md` DeleteTable entry and admin guide `control_plane_delay_seconds` row | #395 (in its diff) | | ADR-0019 and the Cassandra RFC's "Backup and restore" section marked as superseded; the admin and upgrade manuals' backup sections | PR 11 | | `docs/design/README.md` line 22 | PR 2 | @@ -249,7 +253,7 @@ Documentation: the differences doc's PITR row, rewritten as section 6 says. Goal: the catalog index of section 4 on every backend and the engine-side Describe, List and Delete over it, with the legacy reader beside it. Its line names the interfaces and carries the acceptance: - PR 6: catalog index per section 4, the `BackupCatalog` trait and `JobClaim` of section 3 implemented by every backend, the `backup_index` row (a new table or collection on PostgreSQL, SQLite and MongoDB; the reused legacy tables with `store_kind` set, plus `backup_index_by_status`, on Cassandra) and the restore job fields on #384's registration record, with the migration's backfill of `status`, `heartbeat_at` (the epoch, not NULL) and a sentinel `owner_generation` on the registrations that exist (migration id 5 on PostgreSQL and SQLite; Cassandra V006 and MongoDB's step `002_backup_catalog_index`, both writing catalog 0.0.5; - the MongoDB `backup_index` collection and `tables` fields, the versioned migration step MongoDB lacks today and its bump from `0.0.2` to that version (section 4; the expected version is copied three times, twice in `bootstrapper.rs` and once in `operations.rs`, and this PR collapses it to one constant); `RESTORE_LOCK_SEED` made `pub` with `#[doc(hidden)]` so its copy in `tests/backup_restore.rs` binds to it; the `store_id` column, the `backup_maintenance` record and Cassandra's write order across its three backup tables), the `[backup]` struct in `crates/config` with the job keys of section 7 (everything but `store`, which PR 7 adds) and the check that `stale_job_after_secs` is at least three times the effective heartbeat period, `max(heartbeat_interval_secs, JOB_HEARTBEAT_INTERVAL)` (section 7), since this PR's reconciler and permit read them, `max_concurrent_jobs` (default 4, at most 1024) replacing the per-backend restore slot semaphores (`restore_server_slot_count` on PostgreSQL and MongoDB removed; SQLite's `backup_server_slot_count` kept for snapshot opens until PR 8, which folds it into `max_concurrent_jobs`; the legacy create tries it without waiting and answers the one `LimitExceededException` text, `limit_exceeded_message`) with the request order of 5.1 step 1 and 5.3 step 1, the permit last, and the reconciler period and maintenance-claim check of section 4, engine-side Describe, List (the handler's existing paging, filtering and cursor exceptions kept, over the catalog index merged with the legacy reader through `list_account_backup_rows`, and a table ARN accepted in `TableName` as main's CreateBackup accepts it), Delete (BackupInUse on a CREATING row and while a restore target registered from the backup is CREATING; DELETING ordering; a ValidationException naming no root for a row of another store) over the new rows, `LegacyBackupReader::create` kept on every backend as the CreateBackup path while no store is configured, with its deprecation warning, and on Cassandra whatever the store configuration, with the `LegacyBackupReader` trait and fallback, the legacy restore registering through the new job fields and, with a dead owner, marked FAILED and its target removed through `remove_restore_target` (section 3); the engine adopts `BackupArn`, addresses every backup row by account and backup id (`BackupKey`; `backup_index` keyed by (`account_id`, `backup_id`), no `store_key_prefix` column) and every restore target by account, name and table id, and derives the API's source-table shapes from `source_table_details_json`; the `catalog_id`, written at init and by the migration for an existing catalog; DeleteBackup's in-use check and DELETING write in one transaction under the row's lock (`start_backup_delete`), and on Cassandra `active_restores` serving new rows; the legacy create taking a `max_concurrent_jobs` permit, and Cassandra's legacy create heartbeated so the reconciler removes a dead one's CREATING row; the job row's `kind` and store columns; #384's MongoDB claim markers, Cassandra `restore_owner`, PostgreSQL `RestoreOwner` and their startup sweeps are replaced by the generation column and the reconciler; `RESTORE_LOCK_SEED` becomes the shared seed. Acceptance: all existing backup tests pass unchanged against backups created by the PR 0 binary (run the binary, not the unit), the existing paging tests and `test_restore_into_a_creating_target_and_a_second_restore_of_its_backup_are_refused` included (the vector tests change in PR 8 and 9a, not here); a catalog at #384's version holding a CREATING restore target left by SIGKILL migrates with that registration's `heartbeat_at` at the epoch, and the first reconciler pass removes the target, run on PostgreSQL and on MongoDB; ListBackups with a table ARN in `TableName` returns what the table name returns; DescribeBackup of an ARN of account B that carries the id of account A's backup returns BackupNotFound; a DeleteBackup and a legacy restore of one backup serialize, main's `delete_backup_winning_before_restore_leaves_no_target` and `restore_registration_winning_before_delete_returns_backup_in_use` pass unchanged for the legacy path, the `backup_index` row lock observed on PostgreSQL by a two-transaction test of `start_backup_delete` (one holding the row `FOR UPDATE` while it registers a CREATING target, the delete waiting and then answering BackupInUse); the same two orderings run against the engine's DeleteBackup and RestoreTableFromBackup over an off-host backup (a `backup_index` row), gated on that lock, a DeleteBackup that wins leaving the registration BackupNotFound and no target and a registration that wins leaving DeleteBackup BackupInUse, deferred to PR 9a, which adds the first restore of a `backup_index` row (on this PR no restore of one exists); the reconciler's FAILED-target duty of 4(e) on a FAILED record whose target is still CREATING, listed once stale and removed; a server built before this PR, started against the migrated catalog, is the mixed-version case section 10 forbids and is not tested. Depends on PR 4 (`BackupArn`) and PR 5 (`SnapshotInfo`). + the MongoDB `backup_index` collection and `tables` fields, the versioned migration step MongoDB lacks today and its bump from `0.0.2` to that version (section 4; the expected version is copied three times, twice in `bootstrapper.rs` and once in `operations.rs`, and this PR collapses it to one constant); `RESTORE_LOCK_SEED` made `pub` with `#[doc(hidden)]` so its copy in `tests/backup_restore.rs` binds to it; the `store_id` column, the `backup_maintenance` record and Cassandra's write order across its three backup tables), the `[backup]` struct in `crates/config` with the job keys of section 7 (everything but `store`, which PR 7 adds) and the check that `stale_job_after_secs` is at least three times the effective heartbeat period, `max(heartbeat_interval_secs, JOB_HEARTBEAT_INTERVAL)` (section 7), since this PR's reconciler and permit read them, `max_concurrent_jobs` (default 4, at most 1024) replacing the per-backend restore slot semaphores (`restore_server_slot_count` on PostgreSQL and MongoDB removed; SQLite's `backup_server_slot_count` kept for snapshot opens, which PR 8 keeps as the backend's admission bound and raises the pool for (section 7); the legacy create tries it without waiting and answers the one `LimitExceededException` text, `limit_exceeded_message`) with the request order of 5.1 step 1 and 5.3 step 1, the permit last, and the reconciler period and maintenance-claim check of section 4, engine-side Describe, List (the handler's existing paging, filtering and cursor exceptions kept, over the catalog index merged with the legacy reader through `list_account_backup_rows`, and a table ARN accepted in `TableName` as main's CreateBackup accepts it), Delete (BackupInUse on a CREATING row and while a restore target registered from the backup is CREATING; DELETING ordering; a ValidationException naming no root for a row of another store) over the new rows, `LegacyBackupReader::create` kept on every backend as the CreateBackup path while no store is configured, with its deprecation warning, and on Cassandra whatever the store configuration, with the `LegacyBackupReader` trait and fallback, the legacy restore registering through the new job fields and, with a dead owner, marked FAILED and its target removed through `remove_restore_target` (section 3); the engine adopts `BackupArn`, addresses every backup row by account and backup id (`BackupKey`; `backup_index` keyed by (`account_id`, `backup_id`), no `store_key_prefix` column) and every restore target by account, name and table id, and derives the API's source-table shapes from `source_table_details_json`; the `catalog_id`, written at init and by the migration for an existing catalog; DeleteBackup's in-use check and DELETING write in one transaction under the row's lock (`start_backup_delete`), and on Cassandra `active_restores` serving new rows; the legacy create taking a `max_concurrent_jobs` permit, and Cassandra's legacy create heartbeated so the reconciler removes a dead one's CREATING row; the job row's `kind` and store columns; #384's MongoDB claim markers, Cassandra `restore_owner`, PostgreSQL `RestoreOwner` and their startup sweeps are replaced by the generation column and the reconciler; `RESTORE_LOCK_SEED` becomes the shared seed. Acceptance: all existing backup tests pass unchanged against backups created by the PR 0 binary (run the binary, not the unit), the existing paging tests and `test_restore_into_a_creating_target_and_a_second_restore_of_its_backup_are_refused` included (the vector tests change in PR 8 and 9a, not here); a catalog at #384's version holding a CREATING restore target left by SIGKILL migrates with that registration's `heartbeat_at` at the epoch, and the first reconciler pass removes the target, run on PostgreSQL and on MongoDB; ListBackups with a table ARN in `TableName` returns what the table name returns; DescribeBackup of an ARN of account B that carries the id of account A's backup returns BackupNotFound; a DeleteBackup and a legacy restore of one backup serialize, main's `delete_backup_winning_before_restore_leaves_no_target` and `restore_registration_winning_before_delete_returns_backup_in_use` pass unchanged for the legacy path, the `backup_index` row lock observed on PostgreSQL by a two-transaction test of `start_backup_delete` (one holding the row `FOR UPDATE` while it registers a CREATING target, the delete waiting and then answering BackupInUse); the same two orderings run against the engine's DeleteBackup and RestoreTableFromBackup over an off-host backup (a `backup_index` row), gated on that lock, a DeleteBackup that wins leaving the registration BackupNotFound and no target and a registration that wins leaving DeleteBackup BackupInUse, deferred to PR 9a, which adds the first restore of a `backup_index` row (on this PR no restore of one exists); the reconciler's FAILED-target duty of 4(e) on a FAILED record whose target is still CREATING, listed once stale and removed; a server built before this PR, started against the migrated catalog, is the mixed-version case section 10 forbids and is not tested. Depends on PR 4 (`BackupArn`) and PR 5 (`SnapshotInfo`). Invariants honoured: 4(a) and (b) (the two records and their transitions), 4(d) (every row by `BackupKey`, every target by `RestoreTargetKey`), 4(e) (the claim), 4(f) (the maintenance claim its reconciler period checks), 5.2 (DeleteBackup's lock and ordering against a registration), and the Goals' unchanged request and response shapes. @@ -350,20 +354,20 @@ Documentation rows it corrects: limits doc backup rows (the knobs); upgrade manu ### PR 8: CreateBackup job, worker and reconciler Goal: the CreateBackup job of 5.1 on the filesystem store, with the reconciler and its full duty list, the byte budget on the snapshot streams and the item validator. Its line names the interfaces and carries the acceptance: -- PR 8: CreateBackup job and worker per 5.1 on the filesystem store: snapshot in the request with the per-backend claim, generation-fenced catalog writes, pipelined read/compress/upload, timer heartbeat through the claim connection with abort on any unconfirmed heartbeat, the failure path's delete gated on a matched FAILED write, every job status write conditioned on `status = CREATING` and an ambiguous AVAILABLE update re-read, a table name over 255 bytes refused, cancellation runs the failure path, id minted from the snapshot instant, `LimitExceeded` over `max_concurrent_jobs` and while the maintenance claim is held, the byte budget `SNAPSHOT_FETCH_BYTES` on the PostgreSQL and SQLite snapshot streams, enforced in each page's query, PostgreSQL's cursor replaced by the keyset page under the same snapshot, with the key columns returned under qualified aliases, an outer `ORDER BY` and `n` starting at 16 and sized from the previous page's average row (section 3, 7), the item validator `validate_restorable_item` in `extenddb-core` (one walk, each `N` parsed once, an error naming the item's key and the attribute path), which PR 9a's restore binds, run on every item, with the backup failed after the rest of the snapshot is validated, every refused item logged up to the cap and the count and the first item in the FAILED reason (5.1 step 4, #618), #394's `KeyRangeTracker` and `BackupDocuments::build` in the writer, the server's shutdown awaiting each job's failure path, the PostgreSQL claim connection's keepalive settings, the data-file close rule and the knob's minimum, the SQLite runtime dead-owner test (a slot no task holds), `refuse_backup_of_vector_table` taken out of the off-host path and kept in front of `LegacyBackupReader::create`, `create_backup_refuses_a_table_with_vector_indexes` kept for that path, an off-host round trip of a vector table added and `test_create_backup_of_a_vector_table_is_refused` expecting the outcome of the configured path, read from `EXTENDDB_TEST_BACKUP_STORE`, which the harness exports and CI sets (5.1 step 1), FAILED tombstones, metrics. Acceptance: backup of a 100k-item composite-key table with GSIs on each backend; SIGKILL mid-upload leaves no prefix with a manifest and the row resolves after restart (binary-level); SIGKILL between the manifest put and the AVAILABLE update resolves to AVAILABLE after restart and the prefix survives; SIGTERM mid-upload leaves a FAILED row, an empty prefix and no multipart upload; failure deletes the prefix; a second CreateBackup on the same instance while one runs returns `LimitExceededException`; a 1M-item run records duration and peak RSS, and RSS must not grow from the 100k run to the 1M run (on every backend but `sqlite-memory`, whose open copies the table); the memory test with #394's worst-expansion items stays within section 7's formula, with the worst lines placed after a page of small rows, and no page holds more than the budget plus one row, counted in the stream by a check in `extenddb_storage::snapshot_conformance` that the PostgreSQL and SQLite tests both run; the rows read past each PostgreSQL page's cut stay under its `n` and are recorded beside the RSS figure; `EXPLAIN` of the test table's page query shows an index scan; one cell for each write path #394's `key_order.rs` doc names (BatchWriteItem, TransactWriteItems, UpdateItem on an absent item, ImportTable): a table holding `1E126` or `abc` written through that path (in a non-key attribute and, on a key-ordered table, in the range key; for UpdateItem in the key it creates) gives a FAILED row and no manifest on each backend; with three such items the job validates past the first, logs all three with key, attribute path and reason, and the FAILED reason gives the count and the first and names #618; an item refused for another rule (over `max_item_size_bytes`) gives a reason that does not name #618; the vector wire test fails when `EXTENDDB_TEST_BACKUP_STORE` is unset; without `[backup.store]` CreateBackup of a vector table is refused as on main; a job task that panics on SQLite leaves a row the reconciler resolves without a restart. The reconciler ships in the same PR, with the full duty list of section 4, the S3 leg in CI, and memory tests with incompressible items (64 MiB of random binary attributes, which gzip cannot shrink) and with the worst-expansion shapes of section 7; RSS must match the formula in section 7 in each, proving the encoder streams into the upload and the byte budget and channel bound hold at the largest line. Depends on PR 4, 5, 6, 7. +- PR 8: CreateBackup job and worker per 5.1 on the filesystem store: snapshot in the request with the per-backend claim, generation-fenced catalog writes, pipelined read/compress/upload, timer heartbeat through the claim connection with abort on any unconfirmed heartbeat, the failure path's delete gated on a matched FAILED write, every job status write conditioned on `status = CREATING` and an ambiguous AVAILABLE update re-read, a table name over 255 bytes refused, cancellation runs the failure path, id minted from the snapshot instant, `LimitExceeded` over `max_concurrent_jobs` and while the maintenance claim is held, the byte budget `SNAPSHOT_FETCH_BYTES` on the PostgreSQL and SQLite snapshot streams, in the page query on PostgreSQL and while the rows are pulled on SQLite, PostgreSQL's cursor replaced by the three-layer keyset page under the same snapshot (section 3, 7), the item validator `validate_restorable_item` in `extenddb-core` (one walk, each `N` parsed once, an error naming the item's key and the attribute path), which PR 9a's restore binds, run on every item, with the backup failed after the rest of the snapshot is validated, every refused item logged up to the cap and the count and the first item in the FAILED reason (5.1 step 4, #618), #394's `KeyRangeTracker` and `BackupDocuments::build` in the writer, the server's shutdown awaiting each job's failure path, the PostgreSQL claim connection's keepalive settings, the data-file close rule and the knob's minimum, the SQLite runtime dead-owner test (a slot no task holds), `refuse_backup_of_vector_table` taken out of the off-host path and kept in front of `LegacyBackupReader::create`, `create_backup_refuses_a_table_with_vector_indexes` kept for that path, the create half of an off-host round trip of a vector table added (CreateBackup succeeds, DescribeBackup reaches AVAILABLE with the table's features, ListBackups lists it, and the restore answers the ValidationException naming the store until PR 9a adds the restore half) and `test_create_backup_of_a_vector_table_is_refused` expecting the outcome of the configured path, read from `EXTENDDB_TEST_BACKUP_STORE`, which every CI pytest job that runs the suite sets (`filesystem` on the PostgreSQL job, which appends a filesystem store after `init`, and on the SQLite job, the defaulted root; `s3` on the SQLite job against moto; `none` on the MongoDB and Cassandra jobs) and `devtools/run-tests --pytest` refuses to run without (5.1 step 1); the restore cells of `tests/test_backup_restore_fidelity.py` skip on a server with a store, naming PR 9a, until its restore job lands; FAILED tombstones, metrics. Acceptance: backup of a 100k-item composite-key table with GSIs on each backend with a `SnapshotEngine` (PostgreSQL, SQLite, MongoDB; Cassandra keeps the legacy create and its existing cells); SIGKILL mid-upload leaves no prefix with a manifest and the row resolves after restart (binary-level); SIGKILL between the manifest put and the AVAILABLE update resolves to AVAILABLE after restart and the prefix survives; SIGTERM mid-upload leaves a FAILED row, an empty prefix and no multipart upload; failure deletes the prefix; a second CreateBackup on the same instance while one runs returns `LimitExceededException`; a 1M-item run records duration and peak RSS, and the 1M run's peak must be within 112 MiB of the 100k run's (the budget, the line cap and the channels; on every backend but `sqlite-memory`, whose open copies the table; on MongoDB a 500k run stands in for the 1M run, its load alone taking 27 minutes through the wire); the memory test with #394's worst-expansion items, with the worst lines placed after a page of small rows, has its peak over idle under 1.2 times the sum of section 7's terms at the measured row (the decoded rows in the page and the pipeline, one row's parse, the line buffer, the chunks and SQLite's run-ahead) with `MALLOC_ARENA_MAX=1`, at 12, 24 and 48 worst rows on one SQLite binary and at 12 rows on PostgreSQL, and its peak with glibc's default arenas recorded beside it at the same counts, and no page holds more than the budget plus one row, counted in the stream by a check in `extenddb_storage::snapshot_conformance` that the PostgreSQL and SQLite tests both run; the rows a PostgreSQL page reads past its cut are at most `n - 1`, measured by `EXPLAIN (ANALYZE, BUFFERS)` on a 1,000-small-then-60-worst table (4 read, 1 past the cut at the floor) and pinned by `the_row_bound_remembers_large_rows_among_small_ones` (201 over 53 round trips on a 300-row table with nine 27 MB rows); `EXPLAIN` of the page query on the six key shapes a table can have (hash only, `S`, `N` and `B` range keys, a multi-part range key, a legacy table on the database collation) shows `Limit` over one `Index Scan` on the primary key and no other scan of the relation where the planner's estimate is above `n`, and the bitmap scan with a `Sort` near the end of a small table (PostgreSQL 16), both pinned by a test and stated in the module doc; one cell for each write path #394's `key_order.rs` doc names (BatchWriteItem, TransactWriteItems, UpdateItem on an absent item, ImportTable): a table holding `1E126` or `abc` written through that path (in a non-key attribute and, on a key-ordered table, in the range key; for UpdateItem in the key it creates) gives a FAILED row and no manifest on PostgreSQL and SQLite (the two backends `tests/test_backup_job.py`'s deployment fixture starts; MongoDB's store-backed create is covered by the engine tests and the 100k run, and gets its cell when the fixture gains a MongoDB deployment) (an `abc` range key is refused by the number column on PostgreSQL and SQLite, so the range-key cell carries `1E126`; UpdateItem validates the values its expression sets, so its cell carries the bad value in the key alone); with three such items the job validates past the first, logs all three with key, attribute path and reason, and the FAILED reason gives the count and the first and names #618; an item refused for another rule (over `max_item_size_bytes`) gives a reason that does not name #618; the vector wire test fails when `EXTENDDB_TEST_BACKUP_STORE` is unset; without `[backup.store]` CreateBackup of a vector table is refused as on main; a job task that panics on SQLite leaves a row the reconciler resolves without a restart. The reconciler ships in the same PR, with the full duty list of section 4, the S3 leg in CI, and memory tests with incompressible items (64 MiB of random binary attributes, which gzip cannot shrink) and with the worst-expansion shapes of section 7; RSS is recorded against the terms of section 7 in each, proving the encoder streams into the upload (the incompressible run's 68 MB file passes through a 4 MiB chunk window) and that the byte budget and the channel bound hold at the largest line (under `MALLOC_ARENA_MAX=1` the worst-expansion run's peak does not grow with the number of worst rows, 452, 480 and 455 MiB over idle at 12, 24 and 48 rows on the final SQLite binary; with glibc's default arenas it does, 707, 700 and 1,281 MiB in the same series, the allocator's retention and not the pipeline's terms, which the one-arena series shows do not grow). Depends on PR 4, 5, 6, 7. Invariants honoured: 4(a) (every transition of the backup row a job or the reconciler performs), 4(c) (the reconciler acts only on rows of the configured store and on none while it is unhealthy), 4(e) (the claim, the fencing of every status write, the FAILED-write gate before any delete), 4(f) (the duties that run under a maintenance claim), 5.1 as written, section 2's three store facts, section 7's memory bound and pool rule, and the Drawbacks' #618 regression. -Interfaces it adds: `extenddb_core::validation::validate_restorable_item` (5.1 step 4), `extenddb_storage::snapshot::paging::SNAPSHOT_FETCH_BYTES`, the worker and the reconciler in `crates/server/src/workers.rs`, the metrics of section 9; the reconciler claims through PR 6's `BackupCatalog::claim_backup`. The snapshot and the claim connection are handed to the backup worker over an in-process channel; the worker is a task in `crates/server/src/workers.rs`, spawned with the server's cancellation token and retained handle like the existing workers. It is needed because four write paths store an `N` value such as `1E126` or `abc` unvalidated on main (#618): BatchWriteItem, TransactWriteItems, UpdateItem on an absent item (which creates the item from its `Key`) and ImportTable, the list #394's `key_order.rs` doc gives. So that one run finds every offending item, the first refusal stops encoding and uploading but not validation: the job reads and checks the rest of the snapshot, logs each refused item (its key, attribute path and reason, with the backup ARN) up to 1000 items and counts the rest, and then fails the backup (step 5) with a FAILED reason that gives the number of refused items and the first one, naming #618 only when that item's refusal is an `N` value; so no backup that no restore can load is reported AVAILABLE. The byte budget (section 7 states the bound): PR 8 replaces #395's cursor on PostgreSQL with pages bounded by bytes in the query, each page a keyset `SELECT` in the same `REPEATABLE READ` transaction, ordered by the data table's primary key and starting after the last key of the page before, with a running byte sum that cuts the page at `SNAPSHOT_FETCH_BYTES` (64 MiB, a constant beside `DEFAULT_FETCH_BATCH`, not a configuration key in v1) and always returns at least one row; the snapshot, the lock and which tables stream `key_ordered` are unchanged. On SQLite a page stops at `DEFAULT_FETCH_BATCH` rows or after the row that reaches `SNAPSHOT_FETCH_BYTES` of item text, whichever comes first, and holds at least one row. The cut rule exists in SQL on PostgreSQL and in Rust on SQLite, so the check that no page holds more than the budget plus one row lives in `extenddb_storage::snapshot_conformance`, which both run; a MongoDB batch is already bounded by the server's 16 MiB reply. -Tests that change (5.1 step 1): `create_backup_refuses_a_table_with_vector_indexes` (`crates/engine/src/backup/tests.rs`) stays for that path, PR 8 adds an off-host round trip of a vector table beside it, and `test_create_backup_of_a_vector_table_is_refused` (`tests/test_backup_restore_fidelity.py`) expects the refusal or the round trip according to the server's configured path, which it reads from `EXTENDDB_TEST_BACKUP_STORE=`, a variable the test harness exports and every CI job that runs the suite sets: it derives its branch from the variable, prints which branch ran, and fails if the variable is unset, rather than accepting either outcome; the restore's `ensure_restorable` refusal of every vector backup narrows to "the target has no vector support" only in PR 9a, which rewrites `vector_index_backups_are_refused_in_the_engine` to that case; the frozen legacy reader keeps #384's behaviour for a legacy vector backup (section 3), and section 10 corrects the differences doc's vector row. +Interfaces it adds: `extenddb_core::validation::validate_restorable_item(item, &RestorableKeySchema, &RestorableLimits)` (5.1 step 4; the key argument is a schema every table has, multi-part keys included, each key attribute checked with the format's own `check_key_value`; `RestorableLimits` is built from `&LimitsConfig`: the item size, the hash and range key sizes and the attribute name size), `RestorableItemReason::{AttributeNameTooLong, LineTooLong}` (the latter carrying `{ cap }`), `SHOWN_VALUE_BYTES` (4096, the bound on each text an error carries other than a key value) and `SHOWN_KEY_VALUE_BYTES` (12,288, the bound on a key value), `extenddb_core::limits::DEFAULT_MAX_ITEM_SIZE_BYTES` (the default item limit, the feeder's threshold), `extenddb_core::backup_format::encode_item_line_len_bounded(item, cap)` (the line's length up to `cap`, measured without a buffer), `extenddb_storage::snapshot::paging::SNAPSHOT_FETCH_BYTES` with `fetch_bytes()`, `record_page_bytes` and `record_rows_past_cut` (counters under `test-hooks`), `extenddb_storage::snapshot::source_table_schema` (5.1 step 2), the job (`crates/engine/src/backup/job.rs` with `job/{writer,heartbeat,failure}.rs`) and the reconciler's new duties (`crates/engine/src/backup/reconciler.rs`) in the engine's backup module, the server handing the engine the shutdown token, the metrics collector, the job settings (`JobSettings`) and the manifest's bucket and key prefix through `BackupRuntime` and `ConfiguredStore`, the metrics of section 9, and on `BackupCatalog`: `claim_dead_backup(key, stale_after)` (the takeover of 4(e); `claim_backup` inserts and takes over nothing), `list_backup_rows_older_than(status, store, clock, age)` (`StaleClock::{Heartbeat, Completed}`, the backend's clock; a row with `completed_at` unset is never older on the completed clock), `sweep_restore_registrations` (Cassandra's `active_restores` sweep, a default empty body elsewhere) and `StatusFields.source_table_details_json` (the AVAILABLE write carries the manifest's `source_table`). The request's detached task opens the snapshot, claims the row, checks the maintenance claim and returns the response through a one-shot channel, then continues as the job with the same owned values, so nothing but the response crosses a channel between the request and the job (the pipeline's item and chunk channels and its blocking encoder are the job's own); the runtime's task tracker holds every job, and the server's shutdown cancels and awaits the jobs' failure paths (`BackupRuntime::drain_jobs`, a 30 s bound) before it drains its workers. It is needed because four write paths store an `N` value such as `1E126` or `abc` unvalidated on main (#618): BatchWriteItem, TransactWriteItems, UpdateItem on an absent item (which creates the item from its `Key`) and ImportTable, the list #394's `key_order.rs` doc gives. So that one run finds every offending item, the first refusal stops encoding and uploading but not validation: the job reads and checks the rest of the snapshot, logs each refused item (its key, attribute path and reason, with the backup ARN) up to 1000 items and counts the rest, and then fails the backup (step 5) with a FAILED reason that gives the number of refused items and the first one, naming #618 only when that item's refusal is an `N` value; so no backup that no restore can load is reported AVAILABLE. The byte budget (section 7 states the bound): PR 8 replaces #395's cursor on PostgreSQL with pages bounded by bytes in the query, each page a keyset `SELECT` in the same `REPEATABLE READ` transaction, ordered by the data table's primary key and starting after the last key of the page before, in three layers (the innermost takes `n` rows after the key under `LIMIT`, since a window function runs before `LIMIT`; the middle adds the running byte sum and the rows' count; the outer keeps the rows whose predecessors' bytes are under `SNAPSHOT_FETCH_BYTES` (64 MiB, a constant beside `DEFAULT_FETCH_BATCH`, not a configuration key in v1) and orders them), so a page holds at least one row and at most the budget plus one, and the stream ends when the inner query returned fewer rows than asked and the cut dropped none of them; `n` starts at 16 and is then the budget divided by the largest row of the last pages holding at least a row bound's worth of rows (`RowBoundMemory`), never under 4 and never over `DEFAULT_FETCH_BATCH`; the rows read past the cut (at most `n - 1`) are read by the server and dropped by the outer layer, and each costs the server one serialization of its item text and two spills of it (the window functions buffer the `n` rows before the outer layer drops them), so a page costs the server `n` rows of text, not the rows it returns: sized from the average, a page of 984 small rows and 3 large ones asked the next page for 810 rows of a table that was then large rows only, 5.9 GB of server temp files and 18 to 44 s for the three rows it returned; sized from the largest row it asks for 4 (208 MiB, 1.1 s); the over-read that remains is the second page, and then the page that follows a row bound's worth of small rows, up to `n` rows of up to the line cap, at most once per row bound's worth of small rows; the key columns come back under qualified aliases so the next cursor is the row's own typed key, and the outer `ORDER BY` states a guarantee the executor gives anyway (a mutant without it passes every behavioural test, and the query text pins it); the snapshot, the lock and which tables stream `key_ordered` are unchanged. On SQLite a page stops at `DEFAULT_FETCH_BATCH` rows or after the row that reaches `SNAPSHOT_FETCH_BYTES` of item text, whichever comes first, and holds at least one row; the rows are pulled one at a time, so the stream pulls none past the cut, while the driver's worker thread reads at most five rows ahead of the pull (sqlx's `row_buffer_size`, set on the pool to four from its default of 50, which read every row of a 24-row worst-shape table ahead of the first page, plus the row the worker holds while its `send` waits; 160 MiB at the line cap), and the stream ends only when a page is short of the row bound and no cut happened. The cut rule exists in SQL on PostgreSQL and in Rust on SQLite, so the check that no page holds more than the budget plus one row lives in `extenddb_storage::snapshot_conformance`, which both run; a MongoDB batch is already bounded by the server's 16 MiB reply. +Tests that change (5.1 step 1): `create_backup_refuses_a_table_with_vector_indexes` (`crates/engine/src/backup/tests.rs`) stays for that path, PR 8 adds the create half of an off-host round trip of a vector table beside it (PR 9a adds the restore half), and `test_create_backup_of_a_vector_table_is_refused` (`tests/test_backup_restore_fidelity.py`) expects the refusal or the round trip according to the server's configured path, which it reads from `EXTENDDB_TEST_BACKUP_STORE=`, a variable the test harness exports and every CI job that runs a suite creating backups sets it (the Rust integration suite reads it the same way, `tests/rust/src/backup_restore.rs`): it derives its branch from the variable, prints which branch ran, and fails if the variable is unset, rather than accepting either outcome; the restore's `ensure_restorable` refusal of every vector backup narrows to "the target has no vector support" only in PR 9a, which rewrites `vector_index_backups_are_refused_in_the_engine` to that case; the frozen legacy reader keeps #384's behaviour for a legacy vector backup (section 3), and section 10 corrects the differences doc's vector row. -State transitions it owns: CREATING to AVAILABLE and to FAILED by the job (5.1 steps 4, 5) and by the reconciler (5.1 step 6); every deleting duty of 4(a). The reconciler, in one place: The reconciler runs at startup and then every `reconcile_interval_secs` (default 60, section 7); it acts on no row of the store while the configured store is unhealthy (the legacy duty touches no store and still runs), and it checks the maintenance claim first, and while one is held it runs only the duties that resolve the states a maintenance command waits on: it finishes DELETING rows of the configured store, resolves CREATING backup rows whose owner is dead and fails RUNNING `restore_jobs` rows whose owner is dead rather than resuming them (a resume is a job, which the claim excludes), removing their targets, and runs the FAILED-target duty (such a target keeps its backup in use, so `--forget-backup` refuses until the reconciler has removed it); it skips every other duty, so it never deletes the prefix of a FAILED or DELETED row a `--move-store` is rewriting. Its duties: resolve a CREATING backup row whose owner is dead to AVAILABLE or FAILED; delete the prefix of a FAILED row once its `heartbeat_at` is older than `stale_job_after_secs`, retrying while the prefix lists non-empty; finish a DELETING row (delete the manifest, then the prefix, then mark DELETED once the prefix lists empty); delete the prefix of a DELETED row that still lists objects; remove a FAILED or DELETED backup row once its prefix is empty and its `completed_at` is older than `failed_row_retention_secs` (default 7 days); claim and resume a RUNNING `restore_jobs` row whose owner is dead (once its `heartbeat_at` is older than `stale_job_after_secs` on every backend, 5.3 step 7, and a row with no `restore_progress` row is resumed only if the target's data table is empty, otherwise marked FAILED with reason `target_not_empty`, its target left in place and reported for the operator, since a catalog restored from a dump can name a restore that finished and was written to since; the operator inspects that target at the storage level (a CREATING table accepts no reads) and removes it with `extenddb backup remove-target ` (PR 10), the one command that removes a FAILED restore's CREATING target and its record, through `remove_restore_target`, under the maintenance claim like the other writing commands, since DeleteTable refuses a restore target (5.3 step 6); a legacy restore (`kind = legacy`) is marked FAILED and its target removed the same way instead, section 3; on MongoDB a row with `restore_backfill_pending` is DONE, not RUNNING, and is never resumed while the backfill worker finishes it, 5.3 step 6), skipping it to the next cycle when no job permit is free; finish a FAILED `restore_jobs` row whose target table is still CREATING, the FAILED-target duty (4(e); landed in PR 6, whose `list_dead_restore_jobs` returns such a row once it is stale; `BackupCatalog::remove_restore_target`, section 3, not DeleteTable, which refuses a restore target; never for reason `target_not_empty`, whose target is the operator's, above); remove a CREATING restore target that has no job row (`remove_restore_target`); remove `restore_progress` rows whose table is not a CREATING restore target; remove spool entries under this instance's `spool_path` whose job is not RUNNING with this instance's `spool_host` (5.3 step 3); on Cassandra, remove an `active_restores` entry of a backup row whose target is not a CREATING restore target of that backup; remove a legacy CREATING backup row whose create is dead, with its payload (section 3). The reconciler's writes to a backup row carry what the job's would (`size_bytes`, `item_count`, `source_table_details_json` from the manifest for AVAILABLE; `completed_at` for both outcomes, the retention clock). On PostgreSQL its claim is an advisory lock on a dedicated connection that it closes when the row is resolved. `extenddb backup list --failed` prints FAILED rows with their `failure_reason`. +State transitions it owns: CREATING to AVAILABLE and to FAILED by the job (5.1 steps 4, 5) and by the reconciler (5.1 step 6); every deleting duty of 4(a). The reconciler, in one place: The reconciler runs at startup and then every `reconcile_interval_secs` (default 60, section 7); it acts on no row of the store while the configured store is unhealthy (the legacy duty touches no store and still runs), and it checks the maintenance claim first, and while one is held it runs only the duties that resolve the states a maintenance command waits on: it finishes DELETING rows of the configured store, resolves CREATING backup rows whose owner is dead and fails RUNNING `restore_jobs` rows whose owner is dead rather than resuming them (a resume is a job, which the claim excludes), removing their targets, and runs the FAILED-target duty (such a target keeps its backup in use, so `--forget-backup` refuses until the reconciler has removed it); it skips every other duty, so it never deletes the prefix of a FAILED or DELETED row a `--move-store` is rewriting. Its duties: resolve a CREATING backup row whose owner is dead to AVAILABLE or FAILED; delete the prefix of a FAILED row once its `heartbeat_at` is older than `stale_job_after_secs`, retrying while the prefix lists non-empty; finish a DELETING row (delete the manifest, then the prefix, then mark DELETED once the prefix lists empty); delete the prefix of a DELETED row that still lists objects; remove a FAILED or DELETED backup row once its prefix is empty and its `completed_at` is older than `failed_row_retention_secs` (default 7 days); claim and resume a RUNNING `restore_jobs` row whose owner is dead (the resume is PR 9a's, with the restore job; this release's reconciler logs such a record once; once its `heartbeat_at` is older than `stale_job_after_secs` on every backend, 5.3 step 7, and a row with no `restore_progress` row is resumed only if the target's data table is empty, otherwise marked FAILED with reason `target_not_empty`, its target left in place and reported for the operator, since a catalog restored from a dump can name a restore that finished and was written to since; the operator inspects that target at the storage level (a CREATING table accepts no reads) and removes it with `extenddb backup remove-target
` (PR 10), the one command that removes a FAILED restore's CREATING target and its record, through `remove_restore_target`, under the maintenance claim like the other writing commands, since DeleteTable refuses a restore target (5.3 step 6); a legacy restore (`kind = legacy`) is marked FAILED and its target removed the same way instead, section 3; on MongoDB a row with `restore_backfill_pending` is DONE, not RUNNING, and is never resumed while the backfill worker finishes it, 5.3 step 6), skipping it to the next cycle when no job permit is free; finish a FAILED `restore_jobs` row whose target table is still CREATING, the FAILED-target duty (4(e); landed in PR 6, whose `list_dead_restore_jobs` returns such a row once it is stale; `BackupCatalog::remove_restore_target`, section 3, not DeleteTable, which refuses a restore target; never for reason `target_not_empty`, whose target is the operator's, above); remove `restore_progress` rows whose table is not a CREATING restore target and spool entries under this instance's `spool_path` whose job is not RUNNING with this instance's `spool_host` (5.3 step 3), both from PR 9a, which adds the records they read (the duty that removed a CREATING restore target with no job row is withdrawn: on every backend the job record is the registration itself, so no such state exists after migration 5); on Cassandra, remove an `active_restores` entry of a backup row whose target is not a CREATING restore target of that backup; remove a legacy CREATING backup row whose create is dead, with its payload (section 3). The reconciler's writes to a backup row carry what the job's would (`size_bytes`, `item_count`, `source_table_details_json` from the manifest for AVAILABLE; `completed_at` for both outcomes, the retention clock). On PostgreSQL its claim is an advisory lock on a dedicated connection that it closes when the row is resolved. `extenddb backup list --failed` prints FAILED rows with their `failure_reason`. -Kill points (testing summary): between a data file upload and the manifest; between the manifest put and the AVAILABLE update; SIGTERM during a backup. Main tests that must pass unchanged: #384's backup tests other than the vector tests named above. Documentation rows it corrects: the differences doc's backup consistency row (its MongoDB half, "not point-in-time" today, which the snapshot read concern of section 3 makes one; its Cassandra half stays true), its vector-backup row ("Backing up and restoring a table that has vector indexes": supported on the off-host path from PR 8 and 9a, refused only into a target without vector support, CreateBackup on the legacy create, without `[backup.store]` or on Cassandra, still refused as on main, the legacy reader's behaviour kept for legacy backups; 5.1 step 1), its restore-interruption row (one reconciler with the duties above in place of the per-backend sweeps), a new row for CreateBackup of a table holding an `N` value #618 let in (the backup fails on the off-host path, where main backs it up and restores it; 5.1 step 4), the limits doc's line cap and file size rows, the usage guide's "backups not supported" line; upgrade manual: on the off-host path a backup of a table holding an `N` value the wire refuses (#618) fails where main backs it up, which meets file-backed SQLite without a `[backup]` section and any deployment once a store is configured (5.1 step 4), the reason reaches only the operator (`backup list --failed`, and the log, which names each offending item's key and attribute path with the backup ARN a tenant's support case can cite), and the way out is to rewrite each named item. +Kill points (testing summary): between a data file upload and the manifest; between the three documents and the manifest (`before_manifest`); between the manifest put and the AVAILABLE update; SIGTERM during a backup. Main tests that must pass unchanged: #384's backup tests other than the vector tests named above. Documentation rows it corrects: the differences doc's backup consistency row (its MongoDB half, "not point-in-time" today, which the snapshot read concern of section 3 makes one; its Cassandra half stays true), its vector-backup row ("Backing up and restoring a table that has vector indexes": supported on the off-host path from PR 8 and 9a, refused only into a target without vector support, CreateBackup on the legacy create, without `[backup.store]` or on Cassandra, still refused as on main, the legacy reader's behaviour kept for legacy backups; 5.1 step 1), its restore-interruption row (one reconciler with the duties above in place of the per-backend sweeps), a new row for CreateBackup of a table holding an `N` value #618 let in (the backup fails on the off-host path, where main backs it up and restores it; 5.1 step 4), the limits doc's line cap and file size rows, the usage guide's "backups not supported" line; upgrade manual: on the off-host path a backup of a table holding an `N` value the wire refuses (#618) fails where main backs it up, which meets file-backed SQLite without a `[backup]` section and any deployment once a store is configured (5.1 step 4), the reason reaches only the operator (`backup list --failed`, and the log, which names each offending item's key and attribute path with the backup ARN a tenant's support case can cite), and the way out is to rewrite each named item. ### PR 9a: restore job Goal: the restore job of 5.3 on the same backend over a filesystem store. Its line names the interfaces and carries the acceptance: -- PR 9a: restore job per 5.3, same backend, filesystem store: request creates table and `restore_jobs` row atomically in a detached task and returns CREATING (`TableAlreadyExistsException` or `TableInUseException` on a name race, as #384 returns them; `LimitExceededException` over the job limit); Cassandra claim and heartbeat by lightweight transaction; job verifies every file before loading any, with `.verified` spool markers; `bulk_load(target_table_id, claim, file_index, start_at, items)` per backend with the progress record as 5.3 step 5 specifies for each (PostgreSQL data migration 005 for `restore_progress`, reserved here) and the `BackupCatalog` restore methods of section 3; the MongoDB terminal sequence of 5.3 step 6 (DONE and `restore_backfill_pending` in one transaction, the worker's flip conditioned on the flag); `spool_host` as host and spool path; the item validator PR 8 adds, bound with the target's `max_item_size_bytes` (`N` normalization included); items read through `ExtenddbManifest::reader_prehashed`; the restore's vector refusal narrowed to a target without vector support, with `vector_index_backups_are_refused_in_the_engine` rewritten to that case; duplicate rejection across and within files (#394's per-file key-range check on key-ordered backups, the count after the last file on unordered ones, the MongoDB post-resume count check); the manifest and sidecar spooled for a resume; a PostgreSQL resume or target deletion only after the stale window; conditional ACTIVE flip before the progress rows are deleted, the job's target removal (`remove_restore_target`, not DeleteTable) gated on a matched FAILED write, DeleteTable on a restore target refused as #384 refuses it, the in-use rule of 5.2 checked in step 1 before the permit and again in step 4's registration together with `status = AVAILABLE` under the backup row's lock (MongoDB's `$inc`, Cassandra's `active_restores` lightweight transaction), multi-part-key tables restored; `bulk_load` writing index rows in the batch's transaction, with the base rows, on PostgreSQL and SQLite and the ACTIVE flip setting every index ACTIVE there; download transport errors retried for 15 minutes; a restore stopped by shutdown left RUNNING; the store and `kind` recorded on the job row and a takeover writing its `spool_host`; a PostgreSQL resume with no progress row only into an empty target. Acceptance: round trip preserves items, keys, GSIs, LSIs, throughput, SSE, vector where supported, and restores the table `STANDARD` whatever its class (#384's `table_class_is_not_restored` passes unchanged); tampering file 2's sha256 leaves a table that was never writable and is gone after the job fails, with no item ever written; SIGKILL after file 1's commit then restart writes exactly the backup's item count on every backend (MongoDB via duplicate-key suppression); SIGKILL at each kill point of the testing summary leaves a table that either finishes with the right count or is gone (on MongoDB, killed between the DONE write and the worker's flip, the restarted worker finishes it and the reconciler leaves it alone); an unordered backup carrying one key in two files fails into every target, Cassandra included, and a key-ordered backup whose file leaves its `key_range` fails; a second restore of a backup while a target restored from it is CREATING, and a DeleteBackup of that backup, return `BackupInUseException` on every backend, on MongoDB also between the job's DONE write and the worker's ACTIVE flip, and #384's `test_restore_into_a_creating_target_and_a_second_restore_of_its_backup_are_refused` passes unchanged against the new path; the two orderings over an off-host backup (a `backup_index` row), gated on the `backup_index` row lock, run here for the first time, deferred from PR 6 (which has no restore of such a row): a DeleteBackup that wins leaves the registration BackupNotFound and no target, a registration that wins leaves DeleteBackup BackupInUse; on MongoDB two concurrent registrations of one backup leave one target and return `BackupInUseException` to the other; a restore of an ARN of account B that carries the id of account A's backup returns BackupNotFound; on SQLite and PostgreSQL a Query on each GSI and SearchVectors return the restored items as soon as the target reports ACTIVE; a SIGTERM during a restore leaves its row RUNNING and the restore finishes after the restart; a store outage shorter than the retry bound mid-download leaves the restore RUNNING, and it completes once the store answers. Depends on PR 8. +- PR 9a: restore job per 5.3, same backend, filesystem store (PR 8 leaves it the engine's `check_catalog_row_for_restore`, which answers `ValidationException` "Backup is in a backup store; this release restores in-catalog backups only" for a row of the configured store, the 24 restore cells of `tests/test_backup_restore_fidelity.py` and `tests/test_auth_permissions.py`'s restore-authorization cell skipped on a server with a store, and the vector wire test's restore assertion, all four to replace with the restore): request creates table and `restore_jobs` row atomically in a detached task and returns CREATING (`TableAlreadyExistsException` or `TableInUseException` on a name race, as #384 returns them; `LimitExceededException` over the job limit); Cassandra claim and heartbeat by lightweight transaction; job verifies every file before loading any, with `.verified` spool markers; `bulk_load(target_table_id, claim, file_index, start_at, items)` per backend with the progress record as 5.3 step 5 specifies for each (PostgreSQL data migration 005 for `restore_progress`, reserved here) and the `BackupCatalog` restore methods of section 3; the MongoDB terminal sequence of 5.3 step 6 (DONE and `restore_backfill_pending` in one transaction, the worker's flip conditioned on the flag); `spool_host` as host and spool path; the item validator PR 8 adds, bound with the target's `RestorableLimits` (item, hash key, range key and attribute name sizes; `N` normalization included); items read through `ExtenddbManifest::reader_prehashed`; the restore's vector refusal narrowed to a target without vector support, with `vector_index_backups_are_refused_in_the_engine` rewritten to that case; duplicate rejection across and within files (#394's per-file key-range check on key-ordered backups, the count after the last file on unordered ones, the MongoDB post-resume count check); the manifest and sidecar spooled for a resume; a PostgreSQL resume or target deletion only after the stale window; conditional ACTIVE flip before the progress rows are deleted, the job's target removal (`remove_restore_target`, not DeleteTable) gated on a matched FAILED write, DeleteTable on a restore target refused as #384 refuses it, the in-use rule of 5.2 checked in step 1 before the permit and again in step 4's registration together with `status = AVAILABLE` under the backup row's lock (MongoDB's `$inc`, Cassandra's `active_restores` lightweight transaction), multi-part-key tables restored; `bulk_load` writing index rows in the batch's transaction, with the base rows, on PostgreSQL and SQLite and the ACTIVE flip setting every index ACTIVE there; download transport errors retried for 15 minutes; a restore stopped by shutdown left RUNNING; the store and `kind` recorded on the job row and a takeover writing its `spool_host`; a PostgreSQL resume with no progress row only into an empty target. Acceptance: round trip preserves items, keys, GSIs, LSIs, throughput, SSE, vector where supported, and restores the table `STANDARD` whatever its class (#384's `table_class_is_not_restored` passes unchanged); tampering file 2's sha256 leaves a table that was never writable and is gone after the job fails, with no item ever written; SIGKILL after file 1's commit then restart writes exactly the backup's item count on every backend (MongoDB via duplicate-key suppression); SIGKILL at each kill point of the testing summary leaves a table that either finishes with the right count or is gone (on MongoDB, killed between the DONE write and the worker's flip, the restarted worker finishes it and the reconciler leaves it alone); an unordered backup carrying one key in two files fails into every target, Cassandra included, and a key-ordered backup whose file leaves its `key_range` fails; a second restore of a backup while a target restored from it is CREATING, and a DeleteBackup of that backup, return `BackupInUseException` on every backend, on MongoDB also between the job's DONE write and the worker's ACTIVE flip, and #384's `test_restore_into_a_creating_target_and_a_second_restore_of_its_backup_are_refused` passes unchanged against the new path; the two orderings over an off-host backup (a `backup_index` row), gated on the `backup_index` row lock, run here for the first time, deferred from PR 6 (which has no restore of such a row): a DeleteBackup that wins leaves the registration BackupNotFound and no target, a registration that wins leaves DeleteBackup BackupInUse; on MongoDB two concurrent registrations of one backup leave one target and return `BackupInUseException` to the other; a restore of an ARN of account B that carries the id of account A's backup returns BackupNotFound; on SQLite and PostgreSQL a Query on each GSI and SearchVectors return the restored items as soon as the target reports ACTIVE; a SIGTERM during a restore leaves its row RUNNING and the restore finishes after the restart; a store outage shorter than the retry bound mid-download leaves the restore RUNNING, and it completes once the store answers. Depends on PR 8. Invariants honoured: 4(b) (the restore record and its transitions), 4(d) (every target by `RestoreTargetKey`), 4(e) (the claim, the FAILED-write gate before the target is removed, the PostgreSQL stale window), 4(f) (the job checks the maintenance claim after its row exists), 5.2 (the registration under the backup row's lock), 5.3 as written, section 1's duplicate rule and vector refusal, and the Goals' fidelity list. Interfaces it adds: `bulk_load` (section 3) and the restore methods of `BackupCatalog`: @@ -448,7 +452,7 @@ Phase 3, later (v1.x): - Unit: manifest serde and golden files, key mapping and ARN parsing, store conformance. - Integration, every backend, every PR: round trip with full configuration preservation, composite keys, GSIs and LSIs, vector indexes where supported, legacy fallback, PITR surface. - Consistency: concurrent-writer snapshot test on every backend. -- Crash: SIGKILL during upload and during restore, verified by the reconciler tests. These extend the SIGKILL harness the migration tests already use. Backup kill points: between a data file upload and the manifest; between the manifest put and the AVAILABLE update (the one path to the reconciler's AVAILABLE outcome). Restore kill points: after a file's transaction commits and before the next file starts; MongoDB between `insert_many` and the progress write (the path that exercises E11000 suppression and the count check); Cassandra between a batch's last write and its progress write; SQLite mid-batch; PostgreSQL between the ACTIVE flip and the progress-row deletion; MongoDB between the DONE write (with `restore_backfill_pending`) and the worker's ACTIVE flip, where the reconciler must leave the row alone and the restarted worker must finish; between the FAILED write and the target removal of step 7; a fenced-out restore job that resumes after a successor claimed the row (its FAILED write matches zero rows and the successor's table survives); PostgreSQL, a restore job whose catalog session ended (`pg_terminate_backend` on its claim connection) while its data writes continue: no successor loads into or deletes the target until `heartbeat_at` is stale, and the job aborts at its next heartbeat. DeleteBackup kill points: after the DELETING write; after the manifest delete; after `delete_prefix` and before the DELETED write. Shutdown: SIGTERM during a backup (FAILED, empty prefix, no multipart upload left) and during a restore (row left RUNNING, finished after the restart). +- Crash: SIGKILL during upload and during restore, verified by the reconciler tests. The backup kill points are run by hand against the built binary with the script posted with PR 8 (`kill-points.sh`, outside the repository: a server built with `extenddb-app/backup-test-hooks`, paused by `EXTENDDB_BACKUP_PAUSE` at the named point, killed by PID and restarted, with the row and the prefix watched through `extenddb backup list` and the store), not by a pytest cell; the restore kill points extend the SIGKILL harness the migration tests already use. Backup kill points: between a data file upload and the manifest; between the three documents and the manifest; between the manifest put and the AVAILABLE update (the one path to the reconciler's AVAILABLE outcome). Restore kill points: after a file's transaction commits and before the next file starts; MongoDB between `insert_many` and the progress write (the path that exercises E11000 suppression and the count check); Cassandra between a batch's last write and its progress write; SQLite mid-batch; PostgreSQL between the ACTIVE flip and the progress-row deletion; MongoDB between the DONE write (with `restore_backfill_pending`) and the worker's ACTIVE flip, where the reconciler must leave the row alone and the restarted worker must finish; between the FAILED write and the target removal of step 7; a fenced-out restore job that resumes after a successor claimed the row (its FAILED write matches zero rows and the successor's table survives); PostgreSQL, a restore job whose catalog session ended (`pg_terminate_backend` on its claim connection) while its data writes continue: no successor loads into or deletes the target until `heartbeat_at` is stale, and the job aborts at its next heartbeat. DeleteBackup kill points: after the DELETING write; after the manifest delete; after `delete_prefix` and before the DELETED write. Shutdown: SIGTERM during a backup (FAILED, empty prefix, no multipart upload left) and during a restore (row left RUNNING, finished after the restart). - Distribution matrix for the filesystem default path: unprivileged local launch, the npm launcher, the container image with and without a volume, `sqlite-memory`. - Cross-backend matrix: nightly. - Live DynamoDB: Phase 2, measured. @@ -461,7 +465,7 @@ Phase 3, later (v1.x): - Every failure path deletes the prefix or leaves it unreachable from List, and the row never says AVAILABLE for an incomplete prefix. - Restore writes no item before every data file's checksum has verified; a verification failure deletes the CREATING target table, which never accepted a read or a write; the request refuses before creating the table when the target cannot represent the manifest or the key ranges overlap. - A file's data and its progress record commit in one transaction on PostgreSQL and SQLite (per 10k-item batch on SQLite, with the items-applied count), so no item is applied twice; on MongoDB the progress record follows the insert, re-application of the one unrecorded file is idempotent by duplicate-key suppression (code 11000 only; any other write error fails the restore), and the post-resume document count is checked against the manifest; on Cassandra the progress record follows each batch under a lightweight transaction and at most one batch is re-applied. A key-ordered restore runs the per-file key-range check on every file, and an unordered one ends with a count of the target equal to the manifest's item count. On PostgreSQL the ACTIVE flip precedes the deletion of progress rows; on MongoDB the job's DONE write and `restore_backfill_pending` are one transaction, and the worker's ACTIVE flip is conditioned on that flag and CREATING, not on the job's generation. -- Every status write by a job is conditional on its owner generation and on the row still being in the job's state (`CREATING` for a backup, `RUNNING` for a restore); a job whose conditional write matches no row, or whose conditional write errors with an unknown outcome, performs no further store or table operation, deletes (of a prefix or of a restore target) included; the reconciler takes the same claim as a worker and leaves `heartbeat_at` untouched. +- Every status write by a job is conditional on its owner generation and on the row still being in the job's state (`CREATING` for a backup, `RUNNING` for a restore); a job whose conditional write matches no row, or whose conditional write's outcome stays unknown after the re-read, performs no further store or table operation, deletes (of a prefix or of a restore target) included; the reconciler takes the same claim as a worker and leaves `heartbeat_at` untouched. - The ARN is parsed by `BackupArn` once, in the engine, before any catalog or store access; no other code splits ARN strings. - Account scoping is checked on the ARN before any store access, every backup row is read by account and backup id and accepted only when its account, table and ARN equal the request's, and every restore-target method takes the target's account, name and table id. - The backup job validates every item with `validate_restorable_item` before encoding it, keeps validating after the first refusal, and the restore binds the same function. @@ -533,3 +537,4 @@ Changed after a review of the draft against main `9f43d03` and PRs #384, #335, # 26. (2026-10-30, PR 6 as implemented on #666's registry.) PR 6's migration is id 5 on PostgreSQL and SQLite (one `MigrationId` list for both, ids 1 to 4 taken), and the Cassandra file is V006 and the MongoDB step `002_backup_catalog_index`, both writing `0.0.5` so that one number names one change on every backend; the literals in 4(i) and PR 6's section are updated, PR 1's line keeps its own numbers and renumbers if it lands after this. 27. (2026-10-30, PR 6.) The FAILED-target duty lands in PR 6: `list_dead_restore_jobs` also returns a FAILED `restore_jobs` row whose target is still CREATING once the row is stale, and the reconciler removes the target without a second FAILED write, which closes the state the job's or the reconciler's failure path leaves when the FAILED write lands and the removal does not (4(e), PR 6, PR 8). PostgreSQL's dead rule is the heartbeat alone, as on MongoDB and Cassandra: the advisory lock is the claim's exclusion, not the liveness test, and the `IS NULL` arm item 21 named does not exist, the column being NOT NULL (4(e), 5.3 step 7, PR 6). The stale floor is computed from the period the jobs run at, `max(heartbeat_interval_secs, JOB_HEARTBEAT_INTERVAL)`, the one constant every backend's heartbeat loop reads until the knob reaches the jobs, and `max_concurrent_jobs` has an upper bound (5.1 step 4, section 7, PR 6). SQLite's `backup_server_slot_count` is kept for snapshot opens until PR 8, the legacy create trying it without waiting and answering the one `LimitExceededException` text (PR 6). The engine's `BackupArnSpellings` carries an optional parsed ARN per spelling, so a legacy backup of a name the store-prefix rules refuse (dot-only, over 255 bytes) still describes, restores and deletes through the legacy reader (4(d), PR 6). DeleteBackup over a row runs the in-use check before the other-store check, as a restore does, builds the description before the DELETING write, and a row whose source table the build cannot read is left out of ListBackups rather than failing the listing (5.2, PR 6). The acceptance cell running the two orderings over a `backup_index` row is deferred to PR 9a, with the row lock observed by a two-transaction test of `start_backup_delete` in PR 6 (PR 6). On Cassandra an index row's list rows exist for as long as the row does, whatever its status, readers re-reading the authoritative row; the legacy create keeps its publish-time rule (PR 6). The reconciler records the worker metrics of section 9 (PR 6). On Cassandra the migration runner writes `catalog_version` after the last file's statements, its backfill step and its `schema_history` row, so a run that fails in the step leaves the previous version and `migrate --yes` runs the file again; a Cassandra job stops after three heartbeat errors in a row, as after a heartbeat that does not apply, and the fence the reconciler puts on a dead create's row takes a fresh `owner_generation` (4(e), PR 6). MongoDB's RestoreTableFromBackup waits a bounded time for the registration and answers `InternalServerError` past it while the registration goes on; what the request leaves is stated in 5.3 and the differences document (5.3, PR 6). CreateBackup checks `server.region` before forming a backup ARN and refuses, naming the setting, under a region outside `[a-z0-9-]`, where main created a backup no operation could then address; the store-prefix rules on the name are applied before the ARN is formed so a name over 255 bytes is refused naming the limit whatever its length (5.1 step 1, PR 6). The `LimitExceededException` text names the bound that was met beside its number (`max_concurrent_jobs`, SQLite's snapshot slots, or PostgreSQL's connection slots for the claim connection), one implementation (`limit_exceeded_message`) every refusal binds (5.1 step 1, section 7, PR 6). Section 7 states, where it bounds snapshots by half the pool, that the pool is the raised one (section 7, PR 6). Item 21's `IS NULL` arm is withdrawn by this item alone; item 21 keeps its wording, as item 22 requires. On Cassandra, where the removal is a claim and then a drop, the FAILED-target duty also lists a FAILED record whose target is DELETING with its registration, and `remove_restore_target` resumes the removal from the drop (4(e), PR 6). On MongoDB the in-use check of a restore's registration and of DeleteBackup reads at the transaction read concern, majority-committed data, while DescribeTable reads at `local`, so a request issued within the majority-commit lag after DescribeTable first shows the target ACTIVE can still answer `BackupInUseException` and succeeds when retried; the differences document's concurrent-restores row states it with the measured lag (5.3, PR 6). The rule that a job performs no further operation after a write it cannot confirm is scoped to the conditional write itself: a legacy restore whose claim connection (the heartbeat's) errors runs 5.3 step 7, the conditional FAILED write on a pooled connection and the removal only when that matched, so on PostgreSQL the target is removed at the next heartbeat rather than by the reconciler once the record is stale (4(e), 5.3 step 7, PR 6). On MongoDB `remove_restore_target` deletes the target's catalog document first and returns the target removed once that is gone: a collection drop that fails after it is logged naming the collections and leaves them for `backup sync` (PR 10), there being no record for a later reconciler cycle to retry from (4(e), PR 6). 28. (2026-10-30, PR 7 as implemented.) A store problem never gates the in-catalog create: with a configured store that is unhealthy, CreateBackup writes the in-catalog format as it does with no store, and only the store-backed create of PR 8 is refused by `require_healthy_store`; a defaulted SQLite root whose identity check fails (a catalog re-created by `destroy` and `init` under the old root's `extenddb-store.json`) is the same warning as a failed validation, leaving the store unconfigured, and `destroy` and the admin guide say the file must go with the catalog (section 7, PR 7). An S3 store's `validate` and identity check run after the listener is up, in the reconciler's first cycle, with the store `checking` until the first result reports; a filesystem store's run before the listener; the configuration errors that need no request still stop startup; a `validate` refusal of an S3 bucket after the listener makes the store unhealthy (`failed validation`) rather than stopping startup (section 7, 9, PR 7). `/health`'s `reason` is the problem's category alone; the ids and ARNs go to the operator's log line (section 9, PR 7). The identity file is read up to 64 KiB. The S3 root label is built from the endpoint the client uses, in one spelling. A check that cannot read the catalog leaves the store's health unchanged. An existing spool must be owner-only. The maintenance claim is heartbeated on a timer from the take to the release, and PostgreSQL's `check_maintenance_claim` tests the advisory lock before the heartbeat's age (4(f), PR 7). The jobs' side of the claim is implemented in the engine for the in-catalog create and restore, before the permit and, since those paths commit their row inside the backend, before the row exists; the ordering 5.1 step 2 states applies to the store-backed job PR 8 adds (5.1, 5.3, PR 7). `backup list` opens the SQLite catalog without the serve lock, reads the identity file without creating it and runs on an unhealthy store; on SQLite `backup purge` runs the reconciler's duties with a zero window before its wait; a CREATING row with a dead owner is purged as any row, through `remove_dead_creating_backup_row`, a `BackupCatalog` method of PR 7; a FAILED row with a young heartbeat is waited on; the dry run exits 0 (PR 7, PR 10). Cassandra's claim reads run at `LOCAL_SERIAL` and an insert whose answer was lost is settled by a re-read of the generation (PR 7). The PR 10 comment on `backup_index_by_status` is corrected: V006's entry carries `store_kind` and `store_id`. The body sentences that paraphrased the earlier rules are edited with this item (section 7's configuration-error paragraph, 4(f) and 5.1 step 2 and 5.3 step 4's claim check, PR 6's `store_root` row, PR 7's interfaces and acceptance), and the filesystem store's startup checks are bounded by a timeout that leaves the store `checking` when a mount stops answering after the root is opened (a root that does not answer at startup holds the server before the listener, as the database directory would). Also: the PostgreSQL check takes its shared lock at transaction scope so a cancelled request leaves no lock on the pool; a dropped `HeldClaim` stops its timer; the purge's dry run is a read like `list`; a CREATING row is removed before its prefix; an empty `filesystem.path` is left for `open` to refuse; the spool meets the root's whole rule, parents included; `backup list` reads the catalog's status without the serve lock. The defaulted SQLite root's warning holds only while the catalog has no backup rows for the root (`read_store_ids(Filesystem, root)`: the upgrade, fresh and re-init cases); with rows for it the root has been used and is handled as an explicitly configured one, so its backups never vanish behind a warning: a failure of its identity check or manifest probe is an unhealthy store (reported, its rows' restores and deletes refused, rechecked each cycle), and a refusal of its `open` or `validate` a startup error naming the root (section 7, PR 7's acceptance). The server's `configure` carries the reason for an absent store out (`NotConfigured`), and `backup list` and `backup purge` print it, so a defaulted root that failed its check is no longer reported as "[backup.store] is absent or disabled". The store's check runs under a per-kind bound in every reconciler cycle (`check_configured_store`: 5 s for a filesystem root, the startup bound; 30 s for a bucket, under the SDK's 60 s per-attempt timeout so a timed-out check is dropped before the SDK retries its request), recording a transport failure naming the bound, and the cycle selects the check against the shutdown token; an unbounded check let a stalled S3 endpoint hold each cycle for the SDK's whole retry budget and a stalled mount hold it for good, with shutdown hitting the drain timeout (section 7). A filesystem root's `validate` reports no transport class; an I/O failure of its probe write or of resolving the root at startup degrades the store (`failed validation`) as the same condition one cycle later does, while the owner-only and existence refusals stay configuration errors (section 7). A transport failure's operator line carries the store error's whole cause chain (the SDK's top link is a bare class). Every store error of a check that is neither a transport failure nor a denial is `failed validation`, the category `validate` gives it, so a bucket deleted under a running server reads as a bucket missing at startup. A filesystem root that is or contains the database directory is refused in `resolve_store`, beside the spool rule (section 7). A filesystem store's synchronous root checks run on the blocking pool so the startup bound fires whichever call stalls. The checking store's refusal says its first check is pending; `/health`'s reason list carries `catalog unreadable`; `record_check` reports whether the category changed. Purge's wait set is CREATING and FAILED rows with a live heartbeat and RUNNING restores, DELETING rows being finished rather than waited on, and its refusal rule is that set at the wait's end; 4(f), the PR 7 "Destroy and purge" paragraph and the PR 10 claim paragraph (`MAINTENANCE_LOCK_KEY`, `pg_try_advisory_xact_lock_shared` then the heartbeat age, `LOCAL_SERIAL`) and the PR 6 and PR 7 signatures (`check_maintenance_claim(stale_after)`, `maintenance_claim(holder, stale_after) -> Box`) are edited to the code with this item. A row a DeleteBackup moves to DELETING or DELETED, or removes, between purge's read and its turn is finished and counted as deleted, not left (the deletion is one idempotent sequence on both sides). On SQLite purge waits and removes CREATING rows with a zero stale window (under the serve lock every owner is dead, so a young heartbeat proves nothing), and the server's startup under the serve lock removes a maintenance claim a stopped command left, so a killed purge costs no stale window; on every backend purge releases the claim on `SIGINT` or `SIGTERM`, at once during the wait and before the next row during the deletion. The wait's three failures (jobs still running at the limit, the claim lost, the catalog unreadable) and the signal are reported each as itself. PostgreSQL's factory starts the GSI queue workers only for a server, not for a command reading beside one. Purge's restore part of the wait set is the RUNNING `restore_jobs` records alone: the dead-job listing it reads with a zero window also returns the FAILED records whose target is still CREATING (the FAILED-target duty's, 4(e)), which are the reconciler's to finish and no job runs under, so they are not waited on; the wait finishes at once on a stopped deployment that holds one. On SQLite the zero-window pass resolves the kinds this release writes; a RUNNING `restore_jobs` record of kind `manifest`, which PR 9a introduces, is left for the reconciler that resumes it and purge waits on it and refuses naming the table, the one state the PR 10 paragraph's "never makes it wait or refuse" does not cover. The reconciler finishes the configured store's DELETING rows while the maintenance claim is held, as 4(f) lists. The binary ends its tokio runtime with a bounded shutdown (5 s) after `serve` and after every command: a filesystem store's check abandoned at its bound leaves its call parked on a blocking thread until the mount answers, and a plain drop of the runtime joined that thread, so a server whose root had stopped answering drained its workers at SIGTERM and then stayed alive until SIGKILL. Each call of a store check runs as a task of its own that an abandoned check does not take with it, and the store records that a call is in flight; no check starts while one is (the store is recorded `unreachable` again, with the reason), so a mount that stays stalled parks one thread rather than one per cycle. A command's identity read (`backup list`, the dry run) runs under the store's check bound as the server's check does, where the unbounded read waited out the SDK's three attempts of 60 s on an endpoint that accepts and never answers. An empty `filesystem.path` is compared with nothing in `resolve_store` (compared as written it was the working directory, refused by name as the database directory when the server starts there) and left for `open` to refuse. A root that refuses `link(2)` with `EPERM` is reported `failed validation` (the README and the deployment guide said `unreachable`). `tests/test_backup_store_config.py` is excluded from `devtools/run-tests`'s pytest run, as its docstring says; a catalog whose rows cannot be read keeps a defaulted root configured, and that arm is tested. +29. (2026-10-31, PR 8 as implemented.) The job and the reconciler live in the engine's backup module (`crates/engine/src/backup/{job,reconciler}.rs`), not in `crates/server/src/workers.rs`, and nothing crosses an in-process channel between the request and the job but the response (the pipeline's item and chunk channels are the job's own): the request's detached task opens the snapshot, claims the row, checks the maintenance claim and answers through a one-shot channel, then continues as the job; the server hands the engine the shutdown token, the metrics collector, the job settings and the manifest's bucket and key prefix through `BackupRuntime` and `ConfiguredStore`, and `drain_jobs` (30 s) runs before the workers drain (PR 8); a stopped pipeline's sink answers the encoder's write with `BrokenPipe`, which nothing retries, where an `Interrupted` answer (std's "try again") kept a job cancelled in the middle of a 300k-row table spinning in the compressor's trailer write through the whole drain with its row left CREATING; the job ends within milliseconds of the signal on every store (5.1 step 5, PR 8). The reconciler's takeover is `BackupCatalog::claim_dead_backup(key, stale_after)`, not `claim_backup`, which inserts; `list_backup_rows_older_than(status, store, clock, age)` with `StaleClock::{Heartbeat, Completed}` carries the database's clock as a backend predicate for the FAILED-prefix delay and the tombstone retention, so the engine never holds a clock; `sweep_restore_registrations` is the Cassandra `active_restores` sweep with a default empty body; `StatusFields` gains `source_table_details_json`, which the job's and the reconciler's AVAILABLE writes fill with the manifest's `source_table` (4(e), PR 8). The PostgreSQL page is three layers, since a window function runs before `LIMIT`, and the stream ends when the inner query returned fewer rows than asked and none was cut (a page cut short of the row bound is not the end); the outer `ORDER BY` is a stated guarantee the executor gives anyway; SQLite's stream pulls no row past its cut (the driver's worker reads at most five rows ahead of it: the four `row_buffer_size` on the pool buffers and the one it holds while its `send` waits), so the rows-past-cut figure is PostgreSQL's alone (section 3, PR 8). `heartbeat_interval_secs` reaches the store-backed job as `max(heartbeat_interval_secs, JOB_HEARTBEAT_INTERVAL)`; the in-catalog paths keep the constant until PR 14, which corrects item 27's "until the knob reaches the jobs" (5.1 step 4, section 7). The job's pre-put check is two periods, not one (a beat sent at the tick returns up to a period later), which stays inside the three-period floor; the check measures from the instant taken before the claim's insert, not from the heartbeat handle's start, so an unbounded maintenance check (a catalog query held behind the pool) cannot admit a first put on a row already past `stale_job_after`; when the pipeline returns no put is in flight, so the failure path's `delete_prefix` follows the FAILED write with nothing to await; a lost or stalled heartbeat ends the job with no catalog write, the claim connection being the one that failed; each unknown outcome of a status write is followed by a re-read; the write is sent once more at most; after the second write a re-read that shows the row in the target status counts as matched and anything else as matching no row; within one cycle the duties run dead-owner first and FAILED-prefix after it, so a row the cycle just failed loses its prefix in that cycle once the dead owner's beat is stale (5.1 steps 4 to 6, 4(e)). SQLite's snapshot slot is kept as the backend's admission bound and the pool raised to `2 * (max_concurrent_jobs + 1)`, at most 34, which is what item 27's "folds it into `max_concurrent_jobs`" became (section 7, PR 6). On SQLite `claim_dead_backup` reads no window: a row whose slot no task holds is dead at once, and the engine calls it on every CREATING row each cycle (one set probe per live row); the dead owner's handle cannot observe `NotMatched` after a takeover on SQLite or PostgreSQL, whose slot and lock are the owner's life, and the engine reaches a taken-over row through `Unknown` and the re-read (4(e)). On MongoDB and Cassandra a successor is a live owner to a later taker only once it heartbeats, and the reconciler writes its one status before any beat, so two instances' reconcilers can both take over one dead row in that gap; the generation fence makes the loser's write `NotMatched` and one resolution lands; Cassandra's takeover condition carries the heartbeat it read (`IF owner_generation = AND backup_status = 'CREATING' AND heartbeat_at = `), which puts the liveness test inside the write; Cassandra's `list_backup_rows_older_than` reads the coordinator clock before its scan where `list_dead_restore_jobs` reads it after, both conservative; MongoDB's `Completed` listing requires the column present, since `age_is_stale` alone reads an absent column as the epoch (4(e), PR 8). MongoDB's stream applies no cut of its own (the server's 16 MiB reply bound is under the budget) and feeds no page-bytes counter, so `run_page_bounds` runs on PostgreSQL and SQLite only (section 3, PR 8); Cassandra's registrations sweep is one range read of `backups_by_arn` per cycle, enough while a catalog holds thousands of backups and not more, and the `backup_size_bytes` of `backups_by_account` and `backups_by_table` is written at the claim and not refreshed by `set_status`, which the index listings do not read (they re-read the row), as before PR 8 (PR 6). The spool sweep and the `restore_progress` sweep land with PR 9a, which adds the records they read, and the duty that removed a CREATING restore target with no job row is withdrawn: the job record is the registration on every backend (section 4, PR 8). The vector "round trip" PR 8 adds is its create half, the restore half being PR 9a's; the vector wire test asserts the store's ValidationException on the restore until then, and the fidelity suite's 24 restore cells skip on a server with a store (PR 8, PR 9a). `source_table.time_to_live` is read beside the open, outside the snapshot, and may trail it by the open's duration; DescribeBackup's `ItemCount` is the stats pass's figure while CREATING and the stream's count once AVAILABLE (5.1 step 2). Section 9 lists the metrics as `/metrics` emits them, the `MetricName` variants PR 8 adds (`BackupJobs`, `BackupJobDuration` in microseconds, `BackupBytesUploaded`, `BackupSnapshotAge`, `BackupStoreErrors`, the outcome and the store call in the operation dimension) as JSON under those names, and says the snake-case spellings are a converter's names and not a rendering the server does (section 9). `EXTENDDB_TEST_BACKUP_STORE` is set by every CI job that runs a suite creating backups, the pytest jobs and the Rust integration jobs (`filesystem` on PostgreSQL, which appends a store after `init`, and on SQLite, the defaulted root; `s3` on the SQLite job against moto; `none` on MongoDB and Cassandra and on the PostgreSQL Rust integration jobs; `run-integration-dev-mode` runs `tests/test_dev_mode_authorization.py` alone, which creates no backup, without it) and `devtools/run-tests --pytest` refuses without it (section 10, PR 8). PostgreSQL's snapshot-slot refusal now goes through `limit_exceeded_message` naming `[storage.postgres] pool_size snapshot-slot` (item 26's one-implementation rule). The acceptance's "on each backend" for the 100k backup reads "on each backend with a `SnapshotEngine`", Cassandra keeping the legacy create, and for the #618 cells "on PostgreSQL and SQLite", the two deployments `tests/test_backup_job.py`'s fixture starts (MongoDB's store-backed create is covered by the engine tests and the 100k run; no CI job runs the suite on a MongoDB build); an `abc` range key is refused by the number column on PostgreSQL and SQLite, so the range-key cell carries `1E126`, and UpdateItem validates the values its expression sets, so its cell carries the bad value in the key alone (PR 8). The 1M-item RSS runs and the kill points were measured on a 16-core, 30 GiB host (PostgreSQL 16.15 and MongoDB 7.0 in containers on loopback, the bundled SQLite 3.46) and the figures recorded with the PR in `docs/design/15-backup-restore-bench-pr8.md`; the memory bound holds (the 1M run's peak within 10 MiB of the 100k run's on PostgreSQL and SQLite, and the 500k run's within 5 MiB on MongoDB, where the 500k load alone took 27 minutes through the wire on that host; the bound is 112 MiB). Section 7's memory paragraph is rewritten to the terms the worst-expansion run measured: the earlier "the budget plus one row of text, plus its decoded form" left out the page's decoded rows waiting for the pipeline, the items in the feeder, the item channel and the encoder, one row's parse through the deserializer's intermediate value, and the line buffer, and it said nothing of the allocator; with 27.7 MB rows (three to a page) the terms sum to about 400 MB and the high-water mark over idle was 880 MiB on PostgreSQL and 580 to 760 MiB on SQLite across runs (507 MiB under `MALLOC_ARENA_MAX=1`), with MongoDB measured at 474 MiB on 13.5 MB rows, its 16 MiB document cap refusing the full shape; SQLite's driver run-ahead was bounded (sqlx's `row_buffer_size`, 50 rows by default, set to four on the pool: the 24-row run peaked at 844 MiB over idle with 50 and 696 MiB with four, back to back, and a 1M-row backup took 11.4 s with either), and whether the peak then grows with the number of worst rows is settled by the 12, 24 and 48-row series of the 2026-11-01 sentences below (it does with glibc's default arenas and does not under `MALLOC_ARENA_MAX=1`), and a one-item channel saved about 50 MiB of it for a 1M-row backup 75% slower (12.4 s to 21.6 s, both orders), so `ITEM_CHANNEL` stays at four. The PR 8 acceptance's "stays within section 7's formula" and "RSS must match the formula" read "under 1.2 times the sum of section 7's terms at the measured row with `MALLOC_ARENA_MAX=1`, at 12, 24 and 48 worst rows on one binary, the default-arena peak recorded beside it" (section 7, PR 8). The body sentences each of these corrects are edited with this item (4(e), 5.1 steps 2, 4, 5 and 6, section 7's knob comments, memory paragraph and pool paragraph, section 9, section 10's table, PR 6's slot sentence, PR 8's acceptance, interfaces, page paragraph and duty list, PR 9a's lead-in); earlier items keep their wording, as item 22 requires. (2026-11-01, PR 8 as measured and corrected.) The PostgreSQL page's row bound `n` is sized from the previous page's largest row, not its average, and the floor after the first page is 4 (`MIN_NEXT_PAGE_ROWS`; `MIN_PAGE_ROWS`, 16, is the first page's alone), because the server serializes and spills every one of the `n` rows the inner `LIMIT` reads, so a page costs the server `n` rows of text and not the rows it returns: after a page of 984 small rows and 3 large ones the average sized the next page at 810 rows of a table that was then large rows only, 5.9 GB of server temp files and 18 to 44 s for the three rows it returned, where the largest row sizes it at 4 (208 MiB of temp, 1.1 s), and a page of 27 MB rows now reads one row past its cut rather than thirteen (its steady state 1.2 s a page, 4.4 s before); the over-read that remains is the page that follows a row bound's worth of small rows, up to `n` rows of up to the line cap, which the sentence of the same day below bounds (section 3, 7, PR 8). The `EXPLAIN` acceptance holds for a page with the planner's estimate above `n` and not everywhere: on a table of 1,060 rows a page near the end whose `n` is above the rows left gets a bitmap scan of the primary key index and a `Sort` of the rows after the seek under the `Limit` (PostgreSQL 16), while on the 100k-row table the index scan holds ten rows from the end; both are pinned by a test and stated in the module doc (PR 8). PostgreSQL's `fetch_page` pulls the page's rows from the connection one at a time and parses each as it arrives, so the page's text is no longer a term of section 7 (the record's 83 MB at the worst shape); the item channel and the decoded page are unchanged; with that change the PostgreSQL worst-shape run peaked at 790 MiB (`VmHWM`) in 16.5 s against the head's 905 MiB in 18.2 s, and its 1M run's peak was 4.0 MiB over the 100k run's (section 7, PR 8). `run_page_bounds` fails at once, naming the key yielded twice, when a stream has yielded more items than the table holds, so an inclusive seek fails in 3 s where it ran until another test's lock wait gave out (PR 8). SQLite's run-ahead is five rows, not four: sqlx's worker steps and copies the next row before its `send` on the four-row channel blocks, 160 MiB at the line cap (section 7, PR 8). The allocator: the peak over idle of the worst-expansion run grows with the number of large rows under glibc's default arenas and is flat under `MALLOC_ARENA_MAX=1`, which is documented for operators (the admin guide's `[backup]` rows, the deployment guide's service unit, section 7) rather than set from the binary; section 7's memory paragraph is the sizing rule (the terms at the line cap, about 490 MiB per job on PostgreSQL and 650 MiB on SQLite, times the allocator's factor, times `max_concurrent_jobs`), measured at 27.7 MB rows, 1,000 small rows first, 12, 24 and 48 such rows back to back, in two series of six runs each on one SQLite binary: 466, 588 and 760 MiB over idle with the default arenas and 481, 452 and 480 MiB under `MALLOC_ARENA_MAX=1` on a quiet host, then 707, 700 and 1,281 MiB and 452, 480 and 455 MiB on the final build with the kill-point runs beside it (the terms' sum at that row 545 MiB; runs of the PR's first build on a loaded host reached 1,096 MiB at 48 rows), which replaces this item's earlier "the peak did not move between 12 and 24 worst rows" (section 7, PR 8). The #618 cells run on PostgreSQL and SQLite; the RSS acceptance states its 112 MiB bound and MongoDB's 500k run; the testing summary says the backup kill points are run by hand with the posted script, a `before_manifest` point added (the three documents and the data file under the prefix, no manifest, FAILED `owner dead` and the prefix deleted as at `after_data_file`) and the SIGTERM exit timed by polling (55 ms on SQLite, not the script's 5 s sleep); 5.1 step 1 names the store-health check in its order, before the permit; the record is committed as `docs/design/15-backup-restore-bench-pr8.md` and cited by that path, with its SQLite cache explanation corrected to `cache_size = -32000` (31 MiB per connection, `sqlite_util.rs`) and its two 12-row rows labelled by build; the differences document's backup-creation figures are the record's and its restore-interruption row says the prefix of a SQLite row goes once the dead owner's heartbeat is stale, not in the first cycle; the usage guide's backups row says RestoreTableFromBackup restores in-catalog backups only this release and the upgrade manual says a store-backed backup is listed, described and deleted but its restore answers the ValidationException until the restore job lands (section 5.1, 7, 10, testing summary, PR 8). The harness: `tests/test_abac.py`'s backup cell waits for AVAILABLE before DeleteBackup; the job-bound cell's two diagnostics say what each case is; the fidelity suite reads `/health` once and checks that the server's `backup_store` component is the kind `EXTENDDB_TEST_BACKUP_STORE` names and healthy (absent for `none`), the `none` branch of the vector cell captures the ARN of a create accepted against expectation, `_failed_rows` decodes Rust's `Debug` escapes, a cell pins the whole key of a refused item with a UUID hash key and a ten-digit `N` range key in the log line and in `backup list --failed`; the CI start steps of the PostgreSQL, SQLite and S3 jobs wait for the `backup_store` component to be healthy, the pytest lines carry `-rP` so a passed cell's printed branch reaches the log, and the S3 job dumps moto's log on failure and names what its by-name step covers (PR 8). Also: `validate_restorable_item` takes `&RestorableLimits` (the item size, hash key size, range key size and attribute name size of `LimitsConfig`), not `max_item_size_bytes` alone, so the restore's "every item against the target's limits" (5.3 step 5) is the same call; every untrusted text its error carries (a key value, an attribute name, a path segment, the number the wire's reason echoes) is bounded on its own at 4096 bytes of the text the message writes (a key value at 12,288, six times the hash key limit, so a key of control characters is whole too; 4096 is a round figure that keeps a path segment to four times the longest key attribute name), with a 1024-byte prefix and the length for a longer one, so a key of any legal size appears whole in the log line and the FAILED reason; every such text, the attribute path and the echoed number included, is written as a JSON string's content, with U+0085, U+2028 and U+2029 escaped as `\u0085`, `\u2028`, `\u2029` beside the control characters, so neither a control character nor a Unicode line separator in a tenant's item can add a line to the server's log, for a byte-oriented reader and for one that breaks lines the Unicode way (Python's `str.splitlines`) alike; an item whose data file line would exceed `MAX_LINE_BYTES` under a raised `max_item_size_bytes` is refused by the job's feeder through the same count-and-log path (`RestorableItemReason::LineTooLong`, whose reason names the cap, the line's own length being unknown once the count stops), the line measured on the blocking pool for items over the default item limit only, through a counting writer that holds no line, and such an item's validation (the number rule over every member, CPU work proportional to the item's size) run on the blocking pool with it, an item within the default validated on the feeder's own task, so the encoder's own refusal is never the first to see it; a put that fails while the snapshot stream is between pages ends the pipeline at once (the uploader cancels a token the feeder selects on), so the FAILED write does not wait for the stream; the completion log line carries the stream's item count; a job the shutdown cancelled records `BackupJobs{cancelled}` and neither `WorkerLastSuccess` nor `WorkerErrorCount`; `BackupJobDuration` is microseconds from the snapshot's open to the end of the job's last catalog or store call (the AVAILABLE write, or the FAILED write and the prefix's deletion); the reconciler's `CycleReport` counts a dead-CREATING resolution whose write matched no row as left to its owner (`creating_left_to_owner`), and its cycle line counts the dead legacy restores and the FAILED-record targets apart; `reconciler.rs`'s module doc states the FAILED-prefix duty's timing as the code has it, in this cycle on PostgreSQL, MongoDB and Cassandra and once the dead owner's last beat is stale on SQLite (5.1 steps 4 and 5, section 9, PR 8). (2026-11-01, PR 8 as measured and corrected, the row bound's memory.) The sentence above that the over-read happens "once" was false on a table whose large rows are scattered among small ones: with the bound sized from the previous page alone, a large row set the next bound to 4, four small rows set the one after back to 1000, and that page met large rows again, so the 1000-row page recurred every other page for the whole table (5,555 rows read for 1,000 kept, 13.4 GB of server temp files, on a 1,000-row table with 30 such rows; on the release binary the same bytes took 1.9 times as long and 3.2 times the temp files by key layout alone). The bound is now sized from the largest row of the last pages that together hold at least a row bound's worth of rows (`RowBoundMemory`), so it stays at the floor while large rows keep arriving within that span and returns to the row bound a row bound's worth of small rows after the last large one: 1,901 rows read for 1,000 kept and 3.9 GB on the probe (a memory of four pages read the same 5,555 as one page, since four pages of four rows are sixteen rows), and the interleaved table's backup 54.6 s and 3.2 GB of temp against 72.0 s and 9.2 GB, the grouped table unchanged. The bound that remains, stated as it is: the page that follows a row bound's worth of small rows reads up to `n` rows of up to the line cap, at most once per row bound's worth of small rows, and nothing in the window form stops the server at the budget. A page query that does stop there, a recursive walk of the primary key one index probe per row, was measured and not taken: it read exactly the rows it returned on every shape (the interleaved table in 38.8 s with 2.5 GB of temp) and cost one probe per row, 23.1 s against 11.9 s for the 1M-row backup; the record carries both. `the_row_bound_remembers_large_rows_among_small_ones` pins the memory's figures on a 300-row table with nine such rows (201 rows read past the cut over 53 round trips), `the_page_query_is_an_index_scan_on_every_key_shape` pins the index scan on the six key shapes (hash only, `S`, `N` and `B` range keys, a multi-part range key, a legacy table on the database collation, each at `n` 16 and 1000, first page and a page after a key, and the stream over 20,000 rows), and `page_sql_for` takes the collation flag `open_stream` decides, so the legacy table's query is tested as the engine writes it (section 3, 7, PR 8). Section 7's rule is scoped: the terms are at the default `max_item_size_bytes`, where no stored item's line exceeds the cap, and on an instance that raises it they are at the largest line a stored item expands to; the feeder's measurement of an item over the default limit holds no line (a counting writer that stops at the cap), where the earlier text said its buffer was bounded by the cap and the code held the whole line (section 7, PR 8). The one-arena factor is measured on both backends: the PostgreSQL worst shape at 12 rows peaked at 340 MiB `VmHWM` under `MALLOC_ARENA_MAX=1` against 750 to 788 MiB under the default arenas, 0.82 of the terms at that row on PostgreSQL (413 MiB, no run-ahead), and the setting's cost on the request path is measured: 3.5 to 9.2% on SQLite (GetItem, Query, Scan, paged Scan) and about 2% on PostgreSQL (the scans, 2.1%), so the deployment guide's service unit carries the line commented out with the condition and the cost, and section 7 states both (section 7, PR 8). 4(e)'s unknown-outcome rule reads as `write_status` has it: each unknown outcome is followed by a re-read, the write is sent once more at most, and after the second write a re-read that shows the row in the target status counts as matched, anything else as matching no row; the review checklist's bullet says the same (4(e), PR 8). The PR 8 duty list marks the resume of a dead RUNNING restore as PR 9a's, this release's reconciler logging such a record once; section 10's table gains PR 8's rows (the admin guide's `[backup]` rows and `[backup.store]` section, the deployment guide's service unit, the upgrade manual's asynchronous-create and rollback paragraphs, the design README's row 15b); the PR 9a lead-in names `RestorableLimits`; the ratios are given to two places (0.86 to 2.35; 2.35 at 48 rows); the SIGTERM exit is 55 ms in every statement; the record attributes the PostgreSQL worst run's idle to the restarted process's startup work on a table of 27.7 MB rows, not to the 1M load (sections 4, 7, 10, PR 8, PR 9a). (2026-11-01, PR 8 as corrected.) Section 3's parenthesis on the PostgreSQL cut, the PR 8 bullet, its two paging acceptance cells and the Interfaces paragraph's `n` sentence described the recursive walk that was measured and not taken (one row per step, no row read past the cut, two index probes) and the rule before the memory (`n` from the previous page's largest row, the over-read "once"); each is corrected to the shipped form: the three-layer window page with up to `n - 1` rows read past the cut, `n` sized from the memory, the first paging cell naming the record's `EXPLAIN (ANALYZE, BUFFERS)` figure (4 read, 1 past the cut at the floor) and `the_row_bound_remembers_large_rows_among_small_ones` (201 over 53 round trips), the `EXPLAIN` cell the plans the tests pin (`Limit` over one `Index Scan`; the bitmap scan with a `Sort` near the end of a small table) (section 3, PR 8). The over-read's statement gains the second page, which follows the first page's 16 rows and asks for the row bound: the over-read that remains is the second page, and then the page that follows a row bound's worth of small rows, each up to `n` rows of up to the line cap, said the same way in the PR 8 page paragraph, the module doc, the test's doc and the record (the earlier sentence of this item said "the page that follows a row bound's worth of small rows" alone); the floor's cost is stated beside it: after a page holding a row at the line cap the bound stays at the floor for a row bound's worth of rows, 250 pages of four, whatever those rows hold, so a table whose large rows are rarer than one per row bound pays 250 round trips per large row where one page of memory pages it at the row bound (2,585 page queries against 38 on a 20,000-row table with a 27 MB row at every 1,100th key, 25.6 against 25.1 s, 1.06 against 1.09 GB of temp; the record), and the record's "changes no page of a small-row table" reads "of a table with no large row"; the one-page-memory figure the test's doc, the record and the PR text cite is 306 rows past the cut over 7 pages (section 7, PR 8). 4(e) said the PostgreSQL reconciler "reads the heartbeat alone and never probes the lock" and named "the one liveness rule the three share": PostgreSQL's `claim_dead_backup` tries the advisory lock before it tests the heartbeat, so it takes over only a row whose heartbeat is stale and whose lock is free; a stopped owner (SIGSTOP, a paused VM) whose session lives keeps its row until the session ends or it finishes, and a vanished host's lock is released when the server's keepalives give up (about a minute with `CLAIM_KEEPALIVES`), so the takeover waits for the later of the two; 4(e) and 5.1 step 4 now say so (the SIGSTOP sentence scoped to MongoDB and Cassandra, with the PostgreSQL outcome: the job resumes and completes under its own generation), the trait doc of `claim_dead_backup`, the admin guide's `stale_job_after_secs` row and the differences document's dead-owner clause gain "on PostgreSQL, once the owner's claim session has ended as well", and item 27's "the heartbeat alone" stands for PR 6's `remove_dead_creating_backup_row`, which `backup purge` still uses (4(e), 5.1 step 4, PR 8). The one-arena cost on PostgreSQL reads "about 2% (the scans, 2.1%)" in section 7, this item, the admin guide, the deployment guide and the PR text, the record's scans being −2.1%; section 7 labels the base of each ratio (the peak over idle on SQLite, the `VmHWM` on PostgreSQL, about 0.47 over idle); the admin guide's `max_concurrent_jobs` row restates section 7's sizing rule (1.2 times the terms per job under `MALLOC_ARENA_MAX=1`, no bound measured under the default arenas, the PostgreSQL figure as `VmHWM`, 2.35) and its `stale_job_after_secs` row the dead-row outcomes (AVAILABLE with a complete manifest, FAILED without, a restore job's target removed); the upgrade manual's rollback paragraph names `backup sync --adopt-store`, as 4(c) requires (section 7, 10, PR 8). `EXTENDDB_TEST_BACKUP_STORE` is set by every CI job that runs a suite creating backups, the Rust integration jobs included: the SQLite Rust integration job's file-backed server has the defaulted root, so its CreateBackup is the asynchronous one, and `tests/rust`'s backup tests assumed the synchronous create and left their backups; the suite now waits for AVAILABLE, deletes what it makes and reads the variable as the Python suite does, and `run-tests`' usage, `CONTRIBUTING.md` and the developer test guide name the variable and the refusal (section 10, PR 8). The PR 8 kill-point list names `before_manifest`, and section 10's limits-doc row the rewritten "Backup store" row (PR 8). 5.1 step 6 says a job cancelled after its manifest put completes AVAILABLE, the manifest being the commit record, where it said a cancelled job can never complete (5.1 step 6, PR 8). diff --git a/docs/design/README.md b/docs/design/README.md index f9806b336..cc403a0ac 100755 --- a/docs/design/README.md +++ b/docs/design/README.md @@ -20,6 +20,7 @@ This directory contains the authoritative design documents for ExtendDB (extendd | 12 | [Auth / Authz Cache](12-auth-authz-cache.md) | In-process cache for IAM data (credentials, policies, tags, table key info); companion to 05 and 06 | | 13 | [MongoDB Storage](13-storage-mongodb.md) | MongoDB storage backend: engine and catalog traits over the official driver, replica-set transactions | | 15 | [Backup and Restore](15-backup-restore.md) | Overview of the off-host backup series: the invariants, the per-operation contracts and a short section per later PR; the format, store and snapshot documentation lives with their crates (`crates/core/src/backup_format`, `crates/backup-store`, `crates/storage/src/snapshot.rs`) | +| 15b | [Backup memory and duration record, PR 8](15-backup-restore-bench-pr8.md) | The measurements section 7 and revision item 29 of the design cite: peak RSS and duration of the 100k, 1M, worst-expansion and incompressible backups per backend, the allocator series, the PostgreSQL page's cost | ## How to Use These Docs diff --git a/docs/differences-from-dynamodb.md b/docs/differences-from-dynamodb.md index 7e9415ce8..b6397bb44 100755 --- a/docs/differences-from-dynamodb.md +++ b/docs/differences-from-dynamodb.md @@ -87,7 +87,7 @@ adaptation when switching between ExtendDB and the real service. | Vector index update propagation | Eventually consistent, the same model as a GSI | Matches DynamoDB. Maintenance is queued on the same propagation queue as async GSIs, so a search immediately after a write may not see it. Governed by the same `index_propagation_delay_ms` setting; unlike a GSI there is no per-index override. A value of 0 applies maintenance inline in the write's own transaction, which is stricter than the service and exists so a test can assert steady state without waiting. Inline additionally requires the table's propagation queue to be empty: while rows queued during an index build are still draining, a new write queues behind them instead of overtaking them, so a brief search-after-write window exists even at 0 until the queue drains. That window is ordering, not data loss; the write is applied, in order, by the queue worker. That zero behaves differently while an index is still building, and the two backends differ: **PostgreSQL** applies inline only to an index that is already `ACTIVE`, and defers a write to a building index whatever the delay says, because the write must not reach the index ahead of the backfill's older snapshot of the same item; **SQLite** does not check the status on this path, so with a zero delay a write during a backfill is applied inline and bypasses the hold that keeps the two ordered. The SQLite behaviour is tracked as a defect (F-20), not intended, and it is reachable only with the zero delay, which is a test setting. | | `SearchVectors` score at extreme magnitudes (PostgreSQL backend only) | Returns the true distance as a number for any vector of finite components | The score is bounded to a finite value instead, and the bound is not a measured service answer. pgvector accumulates distances in single precision, so magnitudes far below `f32::MAX` overflow inside the extension: Euclidean above about 9.2e18, dot product above about 1.8e19, and cosine at both ends, above about 1.8e19 and below about 3.7e-23, which is where a component's single-precision square rounds to zero. A non-finite score cannot be serialised as JSON at all, so the result is bounded in SQL: `1e308` for an overflowed distance, `-1e308` for an overflowed negated inner product, which the score contract negates so a client sees `1e308` in `Score`, and 1.0 for a cosine that comes back NaN. Ranking is unaffected at the overflow end, because each bound sits at the end its metric overflows towards, so the farthest row stays farthest and the most similar stays most similar; two rows that both overflow tie, and the tie breaks on the base key. At the underflow end cosine loses resolution rather than being bounded, and which side is tiny decides how much. For a tiny **query** vector, its norm underflows while the inner product usually does not, so the quotient is an infinity that pgvector clamps and the reported distance collapses to one of 0, 1 or 2 following the sign of the inner product, with the 1.0 substitute firing only when the vectors are exactly orthogonal and the quotient is therefore 0/0 (measured). For a tiny **stored** vector it is worse: the stored norm is computed in single precision and reaches zero, so the zero-vector guard fires and that row reports 1.0 at every angle, parallel included. A corpus of tiny embeddings therefore loses ranking altogether rather than losing resolution. For a tiny query vector, by contrast, ranking still separates nearer-than-orthogonal from farther and loses resolution only within each half. The SQLite backend owns its own arithmetic, computes in double precision, and reports the true value, which is why this row is scoped to PostgreSQL. | | Vector index deletion window | `UpdateTable` Delete leaves the index in `DELETING` long enough to observe, then removes it | No observable `DELETING` window on either backend: the catalog row is removed inside the `UpdateTable` transaction, so a `DescribeTable` immediately afterwards already omits the index. The index's data table is dropped after that commit, in a separate transaction, and the two backends handle a failure there differently. **PostgreSQL** treats it as best effort, because the data table lives in a different database entirely: a failure is logged and skipped rather than failing the request. **SQLite** propagates it, so a failed drop returns an error from an `UpdateTable` whose catalog change has already committed, which is the more surprising outcome of the two: the index is gone from the catalog and the caller saw a failure. So an operator debugging a leftover `_ddb_vec_*` table should look for that warning rather than assume the delete was incomplete. | -| Backing up and restoring a table that has vector indexes | Restores the table with its vector indexes intact: the configuration survives, items keep their vector attributes, and `SearchVectors` works as soon as the table is `ACTIVE` (measured) | Not supported. On PostgreSQL and SQLite, the two backends with vector indexes, `CreateBackup` of such a table is refused with `ValidationException` "table has vector index(es); restoring a table with vector indexes is not supported by this storage backend", and no backup is created, because a restore cannot carry the index data across. A backup of a vector table taken before catalog 0.0.4 is refused at restore on PostgreSQL with the same message naming the backup; on SQLite such a backup restores without its vector indexes. A backup of a table with no vector indexes restores normally. | +| Backing up and restoring a table that has vector indexes | Restores the table with its vector indexes intact: the configuration survives, items keep their vector attributes, and `SearchVectors` works as soon as the table is `ACTIVE` (measured) | Partial. With a configured `[backup.store]`, `CreateBackup` of such a table succeeds on PostgreSQL and SQLite, the two backends with vector indexes: the backup is written to the store and its manifest records the index specifications, and `DescribeBackup` and `ListBackups` report it as any other backup (wire test on both backends). Without a store (the section absent on PostgreSQL, or `kind = "disabled"`), where `CreateBackup` writes the in-catalog format, it is refused with `ValidationException` "table has vector index(es); restoring a table with vector indexes is not supported by this storage backend" and no backup is created, because that format cannot carry the index data across. `RestoreTableFromBackup` of any backup in the store answers `ValidationException` "Backup is in a backup store; this release restores in-catalog backups only" until a later release's restore job, which restores a vector table into a backend with vector support and refuses it into one without. A backup of a vector table taken before catalog 0.0.4 is refused at restore on PostgreSQL with the same message naming the backup; on SQLite such a backup restores without its vector indexes. A backup of a table with no vector indexes restores normally. | | `SearchVectors` endpoint | Served only on `search-dynamodb..amazonaws.com`. The standard `dynamodb..amazonaws.com` endpoint answers the same request with HTTP 400 `UnknownOperationException` ("This operation is not supported by this endpoint"); every control-plane and item operation stays on the standard endpoint. Signing is unchanged either way (service name `dynamodb`, target prefix `DynamoDB_20120810`) | Served on the same endpoint as every other operation, so a client is pointed at one ExtendDB endpoint for all of them. Two consequences worth knowing: an SDK that resolves a separate search hostname from its endpoint ruleset needs its endpoint overridden to reach ExtendDB, and ExtendDB does **not** reproduce the service's refusal, so a test asserting `UnknownOperationException` for vector search on the base endpoint passes against Amazon DynamoDB and fails here. | | `SearchVectors` result order for equal distances | Measured unstable: three identical searches returned tied rows in three different orders, and a top-k that truncates a tie group keeps an arbitrary subset of it | Deterministic on both backends, by different means. **PostgreSQL** sorts explicitly on the base table's full primary key after the score, so the order is a property of the query. **SQLite** issues no `ORDER BY` and resolves ties by scan order through a stable top-k, so its order is a property of the plan rather than something the query guarantees. Either way a client re-issuing an identical search sees the same order, which is stricter than the service rather than divergent in outcome. Do not rely on the two backends agreeing on which subset of a truncated tie group they keep. | | Vector indexes on the MongoDB backend | Vector indexes and `SearchVectors` are available | Not supported at all. That backend provides no vector search implementation, so the engine's capability gate refuses `CreateTable` and `UpdateTable` carrying `VectorIndexes` and every `SearchVectors` request, with the same capability message a PostgreSQL deployment without pgvector returns. Every other vector row in this document describes the PostgreSQL and SQLite backends. | @@ -116,13 +116,14 @@ Behaviour stated as measured, and every number below, was observed with release | Multi-part base table keys (preview) | Not supported | A backup of a table created with `enable_multipart_keys` is refused at restore with `ValidationException`; the item paths address such tables by their first HASH and RANGE attribute only *(from the code and storage-level tests; not checked on a running server)* | | `DeleteBackup` and `DeleteTable` during a restore | `DeleteTable` on the `CREATING` target is `ResourceInUseException`. `DeleteBackup` during a restore was not measured | While the target is `CREATING`, `DeleteTable` on it is `ResourceInUseException` on all four backends, as on the service. `DeleteBackup` on its source backup is `BackupInUseException` on all four: that refusal is ExtendDB's contract, not measured against the service. Both succeed once the target is `ACTIVE`. How the restore and `DeleteBackup` are ordered differs *(from the code)*. PostgreSQL and SQLite register the restore in the transaction that creates the target, holding the backup row, so a concurrent `DeleteBackup` is ordered strictly before (the restore then finds no backup) or after (it finds the restore). MongoDB runs the restore's registration and `DeleteBackup`'s claim each as one session transaction that first increments a lock field on the backup's catalog document while it is `AVAILABLE`, so the two conflict and one orders wholly before the other: a `DeleteBackup` that wins leaves the registration no `AVAILABLE` backup (`BackupNotFoundException`), a registration that wins makes the `DeleteBackup` `BackupInUseException`. Cassandra uses two lightweight transactions on different partitions: the target's row is inserted first (`IF NOT EXISTS`, which reserves the name), then the restore is registered on the backup's row (`IF backup_status = 'AVAILABLE'`), which `DeleteBackup` claims on the same partition; a concurrent `DescribeTable` can therefore see a `CREATING` target for a restore that then fails with `BackupNotFoundException`, after which the target is removed. On MongoDB `DeleteBackup` writes the backup `DELETING` before it drops the data and `DELETED` after; a `DeleteBackup` whose request fails or is cancelled after the drop started leaves the backup `DELETING`, which `DescribeBackup`, `ListBackups` and a restore treat as gone (`BackupNotFoundException`), and a retry of `DeleteBackup` finishes the drop and answers `DELETED` *(from the code and storage-level tests)*. MongoDB reads the catalog from the primary with read concern `local` regardless of the connection string, which the lock and the job records rely on; the connection string must use `readPreference=primary` (any other read preference is refused at startup). The write concern follows the connection string and must be `majority` (the server default): a connection string with `w=1` is refused at startup, because a `DELETING` write or a job record acknowledged under `w=1` just before a primary failover can be rolled back | | Concurrent restores of one backup | A second `RestoreTableFromBackup` of a backup while a restore from it is still `CREATING` is `BackupInUseException` (measured) | The same on all four backends: the second restore is refused with `BackupInUseException` and creates nothing; the check runs inside the transaction that registers the restore (PostgreSQL, with the backup row locked `FOR UPDATE`; SQLite), inside the claim on the backup document followed by a check for a `CREATING` target of the backup (MongoDB), or inside the lightweight transaction on the backup row (Cassandra), so two simultaneous requests cannot both pass. Once the first target is `ACTIVE` the backup restores again; on MongoDB the check reads majority-committed data (at the default transaction read concern) while `DescribeTable` reads at `local`, so a `RestoreTableFromBackup` or `DeleteBackup` issued within the majority-commit lag after `DescribeTable` first shows the target `ACTIVE` can still be `BackupInUseException` and succeeds when retried (the lag is the journal flush plus any concurrent batch write still holding an earlier oplog slot; measured at 7 to 12 ms, once in about 2,000 observations, with a 3,000-item insert running beside the write). A restore of a backup whose `DeleteBackup` is in progress answers `BackupNotFoundException` on every backend (the service's answer was not measured), by two mechanisms: on MongoDB and Cassandra the delete marks the backup `DELETING` before it drops the data, and the restore finds no `AVAILABLE` backup; on PostgreSQL and SQLite the restore waits for the delete's transaction and then finds no backup. Earlier builds accepted concurrent restores of one backup on PostgreSQL, SQLite, and Cassandra, and on MongoDB after a 2 s retry | -| Restore interrupted by the server stopping | Managed by the service | The copy runs in the server process, so a stopped process leaves its target `CREATING` with a RUNNING restore job record whose heartbeat stops. The engine's reconciler, once at startup and then every `backup.reconcile_interval_secs` (default 60 s), marks the record FAILED and removes the target once the heartbeat is older than `backup.stale_job_after_secs` (default 300 s) on the database's clock, on every backend alike, logs the removal with the table name, and then lets `DeleteBackup` and a new restore into the name succeed *(from the code)*. On SQLite a restart ages every record of the stopped process at once (one process serves the file), so the first cycle removes it without the stale window. Until then `DeleteTable` and `DeleteBackup` are refused as during a running restore. Not covered *(from the code)*: on MongoDB, a target whose base copy had finished has its record DONE and is left alone by the reconciler; the index backfill worker finishes it after a restart, and if that backfill fails on every attempt the worker retries it every 5 s with a warning and never gives up, so the target stays `CREATING` (and `DeleteTable` and `DeleteBackup` stay refused) until the cause is fixed or the target's catalog document is removed by hand (no failing case was found to reproduce this). A target left `CREATING` by a build before this release is backfilled by the catalog migration as a RUNNING record with a heartbeat at the epoch, so the first cycle removes it. Heartbeats and the stale window are measured on the database server's clock, so the application hosts' clocks play no part | +| Restore interrupted by the server stopping | Managed by the service | The copy runs in the server process, so a stopped process leaves its target `CREATING` with a RUNNING restore job record whose heartbeat stops. The engine's reconciler, once at startup and then every `backup.reconcile_interval_secs` (default 60 s), marks the record FAILED and removes the target once the heartbeat is older than `backup.stale_job_after_secs` (default 300 s) on the database's clock, on every backend alike, logs the removal with the table name, and then lets `DeleteBackup` and a new restore into the name succeed *(from the code)*. On SQLite a restart ages every record of the stopped process at once (one process serves the file), so the first cycle removes it without the stale window. Until then `DeleteTable` and `DeleteBackup` are refused as during a running restore. Not covered *(from the code)*: on MongoDB, a target whose base copy had finished has its record DONE and is left alone by the reconciler; the index backfill worker finishes it after a restart, and if that backfill fails on every attempt the worker retries it every 5 s with a warning and never gives up, so the target stays `CREATING` (and `DeleteTable` and `DeleteBackup` stay refused) until the cause is fixed or the target's catalog document is removed by hand (no failing case was found to reproduce this). A target left `CREATING` by a build before this release is backfilled by the catalog migration as a RUNNING record with a heartbeat at the epoch, so the first cycle removes it. Heartbeats and the stale window are measured on the database server's clock, so the application hosts' clocks play no part. The same reconciler resolves a store-backed `CreateBackup` the server stopped (one reconciler with one duty list, in place of per-backend sweeps): a `CREATING` backup row whose owner is dead (on PostgreSQL and MongoDB a heartbeat older than `stale_job_after_secs`, on PostgreSQL once the owner's claim session has ended as well; on SQLite, where one process serves the file, every `CREATING` row after a restart and, at runtime, a row whose in-process slot no task holds, so a job task that died is resolved by the next cycle without a restart) is taken over under a fresh owner generation and becomes `AVAILABLE` when a complete, verified manifest is under its prefix in the configured store (the process died between the manifest write and the status write) and `FAILED` otherwise, never continuing the upload; the objects of a `FAILED` backup are deleted once its heartbeat is older than `stale_job_after_secs`; a `DELETING` backup's deletion is finished; a `DELETED` backup's prefix that still lists objects is swept; and `FAILED` and `DELETED` rows are removed once their prefix is empty and their `completed_at` is older than `failed_row_retention_secs` (7 days). Measured *(binary-level, this release)*: SIGKILL between the data file and the manifest left the row `CREATING` and the data file under the prefix, and after the restart the row was `FAILED` in the first cycle on SQLite and 46 s after the kill on PostgreSQL (`stale_job_after_secs = 45`), with the prefix deleted in the same cycle on PostgreSQL (the stale heartbeat is what made the owner dead) and once the dead owner's heartbeat was stale on SQLite, 45 s after the kill (the slot made the owner dead at once; the prefix waits for the heartbeat clock on every backend); the same at a kill between the three documents and the manifest, which leaves the documents under the prefix and no manifest; SIGKILL between the manifest write and the status write left the five objects, and the row was `AVAILABLE` with the manifest's size and count after the restart (SQLite at once, PostgreSQL after 48 s); SIGTERM during an upload left the row `FAILED` with reason `shutdown` and an empty prefix. These duties run only while the configured store is healthy, and under a maintenance claim only the dead-owner and `DELETING` duties run. A `FAILED` backup is `BackupNotFound` to clients; `extenddb backup list --failed` prints it with its reason | | Point-in-time recovery | 35-day window | Not supported. `RestoreTableToPointInTime` is refused with `ValidationException` ("Point-in-time recovery restore is not yet supported"). `UpdateContinuousBackups` is not refused: enabling point-in-time recovery succeeds, and `DescribeContinuousBackups` then reports it `ENABLED` with an `EarliestRestorableDateTime` 35 days back, although nothing is recorded and no restore to any of those times is possible. Same on all four backends | -| Where backups live | Managed storage, independent of the table | Inside the database the server already uses, so a lost database loses its backups too: on PostgreSQL in the catalog database (`backups`, `backup_items`, `backup_definitions`, `table_restores`), on SQLite in the one database file, on MongoDB as a `backups` metadata collection in the catalog database and one data collection per backup in the data database, on Cassandra as backup metadata in the catalog keyspace and item payloads in the account's keyspace. Take database-level backups for off-host copies until the backup store program ships | -| Backup creation | `CreateBackup` returns while the backup is `CREATING`; `ListBackups` and `DescribeBackup` expose that state until it becomes `AVAILABLE`. `BackupName` and `TableName` are checked as the model says: length 3 to 255 and `[a-zA-Z0-9_.-]+`, with "1 validation error detected: Value '' at 'backupName' failed to satisfy constraint: " texts, joined under "2 validation errors detected:" when both fail (measured). `TableName` may be the table's ARN (measured). A table that does not exist or is still `CREATING` is `TableNotFoundException` "Table not found: " (measured) | `CreateBackup` copies the items before it returns and returns the backup `AVAILABLE` on all four backends. On PostgreSQL, MongoDB, and Cassandra a concurrent `ListBackups` never saw it `CREATING` *(earlier build)*; on SQLite the backup row is visible as `CREATING` to other requests while the copy runs. Measured *(earlier build)*: 1.3 s for 60,000 items on PostgreSQL, 1.9 s for 60,000 on SQLite, 0.2 s for 20,000 on MongoDB, 5.1 s for 20,000 on Cassandra. MongoDB reports `BackupSizeBytes` 0. The name checks are the service's, with its texts (the length clause before the pattern clause when both fail; the service's order was not captured), counting characters, with `TableName` taking the deployment's table-name limits; before them a 200 KB name was stored and returned in every `ListBackups` page (a 500 on Cassandra) and a NUL byte was a 500 on PostgreSQL. `TableName` as an ARN (`arn:aws:dynamodb:::table/`) is accepted on all four backends: another region's resolves, another account's is `AccessDeniedException` (neither measured on the service); an SDK built from the model refuses the ARN form client-side, so only a hand-built request sends it. `TableNotFoundException` "Table not found: " for a missing or `CREATING` table on all four backends, built from the request's name (the backends' own payloads differ). A table the deployment could create but whose name no backup ARN can carry is refused with `ValidationException` where the service backs up any table it can create: a name over 255 bytes (reachable only with `[limits] max_table_name_length` raised) or made only of dots ("TableName cannot be backed up: a backup ARN cannot be formed for it: invalid backup ARN: table name of N bytes is longer than 255" / "... invalid backup ARN: table name \"...\" is only dots"), and every table under a `server.region` outside `[a-z0-9-]` (the `BackupArn` row); the checks run after the table's ACTIVE probe and before the job permit, so the refusal takes no slot | +| Where backups live | Managed storage, independent of the table | With a configured `[backup.store]` (a filesystem root, which is the default beside a SQLite database file, or an S3 bucket and prefix) on PostgreSQL, SQLite and MongoDB: under the store's root, at `/
//`, as gzip data files of DynamoDB JSON lines (`data/000001.json.gz`, ...) and the manifest documents (`extenddb-manifest.json` last, the commit record), with a `backup_index` row in the catalog as the index; a lost database loses the rows, not the bytes (a later release's `backup sync` re-adopts a root's backups into a new catalog). Without a store, and on Cassandra always, inside the database the server already uses, so a lost database loses its backups too: on PostgreSQL in the catalog database (`backups`, `backup_items`, `backup_definitions`, `table_restores`), on SQLite in the one database file, on MongoDB as a `backups` metadata collection in the catalog database and one data collection per backup in the data database, on Cassandra as backup metadata in the catalog keyspace and item payloads in the account's keyspace; take database-level backups for off-host copies there | +| Backup creation | `CreateBackup` returns while the backup is `CREATING`; `ListBackups` and `DescribeBackup` expose that state until it becomes `AVAILABLE`. `BackupName` and `TableName` are checked as the model says: length 3 to 255 and `[a-zA-Z0-9_.-]+`, with "1 validation error detected: Value '' at 'backupName' failed to satisfy constraint: " texts, joined under "2 validation errors detected:" when both fail (measured). `TableName` may be the table's ARN (measured). A table that does not exist or is still `CREATING` is `TableNotFoundException` "Table not found: " (measured) | With a configured `[backup.store]` on PostgreSQL, SQLite and MongoDB, `CreateBackup` takes the table's snapshot and writes the catalog row in the request and returns the backup `CREATING`; a job then uploads it and the backup becomes `AVAILABLE`, which `DescribeBackup` and `ListBackups` show as on the service (`BackupSizeBytes` is 0 while `CREATING` and the compressed size once `AVAILABLE`; `ItemCount` is the table's count as last counted while `CREATING`, which is 0 for a table the stats pass has not reached yet, such as one loaded seconds before CreateBackup, and the backup's exact count once `AVAILABLE`). The backup holds every write acknowledged before the request was received, which is stricter than the service's one-minute window. A backup the job cannot finish (a store failure, an item no restore can load, a shutdown) is `FAILED`, which is `BackupNotFound` to clients, and its objects are deleted (the restore-interruption row). Measured *(this release, 16-core host, filesystem store, the record in `docs/design/15-backup-restore-bench-pr8.md`)*: 100,000 items of a composite-key table with two GSIs `AVAILABLE` in 1.85 s on PostgreSQL and 1.44 s on SQLite, 1,000,000 in 36.1 s and 12.45 s. Without a store, and on Cassandra always, `CreateBackup` copies the items before it returns and returns the backup `AVAILABLE`. On PostgreSQL, MongoDB, and Cassandra a concurrent `ListBackups` never saw an in-catalog backup `CREATING` *(earlier build)*; on SQLite the backup row is visible as `CREATING` to other requests while the copy runs. Measured *(earlier build, in-catalog)*: 1.3 s for 60,000 items on PostgreSQL, 1.9 s for 60,000 on SQLite, 0.2 s for 20,000 on MongoDB, 5.1 s for 20,000 on Cassandra. MongoDB's in-catalog backups report `BackupSizeBytes` 0. The name checks are the service's, with its texts (the length clause before the pattern clause when both fail; the service's order was not captured), counting characters, with `TableName` taking the deployment's table-name limits; before them a 200 KB name was stored and returned in every `ListBackups` page (a 500 on Cassandra) and a NUL byte was a 500 on PostgreSQL. `TableName` as an ARN (`arn:aws:dynamodb:::table/`) is accepted on all four backends: another region's resolves, another account's is `AccessDeniedException` (neither measured on the service); an SDK built from the model refuses the ARN form client-side, so only a hand-built request sends it. `TableNotFoundException` "Table not found: " for a missing or `CREATING` table on all four backends, built from the request's name (the backends' own payloads differ). A table the deployment could create but whose name no backup ARN can carry is refused with `ValidationException` where the service backs up any table it can create: a name over 255 bytes (reachable only with `[limits] max_table_name_length` raised) or made only of dots ("TableName cannot be backed up: a backup ARN cannot be formed for it: invalid backup ARN: table name of N bytes is longer than 255" / "... invalid backup ARN: table name \"...\" is only dots"), and every table under a `server.region` outside `[a-z0-9-]` (the `BackupArn` row); the checks run after the table's ACTIVE probe and before the job permit, so the refusal takes no slot | | `DeleteBackup` response | The full `DescribeBackup` description with `BackupStatus` `DELETED`, including `SourceTableDetails.ProvisionedThroughput` (a required member) and `SourceTableFeatureDetails` (measured) | The same on all four backends: the engine reads the backup's recorded definition before the delete and fills the description as `DescribeBackup` does, with `BackupStatus` `DELETED`. A backup whose definition this build cannot read or that does not parse is still deleted and described without feature details (logged at warn), which means a damaged `PROVISIONED` backup is described with on-demand `ProvisionedThroughput` `{0, 0}`: ExtendDB's choice, deletability over the throughput member of a row that is being removed, where `DescribeBackup` on the same row is an `InternalServerError`; a failure to reach the database fails the request and deletes nothing. | | `DescribeBackup` when the definition cannot be read | Not applicable | A definition row that does not parse, a renamed column, or a revoked privilege (`Internal` from the backend) is an `InternalServerError`, as it is for `RestoreTableFromBackup` on the same read; only a definition version this build does not know is described without `SourceTableFeatureDetails`. | -| Consistency of a backup | Point-in-time snapshot of the table | **PostgreSQL** and **SQLite** take the items from one snapshot. How *(from the code)*: PostgreSQL holds the table's catalog row shared from the first definition read until the data snapshot (`REPEATABLE READ`, pinned by a read of the table before the row is released) is taken, so the definition recorded is the one in force for the items read; the catalog and the data are separate databases, so this is a barrier, not one cross-database snapshot. SQLite reads all items in one deferred read transaction opened while the engine's write lock was held for the definition read, so writers continue during the copy; that long reader pins WAL checkpointing, so the WAL grows for the duration of the backup by the size of the backup rows written plus any concurrent writes. **MongoDB** (`$out` from the live collection) and **Cassandra** (paging the live table) are not point-in-time: items copied early reflect the table as it was when the copy started, items copied late reflect writes made during the copy. A backup can therefore hold a write without an earlier write it followed, a state the table was never in. Measured *(earlier build)* with a writer that updated an early item and then a late item, pair after pair, during `CreateBackup`: 16 of 66 pairs on MongoDB and 123 of 800 on Cassandra had the late write without the early one; 0 on PostgreSQL (of 228) and SQLite (of 230). Back up a MongoDB or Cassandra table while it is not being written if the backup must be consistent | +| Consistency of a backup | Point-in-time snapshot of the table | **PostgreSQL** and **SQLite** take the items from one snapshot, and so does **MongoDB** with a configured `[backup.store]`. How *(from the code)*: with a store, the request opens the table snapshot (`SnapshotEngine`, the module docs of `crates/storage/src/snapshot.rs` and the backends' `snapshot_engine.rs` are the record): on PostgreSQL one `REPEATABLE READ` transaction holds the definition and the items on the shared layout, and on the separate layout the catalog row is held shared until the data transaction is pinned, the items then paged by the data table's primary key under that transaction, each page cut at 64 MiB of item text; on SQLite one deferred read transaction opened without the engine's write lock, paged by key with the same cut; on MongoDB a session with `readConcern: snapshot`, so items copied late reflect the same instant as items copied early. Without a store, PostgreSQL holds the table's catalog row shared from the first definition read until the data snapshot (`REPEATABLE READ`, pinned by a read of the table before the row is released) is taken, so the definition recorded is the one in force for the items read; the catalog and the data are separate databases, so this is a barrier, not one cross-database snapshot. SQLite reads all items in one deferred read transaction opened while the engine's write lock was held for the definition read, so writers continue during the copy; that long reader pins WAL checkpointing, so the WAL grows for the duration of the backup by the size of the backup rows written plus any concurrent writes. **MongoDB without a store** (`$out` from the live collection) and **Cassandra** (paging the live table) are not point-in-time: items copied early reflect the table as it was when the copy started, items copied late reflect writes made during the copy. A backup can therefore hold a write without an earlier write it followed, a state the table was never in. Measured *(earlier build)* with a writer that updated an early item and then a late item, pair after pair, during `CreateBackup`: 16 of 66 pairs on MongoDB and 123 of 800 on Cassandra had the late write without the early one; 0 on PostgreSQL (of 228) and SQLite (of 230). Back up a Cassandra table, or a MongoDB table on a server without a store, while it is not being written if the backup must be consistent | +| `CreateBackup` of a table holding an `N` value the wire did not validate | Every stored value is a valid number, and a backup holds the table as it is | With a configured `[backup.store]` the backup fails. Four write paths of earlier builds stored an `N` value such as `1E126` or `abc` without validating it (#618: `BatchWriteItem`, `TransactWriteItems`, `UpdateItem` on an absent item in the key it creates, `ImportTable`), and the store-backed job validates every item before encoding it with the rule the restore applies (`validate_restorable_item`: every `N` and `NS` member a valid number, the nesting depth, the key shape (each key attribute present, of its declared type, not empty, an `N` key normalized), the item size against the instance's `max_item_size_bytes`, the hash and range key sizes and the attribute name sizes against the instance's limits, and for an item over the default item limit the data file line against the format's 32 MiB cap), so no backup that no restore can load is reported `AVAILABLE`. After the first refusal the job reads and validates the rest of the snapshot, logs each refused item with its key, attribute path and reason beside the backup ARN (up to 1,000, then a count), and fails the backup with a reason giving the count and the first item, naming #618 only when that item's refusal is an `N` value ("3 item(s) cannot be restored; first: key {...}, attribute amount: Number overflow... (#618)"); an item over `max_item_size_bytes` (stored under a higher limit) fails it the same way with a reason naming the size and the limit and not #618. The client sees `CREATING` and then `BackupNotFound`; the reason reaches the operator (`extenddb backup list --failed`, and the log, which a tenant's support case can cite; every tenant-supplied text in the line is written as a JSON string's content, control characters and the Unicode line separators U+0085, U+2028 and U+2029 escaped, so a newline or a line separator in an attribute name or a number value cannot forge a second log record); the way out is to rewrite or delete each named item through the wire (a bad range key cannot be rewritten, since a PutItem with that key is refused; `DeleteItem` removes it). The in-catalog create (no store, Cassandra) backs such a table up and restores it as before (wire tests on PostgreSQL and SQLite for each path) | | `ListBackups` paging | At most 100 summaries per page, with `LastEvaluatedBackupArn`; `Limit` 1 to 100. `TimeRangeLowerBound`, `TimeRangeUpperBound`, and `BackupType` filter the list. `TableName` is checked as the model says (3 to 255, `[a-zA-Z0-9_.-]+`, the "N validation error(s) detected" texts). An `ExclusiveStartBackupArn` of another account is `ValidationException` "Invalid Request: The supplied ExclusiveStartBackupArn does not match the account"; of another region, or of a table other than `TableName`'s, `ValidationException` ("does not match the region", "does not match the table name predicate"); a time bound before the epoch is `ValidationException`; a lower bound above the upper bound is `ValidationException` "Time range lower bound must be less than or equal to upper bound" (all measured) | The same paging on all four backends: `Limit` outside 1 to 100 is `ValidationException`, `ExclusiveStartBackupArn` resumes after the given backup, and summaries are ordered by creation time then ARN. Summaries carry `TableId`. `TimeRangeLowerBound` (inclusive) and `TimeRangeUpperBound` (exclusive) keep backups created in that range, and `BackupType` keeps backups of that type (`USER` when absent, every type for `ALL`); every ExtendDB backup is `USER`, so `SYSTEM` and `AWS_BACKUP` list nothing. The filters apply before paging, so `Limit` and the cursor count only backups that pass. Measured on SQLite and PostgreSQL with whole-second bounds between two backups taken 2 s apart; the millisecond edges are from the engine's unit tests. The request checks above are the service's, with its texts where captured: the account text in full; the region and table-name texts complete the captured fragments on the account text's pattern ("Invalid Request: The supplied ExclusiveStartBackupArn does not match the region" / "... the table name predicate"), checked in that order after the ARN's shape; a bound before the epoch is refused with ExtendDB's own text naming the member; equal bounds list nothing. An `ExclusiveStartBackupArn` that is not in the listing (excluded by the filters, or deleted since the previous page) is placed by the creation time in its id; one whose id carries no time (the 13-digit ids of earlier builds, with or without the hex suffix: only the 14-digit form carries one) is then a `ValidationException`. A cursor in another region is accepted only when it is the stored ARN of a backup of the account (or of `TableName`), which is what `LastEvaluatedBackupArn` hands out for a backup created under an earlier `server.region`, so paging passes such a backup; any other cursor in another region is refused ("does not match the region"). If that backup is deleted between two pages, its ARN is no longer a stored one and the next page is refused the same way (a same-region cursor would be placed by its time instead); start the listing again. Under a `server.region` outside `[a-z0-9-]` the cursor is still the stored ARN: a listing of backups created under a valid region pages, and a cursor that is the ARN of a backup an earlier release created under the invalid region is refused with "Invalid Request: BackupArn is not valid". `TableName` is a table name or a table ARN, as `CreateBackup` accepts it (the service accepts both, measured on DynamoDB): the ARN form lists what the name lists, another account's ARN is `AccessDeniedException`, and the cursor's table check runs against the resolved name. A numeric `TableName` is refused with the text the service uses for a map ("Start of structure or map found where not expected"), not measured for a number. Every `ListBackups` request reads all of the account's backups (or all of the table's, with `TableName`) from storage and pages them in memory, so walking N backups 100 at a time reads about N²/100 rows in total *(from the code)* | | Backup ARN and creation time | Backup id is 14-digit zero-padded epoch milliseconds, `-`, 8 hex digits; `BackupCreationDateTime` has sub-second precision | The same shape on all four backends; `BackupCreationDateTime` is the millisecond in the ARN | | Running two servers against one MongoDB catalog | Not applicable | MongoDB uses the fixed database names `extenddb_catalog` and `extenddb_data` whatever `--catalog-db` and `--data-db` say, and takes no serve lock: two servers on one replica set share one catalog and both start. Each server's reconciler acts only on a restore whose job record's heartbeat is older than `backup.stale_job_after_secs` (default 300 s) on the database's clock, so one server does not remove another live server's restore *(from the code and storage-level tests)*. Running more than one server per catalog is otherwise unsupported (see the deployment guide) | diff --git a/docs/dynamodb-limits.md b/docs/dynamodb-limits.md index eecff2545..0d7ca8106 100755 --- a/docs/dynamodb-limits.md +++ b/docs/dynamodb-limits.md @@ -128,12 +128,13 @@ Source: [AWS DynamoDB Service Quotas](https://docs.aws.amazon.com/amazondynamodb | Limit | DynamoDB Value | Status | Notes | |-------|---------------|--------|-------| -| Concurrent restores | 50 | Partial | Every backend: one server process runs at most `[backup] max_concurrent_jobs` (default 4, at most 1024) backup and restore jobs at once, `CreateBackup` and `RestoreTableFromBackup` together and not per account; a request over the bound is refused at once with `LimitExceededException` ("Too many backup and restore jobs are running on this server (at most N at once, the [backup] max_concurrent_jobs bound); retry later") and creates nothing, with no wait (storage-level and engine tests; measured on PostgreSQL with `max_concurrent_jobs = 1`: a running restore made a CreateBackup and a second restore refuse in under 60 ms). On PostgreSQL `pool_size` and `catalog_pool_size` are raised to at least `2 * max_concurrent_jobs + 10` so every admitted job has a connection. SQLite also bounds the snapshot slots `CreateBackup` and table snapshots (#395) share, a count derived from `[storage.sqlite] pool_size` (the formula is stated once, on that setting's documentation in `SqliteConfig`; 4 for the default pool) so half the pool stays free for ordinary requests; a `CreateBackup` that finds no slot free is refused at once with the same `LimitExceededException`, its parenthesis naming that bound and its count in place of `max_concurrent_jobs` ("at most N at once, the [storage.sqlite] pool_size snapshot-slot bound"); a PostgreSQL job whose dedicated claim connection the database server refuses is refused with the same exception naming the server's connection slots ("the database server has no connection slot left for the job"), with no count; the same error while `extenddb backup purge` holds the catalog's maintenance claim names the command, not a bound. A MongoDB restore request that is not registered within 60 seconds answers an internal error while the restore continues | +| Concurrent restores | 50 | Partial | Every backend: one server process runs at most `[backup] max_concurrent_jobs` (default 4, at most 1024) backup and restore jobs at once, `CreateBackup` and `RestoreTableFromBackup` together and not per account; a request over the bound is refused at once with `LimitExceededException` ("Too many backup and restore jobs are running on this server (at most N at once, the [backup] max_concurrent_jobs bound); retry later") and creates nothing, with no wait (storage-level and engine tests; measured on PostgreSQL with `max_concurrent_jobs = 1`: a running restore made a CreateBackup and a second restore refuse in under 60 ms). On PostgreSQL `pool_size` and `catalog_pool_size` are raised to at least `2 * max_concurrent_jobs + 10` so every admitted job has a connection. SQLite also bounds the snapshot slots `CreateBackup` and table snapshots (#395) share, a count derived from `[storage.sqlite] pool_size` (the formula is stated once, on that setting's documentation in `SqliteConfig`; 4 for the default pool) so half the pool stays free for ordinary requests; a server raises a smaller `pool_size` to `2 * (max_concurrent_jobs + 1)`, at most 34 (the slots are capped at 16), with a warning naming both numbers, so the slot bound refuses no job the `max_concurrent_jobs` permit admitted (no change at the defaults: a pool of 10 has 4 slots for 4 jobs); above 16 jobs a 17th concurrent job is refused by the slot. A `CreateBackup` that finds no slot free is refused at once with the same `LimitExceededException`, its parenthesis naming that bound and its count in place of `max_concurrent_jobs` ("at most N at once, the [storage.sqlite] pool_size snapshot-slot bound"); PostgreSQL bounds open table snapshots at half its (raised) pool the same way, naming `[storage.postgres] pool_size snapshot-slot`; a PostgreSQL job whose dedicated claim connection the database server refuses is refused with the same exception naming the server's connection slots ("the database server has no connection slot left for the job"), with no count; the same error while `extenddb backup purge` holds the catalog's maintenance claim names the command, not a bound. A MongoDB restore request that is not registered within 60 seconds answers an internal error while the restore continues | | `BackupName` and `TableName` on `CreateBackup`, `TableName` on `ListBackups` | Length 3 to 255, `[a-zA-Z0-9_.-]+` | Enforced | The service model's constraints, with its texts ("1 validation error detected: Value '' at 'backupName' failed to satisfy constraint: ...", "2 validation errors detected: ...; ..." when length and pattern both fail). Length is counted in characters. `BackupName` takes the model's 3 to 255; a `TableName` member takes the deployment's `[limits]` `min_table_name_length` and `max_table_name_length` (defaults 3 and 255), as `CreateTable` does, so a table the deployment can create can be listed, and can be backed up unless no backup ARN can carry its name: a name over 255 bytes (with `max_table_name_length` raised) or made only of dots can be created but `CreateBackup` refuses it with `ValidationException` "TableName cannot be backed up: a backup ARN cannot be formed for it: invalid backup ARN: table name of N bytes is longer than 255" or "... invalid backup ARN: table name \"...\" is only dots" (the service backs up any table it can create; see the differences document's backup creation row). Both members may also be the table's ARN. The value is echoed in the message as sent | | `BackupArn` | 37 to 1024 characters | Maximum enforced; minimum implied | Surrounding ASCII whitespace is trimmed first, then over 1024 bytes, not of the backup ARN shape, a control or non-ASCII character, or a region outside `[a-z0-9-]`, an account that is not digits, or a table name outside `[a-zA-Z0-9_.-]` is `ValidationException` "Invalid Request: BackupArn is not valid"; the right shape with an id that is not a backup id is "Invalid Backup ARN"; a digit-only account that is not the caller's is `AccessDeniedException`. No minimum is checked: the shortest string of the right shape is longer than 37 characters. Another region of the caller's account resolves (the server's region is tried first, then the ARN as written). `ListBackups`'s `ExclusiveStartBackupArn` must also match the account, the region, and `TableName` when given | | `ListBackups` page size | 100 | Enforced | `Limit` 1 to 100, default 100; more rows come back through `LastEvaluatedBackupArn` and `ExclusiveStartBackupArn`. Each request reads every backup of the account (or of the table, with `TableName`) and pages in memory, so walking N backups reads about N²/100 rows. `TimeRangeLowerBound`, `TimeRangeUpperBound`, and `BackupType` filter the list before paging; a bound before the epoch, or a lower bound above the upper bound ("Time range lower bound must be less than or equal to upper bound"), is `ValidationException` | -| Backup store | A backup lives in the service's own storage | N/A | `[backup.store]` selects where ExtendDB's backups will live from a later release's CreateBackup on (this release's writes the in-catalog format and the store serves restores and deletes of the backups it holds): a filesystem root (`backups/` beside a SQLite database file when the section is absent), an S3 bucket and prefix, or `disabled`. The knobs are `[backup]`: `data_file_target_bytes` 64 MiB (minimum 32 MiB), `heartbeat_interval_secs` 15, `stale_job_after_secs` 300 (at least three heartbeats), `failed_row_retention_secs` 7 days, `reconcile_interval_secs` 60, `max_concurrent_jobs` 4 per instance, `spool_path` (required with an S3 store except beside a SQLite database file). A filesystem store holds every backup's compressed size under its root, beside the database on the SQLite default, and `extenddb destroy` leaves it; a restore from an S3 store holds the compressed backup size at `spool_path` while it runs. Instances that share a catalog must share one store root; an unhealthy store refuses the restore and delete of store-backed backups (CreateBackup, which writes the in-catalog format in this release, is not refused) with `ValidationException` naming the store's kind, and the server keeps serving (`/health` reports `degraded`). See the admin guide's `[backup.store]` | -| Backup contents | Key schema, attribute definitions, items, indexes, billing mode, provisioned throughput, on-demand throughput (table and GSI level), SSE. Table class is not carried (a restore is `STANDARD`) | Partial | ExtendDB restores the same set, except a GSI's on-demand throughput (it records none per index); table class is not restored, as on the service. Streams, TTL, tags, and deletion protection are not restored, as on the service. Backups are stored inside the database the server uses; take database-level backups for off-host copies | +| Backup store | A backup lives in the service's own storage | N/A | `[backup.store]` selects where CreateBackup writes its backups on PostgreSQL, SQLite and MongoDB (Cassandra, and any backend without a store, writes the in-catalog format) and where restores and deletes of store-backed backups read: a filesystem root (`backups/` beside a SQLite database file when the section is absent), an S3 bucket and prefix, or `disabled`. The knobs are `[backup]`: `data_file_target_bytes` 64 MiB (minimum 32 MiB), `heartbeat_interval_secs` 15, `stale_job_after_secs` 300 (at least three heartbeats), `failed_row_retention_secs` 7 days, `reconcile_interval_secs` 60, `max_concurrent_jobs` 4 per instance, `spool_path` (required with an S3 store except beside a SQLite database file). A filesystem store holds every backup's compressed size under its root, beside the database on the SQLite default, and `extenddb destroy` leaves it; a restore from an S3 store holds the compressed backup size at `spool_path` while it runs. Instances that share a catalog must share one store root; an unhealthy store refuses CreateBackup where it writes to the store and the restore and delete of store-backed backups with `ValidationException` naming the store's kind, and the server keeps serving (`/health` reports `degraded`). See the admin guide's `[backup.store]` | +| Backup data file | Not specified | N/A | A data file holds one item per line as DynamoDB JSON, gzip-compressed, in key order on a table whose key schema is orderable (`item_order` in the manifest); a line is at most 32 MiB (`MAX_LINE_BYTES`, which holds the largest line a 400 KB item can expand to, 27.7 MB); a file closes at the first item that takes its uncompressed bytes to or past `[backup] data_file_target_bytes` (default 64 MiB, minimum 32 MiB, the line cap), so it exceeds the knob by less than one line. The snapshot streams on PostgreSQL and SQLite page the table by key and cut each page at 64 MiB of item text (`SNAPSHOT_FETCH_BYTES`, a constant), so a page holds at most that plus one row; MongoDB's batches are bounded by the server's 16 MiB reply | +| Backup contents | Key schema, attribute definitions, items, indexes, billing mode, provisioned throughput, on-demand throughput (table and GSI level), SSE. Table class is not carried (a restore is `STANDARD`) | Partial | ExtendDB restores the same set, except a GSI's on-demand throughput (it records none per index); table class is not restored, as on the service. Streams, TTL, tags, and deletion protection are not restored, as on the service (a store-backed backup's manifest records the stream specification and TTL for `DescribeBackup`). With a configured store on PostgreSQL, SQLite and MongoDB the backup is written under the store's root (the differences document's "Where backups live" row); without one, and on Cassandra, it is stored inside the database the server uses, so take database-level backups for off-host copies there | | Point-in-time recovery | 35-day window | N/A | `RestoreTableToPointInTime` is refused. `UpdateContinuousBackups` accepts enabling point-in-time recovery and reports a 35-day restorable window, but nothing is recorded (see differences-from-dynamodb.md) | | `RestoreTableFromBackup` overrides | `BillingModeOverride`, `ProvisionedThroughputOverride`, `GlobalSecondaryIndexOverride`, `LocalSecondaryIndexOverride`, `SSESpecificationOverride`, `OnDemandThroughputOverride`, `VectorIndexOverride` | Partial | All but `VectorIndexOverride` are applied as the service applies them: an index override list replaces the backup's indexes of that kind (empty restores none, a subset restores those, a GSI entry's throughput is the restored index's), `OnDemandThroughputOverride` is the restored table's on-demand throughput on an on-demand restore. Refused with `ValidationException` before the target is created: an index the backup does not have or a changed index definition, a throughput member contradicting the billing mode, `{0, 0}`, a `PROVISIONED` restore of a GSI without throughput ("Invalid Request: Must override ProvisionedThroughput if BillingMode is overridden to PROVISIONED for index "), a GSI entry's `OnDemandThroughput` on a `PROVISIONED` restore (on an on-demand restore it is accepted and not recorded), and `VectorIndexOverride`. See differences-from-dynamodb.md | diff --git a/docs/manuals/03-usage-guide.md b/docs/manuals/03-usage-guide.md index 861f6bba9..2f78383b2 100755 --- a/docs/manuals/03-usage-guide.md +++ b/docs/manuals/03-usage-guide.md @@ -571,7 +571,8 @@ extenddb reproduces DynamoDB error responses exactly. Common errors: | PartiQL | ✗ (not planned) | | DAX | ✗ (not applicable) | | Global Tables | ✗ (future) | -| Backups / PITR | ✗ (future) | +| On-demand backups (CreateBackup, RestoreTableFromBackup) | Partial: CreateBackup ✓ (to the configured filesystem or S3 backup store on PostgreSQL, SQLite and MongoDB; in-catalog without one and on Cassandra); RestoreTableFromBackup ✓ for in-catalog backups only, store-backed backups from a later release (their restore answers a `ValidationException` naming the store this release; the upgrade manual) | +| PITR | ✗ (future) | | Import/Export (local filesystem) | ✓ (FileSource instead of S3) | --- diff --git a/docs/manuals/05-admin-guide.md b/docs/manuals/05-admin-guide.md index 788058fff..bfd84e3c6 100755 --- a/docs/manuals/05-admin-guide.md +++ b/docs/manuals/05-admin-guide.md @@ -53,13 +53,15 @@ SIGTERM directly. Alternatively start it with ```bash curl --cacert ~/.extenddb/tls/cert.pem https://127.0.0.1:18443/health -# {"status":"healthy","components":{}} +# {"components":{},"status":"healthy"} +# (the order of the keys depends on the build: the default PostgreSQL build +# writes "status" first, the SQLite build "components" first; read the keys) # with a backup store configured: -# {"status":"healthy","components":{"backup_store":{"status":"healthy","kind":"filesystem"}}} +# {"components":{"backup_store":{"kind":"filesystem","status":"healthy"}},"status":"healthy"} # while it cannot be reached: -# {"status":"degraded","components":{"backup_store":{"status":"unhealthy","kind":"s3","reason":"unreachable"}}} +# {"components":{"backup_store":{"kind":"s3","reason":"unreachable","status":"unhealthy"}},"status":"degraded"} # right after startup, before an S3 store's first check has returned: -# {"status":"healthy","components":{"backup_store":{"status":"checking","kind":"s3"}}} +# {"components":{"backup_store":{"kind":"s3","status":"checking"}},"status":"healthy"} ``` `status` is `healthy`, or `degraded` when a component is unhealthy; the HTTP @@ -144,17 +146,17 @@ The knobs backup and restore jobs and the reconciler read; every key has a defau | Key | Default | Description | |-----|---------|-------------| -| `data_file_target_bytes` | `67108864` | Uncompressed bytes per backup data file (minimum 32 MiB); read by the store-backed backup writer | -| `heartbeat_interval_secs` | `15` | The job heartbeat timer. The in-catalog backup and restore paths of this release heartbeat every 15 s whatever this says | -| `stale_job_after_secs` | `300` | Heartbeat age after which a job's owner is dead and the reconciler fails the job and removes its restore target. Must be at least three heartbeat periods, where the period is the larger of `heartbeat_interval_secs` and the 15 s the jobs of this release heartbeat at (so 45 at the least, whatever `heartbeat_interval_secs` says); on SQLite a long write transaction delays a heartbeat, so keep a margin above that floor | -| `failed_row_retention_secs` | `604800` | How long FAILED backup rows are kept for the operator (7 days) | -| `reconcile_interval_secs` | `60` | The reconciler's cycle; it also runs once at startup | -| `max_concurrent_jobs` | `4` | Backup and restore jobs this instance runs at once (at most 1024); a request over it is `LimitExceededException`. On PostgreSQL the pool sizes are raised to `2 * max_concurrent_jobs + 10` when smaller | +| `data_file_target_bytes` | `67108864` | Uncompressed bytes per backup data file (minimum 32 MiB, the cap on one line): a data file closes at the first item that takes it to or past this, so it exceeds it by less than one line | +| `heartbeat_interval_secs` | `15` | The job heartbeat timer. The store-backed backup job heartbeats every `max(heartbeat_interval_secs, 15)` s and starts no store write unless a beat confirmed its claim within two periods, the claim's insert counting as the first beat; the in-catalog backup and restore paths heartbeat every 15 s whatever this says | +| `stale_job_after_secs` | `300` | Heartbeat age after which a job's owner is dead (on PostgreSQL, once the owner's claim session has ended as well) and the reconciler resolves the job: a dead backup job with a complete manifest under its prefix is AVAILABLE, one without is FAILED and its prefix deleted; a dead restore job is FAILED and its target removed. Must be at least three heartbeat periods, where the period is the larger of `heartbeat_interval_secs` and the 15 s the in-catalog jobs heartbeat at (so 45 at the least, whatever `heartbeat_interval_secs` says); on SQLite a long write transaction delays a heartbeat, so keep a margin above that floor | +| `failed_row_retention_secs` | `604800` | How long FAILED and DELETED backup rows of the configured store are kept once their objects are gone (7 days); `extenddb backup list --failed` prints the FAILED ones with their reason | +| `reconcile_interval_secs` | `60` | The reconciler's cycle; it also runs once at startup. Each cycle checks the store, resolves backup and restore jobs whose owner is dead, finishes deletions, deletes the objects of FAILED backups once their heartbeat is older than `stale_job_after_secs`, and removes tombstones past `failed_row_retention_secs` (the differences document's restore-interruption row lists the duties) | +| `max_concurrent_jobs` | `4` | Backup and restore jobs this instance runs at once (at most 1024); a request over it is `LimitExceededException`. On PostgreSQL the pool sizes are raised to `2 * max_concurrent_jobs + 10` when smaller; on SQLite `[storage.sqlite] pool_size` is raised to `2 * (max_concurrent_jobs + 1)`, at most 34, when its snapshot slots (`pool_size / 2 - 1`, between 1 and 16) would refuse a job this knob admits. Memory: a running backup holds about 490 MiB (PostgreSQL) or 650 MiB (SQLite) at most in the pipeline's own terms for a table of items at the 32 MiB line cap, a few tens of MiB for 200-byte items (the figures are for the default `max_item_size_bytes`, 400 KB, where no stored item's line exceeds the cap; on an instance that raises it the terms are at the largest line a stored item expands to, about 68 times the item's size for the worst shape); with glibc's default allocator the process's high-water mark grows with the number of large items a job reads (measured 760 MiB over idle at 48 items of 27.7 MB on a quiet host and 1,281 MiB on a loaded one, 1.39 and 2.35 times the terms at that item size) because the items' strings are allocated on one thread and freed on another and the per-thread arenas keep that memory. Set `MALLOC_ARENA_MAX=1` in the server's environment (the deployment guide's service unit carries the line, commented out) on a host that backs up large items: the peak is then the terms' and does not grow with the table (measured 452 to 481 MiB over idle at 12, 24 and 48 items of 27.7 MB on SQLite, in two series on two hosts' loads, and 340 MiB `VmHWM` (about 200 MiB over idle) at 12 items on PostgreSQL); one arena serializes the worker threads' allocations on one lock and costs the request path a few percent (measured 3.5 to 9.2% on GetItem, Query and Scan on SQLite and about 2% on PostgreSQL (the scans, 2.1%), where the request path waits on the database on PostgreSQL). Size the host for 1.2 times the terms (about 590 MiB on PostgreSQL, 780 MiB on SQLite) times this knob under `MALLOC_ARENA_MAX=1`; without it the peak grows with the large rows a job reads and no bound was measured (1,281 MiB over idle at 48 items of 27.7 MB is the largest seen) | | `spool_path` | (unset) | Restore download area for an S3 store, created `0700` with files `0600`, holding a `.lock` another instance on the host refuses. Required with an S3 store, except on file-backed SQLite where it defaults to `backup-spool/` beside the database file. Refused inside or containing a filesystem store root, and when it is or contains the database file's directory | #### [backup.store] -The store CreateBackup's backups will live in, and the one restores and deletes of store-backed backups read. The section's keys, defaults and the refusals of a bad value are the backup store crate's (`crates/backup-store/src/config.rs`); the table below is what the server adds. One definition of the shape exists, so an unknown key anywhere under `[backup]` is refused at load. +The store CreateBackup writes to on PostgreSQL, SQLite and MongoDB, and the one restores and deletes of store-backed backups read. The section's keys, defaults and the refusals of a bad value are the backup store crate's (`crates/backup-store/src/config.rs`); the table below is what the server adds. One definition of the shape exists, so an unknown key anywhere under `[backup]` is refused at load. | Key | Default | Description | |-----|---------|-------------| @@ -162,9 +164,9 @@ The store CreateBackup's backups will live in, and the one restores and deletes | `filesystem.path` | `backups/` beside a SQLite database file | The root directory of a filesystem store. It must exist, be owned by the extenddb user with mode `0700`, and sit under directories owned by root or that user which others cannot write to; `validate` refuses anything else at startup. A relative path, like a relative `spool_path`, is resolved against the working directory the server was started from (the daemon's own working directory is `/`) | | `s3.bucket`, `s3.prefix`, `s3.region`, `s3.endpoint`, `s3.force_path_style`, `s3.allow_insecure_endpoint`, `s3.upload_part_size_bytes`, `s3.max_concurrent_uploads` | | The S3 store; see the crate's `config` module doc. Credentials come from the AWS SDK's chain and are never in the file | -When the section is absent, the backend decides: on file-backed SQLite it is a filesystem store at `backups/` beside the database file, created `0700` if missing, whose validation or identity-check failure is a logged warning that leaves the store unconfigured rather than a startup error or an unhealthy store while the catalog has no backup rows for that root (so an upgrade never stops a server that never configured backups, and CreateBackup keeps working on one); once the catalog has rows for the default root, the root has been used and is handled as a configured one: a failure of its identity check or manifest probe is an unhealthy store (reported on `/health`, its rows' restores and deletes refused, `backup purge` refusing on it rather than reporting no store), and a refusal of its `open` or validation a startup error naming the root; on PostgreSQL, MongoDB, Cassandra and an in-memory SQLite database there is no default and the store is unconfigured, logged at startup. `kind = "disabled"` leaves it unconfigured on purpose on any backend. Without a configured store, CreateBackup writes the in-catalog backup format and logs a deprecation warning naming `[backup.store]` on each call; with one configured, this release's CreateBackup still writes the in-catalog format (the store-backed writer is a later release's) and the store serves restores and deletes of the backups it holds. Backups of either format stay listable, describable, restorable and deletable. +When the section is absent, the backend decides: on file-backed SQLite it is a filesystem store at `backups/` beside the database file, created `0700` if missing, whose validation or identity-check failure is a logged warning that leaves the store unconfigured rather than a startup error or an unhealthy store while the catalog has no backup rows for that root (so an upgrade never stops a server that never configured backups, and CreateBackup keeps working on one); once the catalog has rows for the default root, the root has been used and is handled as a configured one: a failure of its identity check or manifest probe is an unhealthy store (reported on `/health`, its rows' restores and deletes refused, `backup purge` refusing on it rather than reporting no store), and a refusal of its `open` or validation a startup error naming the root; on PostgreSQL, MongoDB, Cassandra and an in-memory SQLite database there is no default and the store is unconfigured, logged at startup. `kind = "disabled"` leaves it unconfigured on purpose on any backend. Without a configured store, CreateBackup writes the in-catalog backup format and logs a deprecation warning naming `[backup.store]` on each call. With one configured, CreateBackup on PostgreSQL, SQLite and MongoDB writes the backup to the store: the request takes the table's snapshot, writes the catalog row and answers `CREATING`, and a job of the server uploads gzip data files and the manifest under `/
//` below the root and marks the row `AVAILABLE` (the job's log lines carry the ARN: `backup : started, items in the table as last counted by the stats pass, key-ordered` (or `unordered`, for a table whose key shape or column collation has no format order) at the start, the stats pass's count, which is 0 on a table the pass has not reached, and `backup : AVAILABLE, items, bytes written in s` at the end, with the backup's own count); Cassandra, which has no table snapshot, keeps the in-catalog create with a store configured. Backups of either format stay listable, describable and deletable, and in-catalog backups restorable; the restore of a store-backed backup is a later release's and answers `ValidationException` until then. -Startup opens the configured store and, for a filesystem store, validates it and checks its identity before the listener is up (bounded by 5 s: a mount that stops answering after the root is opened leaves the server starting with the store `checking`; a root that does not answer at startup holds the server before the listener, as the database directory would, with the port bound and silent); an S3 store's validation and identity check are requests to the endpoint, and run after the listener is up, in the reconciler's first cycle, so an endpoint that stalls never holds the data plane down (the component is `checking` until the first result, and the operations that touch the store refuse meanwhile as they do while it is unhealthy). A configuration error, every one decided without a request to the store, stops startup with the key, path or bucket in the message: a root that does not exist or fails the owner-only rule, a filesystem root another user can reach, a filesystem root that is or contains the database directory, a bucket name the SDK routes to S3 Express, a bad part size or prefix, an insecure endpoint, an S3 store without `spool_path` on a backend with no default, a `spool_path` that breaks its placement rule, that another live instance holds or that exists with a mode other than `0700`. A transport failure (no region, no credentials, an endpoint that refuses connections or does not answer within the check's bound), a missing or denied bucket, an I/O failure of a filesystem root's probe write or resolution (a full or read-only disk, a stale network handle), or a failed identity check leaves the server running with the store unhealthy: the problem is logged once at error, `/health` reports the `backup_store` component unhealthy (status `degraded`), the restore and delete of a store-backed backup are refused with a `ValidationException` that names the store's kind and the problem's category and not its root, the reconciler acts on no row of the store, and the check runs again each `reconcile_interval_secs` until it passes, at which point the server logs `is healthy again` and resumes without a restart. Each check is bounded (5 s for a filesystem root, 30 s for a bucket): a store that has stopped answering costs a reconciler cycle that bound and nothing more, the cycle's other duties run as before, and the abandoned check is logged; on a filesystem root the abandoned check's call stays parked on one thread until the mount answers, no later cycle starts a check while it is (logged each cycle, the store staying `unreachable`), and `stop` or SIGTERM ends the server within 5 s with that thread still parked. CreateBackup is not refused by an unhealthy store in this release: it writes the in-catalog format, which touches no store; the store-backed create of a later release is. +Startup opens the configured store and, for a filesystem store, validates it and checks its identity before the listener is up (bounded by 5 s: a mount that stops answering after the root is opened leaves the server starting with the store `checking`; a root that does not answer at startup holds the server before the listener, as the database directory would, with the port bound and silent); an S3 store's validation and identity check are requests to the endpoint, and run after the listener is up, in the reconciler's first cycle, so an endpoint that stalls never holds the data plane down (the component is `checking` until the first result, and the operations that touch the store refuse meanwhile as they do while it is unhealthy). A configuration error, every one decided without a request to the store, stops startup with the key, path or bucket in the message: a root that does not exist or fails the owner-only rule, a filesystem root another user can reach, a filesystem root that is or contains the database directory, a bucket name the SDK routes to S3 Express, a bad part size or prefix, an insecure endpoint, an S3 store without `spool_path` on a backend with no default, a `spool_path` that breaks its placement rule, that another live instance holds or that exists with a mode other than `0700`. A transport failure (no region, no credentials, an endpoint that refuses connections or does not answer within the check's bound), a missing or denied bucket, an I/O failure of a filesystem root's probe write or resolution (a full or read-only disk, a stale network handle), or a failed identity check leaves the server running with the store unhealthy: the problem is logged once at error, `/health` reports the `backup_store` component unhealthy (status `degraded`), CreateBackup (where it writes to the store) and the restore and delete of a store-backed backup are refused with a `ValidationException` that names the store's kind and the problem's category and not its root, the reconciler acts on no row of the store, and the check runs again each `reconcile_interval_secs` until it passes, at which point the server logs `is healthy again` and resumes without a restart. Each check is bounded (5 s for a filesystem root, 30 s for a bucket): a store that has stopped answering costs a reconciler cycle that bound and nothing more, the cycle's other duties run as before, and the abandoned check is logged; on a filesystem root the abandoned check's call stays parked on one thread until the mount answers, no later cycle starts a check while it is (logged each cycle, the store staying `unreachable`), and `stop` or SIGTERM ends the server within 5 s with that thread still parked. CreateBackup on Cassandra is not refused by an unhealthy store: its in-catalog create touches no store. The store identity. The root carries `extenddb-store.json`: a random store id and the id of the catalog that owns the root. Startup creates it when the root has no file and the catalog has no rows for the root, and otherwise reads it and compares: a file naming another catalog, a store id that differs from the one the catalog's rows for the root carry, a root with no file while rows exist, or rows carrying more than one id each mark the store unhealthy. Instances that share a catalog must share one store root (one S3 bucket and prefix, or one shared mount): host-local roots configured at the same path give each instance different bytes under one root label, so each check also `head`s the manifest of the newest AVAILABLE backup and marks the store unhealthy when it is absent, which is what a second instance with a local copy of the directory sees. The file is left by `backup purge` and by every deletion; remove it only when retiring the root, or after `extenddb destroy`: a catalog re-created with `init` has a new id, and the old root's file names the old one, so the re-initialised server reports the store owned by another catalog until the file is removed or a new root configured (on the default SQLite root, that is the logged warning above and CreateBackup keeps the in-catalog path). While the catalog has no rows for a root, two host-local directories at one path cannot be told apart (each instance creates its own file and id); point every instance at one shared root before the first backup. The identity file is read up to 64 KiB; a larger object under its key is reported unreadable. @@ -927,6 +929,8 @@ JSON metrics endpoint with DynamoDB CloudWatch-style metric names and dimensions ```bash curl --cacert ~/.extenddb/tls/cert.pem https://127.0.0.1:18443/health # {"status":"healthy","components":{}} +# (the key order depends on the build; the earlier Health Check section +# shows the components a backup store adds) ``` The `components` object carries the `backup_store` component when a backup diff --git a/docs/manuals/06-developer-test-guide.md b/docs/manuals/06-developer-test-guide.md index 4e97f24b0..3eefa6ea2 100755 --- a/docs/manuals/06-developer-test-guide.md +++ b/docs/manuals/06-developer-test-guide.md @@ -150,6 +150,20 @@ devtools/run-tests --extenddb --pytest --filter test_put_item 1. A running extenddb server: `./target/debug/extenddb serve --config extenddb.toml` 2. `EXTENDDB_TEST_ENDPOINT=https://localhost:18443` 3. `EXTENDDB_ADMIN_PASSWORD=` +4. `EXTENDDB_TEST_BACKUP_STORE=`, required for `--pytest` + (and so for `--all`) and read by the Rust integration suite's backup + tests: where the server under test writes its backups. `filesystem` for a + server with a `[backup.store]` filesystem root, which a file-backed SQLite + server without the section has by default (`backups/` beside its database + file); `s3` for an S3 store; `none` for a server without a store + (`kind = "disabled"`, or PostgreSQL, MongoDB and Cassandra without the + section). With a configured store CreateBackup is asynchronous and this + release refuses to restore its backups, so the backup tests derive their + expected outcome from the variable and fail rather than guess when it is + unset; `run-tests --pytest` refuses before any test runs with + `error: --extenddb --pytest requires EXTENDDB_TEST_BACKUP_STORE to be set.` + The suites check the value against the server's `/health` `backup_store` + component, so a wrong value fails too. The `run-tests` script automatically: - Performs health check on the target endpoint @@ -185,8 +199,8 @@ The `run-tests` script automatically: The primary test suite is in Python, testing extenddb through the same AWS SDK interface that customers use. ```bash -# Via run-tests script (recommended) -devtools/run-tests --extenddb --pytest +# Via run-tests script (recommended; EXTENDDB_TEST_BACKUP_STORE as above) +EXTENDDB_TEST_BACKUP_STORE=filesystem devtools/run-tests --extenddb --pytest # Run in parallel (uses 1/3 of CPU cores by default) devtools/run-tests --extenddb --pytest --parallel diff --git a/docs/manuals/07-upgrade-manual.md b/docs/manuals/07-upgrade-manual.md index 9bcb91627..85901f6bd 100755 --- a/docs/manuals/07-upgrade-manual.md +++ b/docs/manuals/07-upgrade-manual.md @@ -6,7 +6,7 @@ This release versions the PostgreSQL and SQLite catalogs by a migration ledger and carries catalog migrations 1 to 5, which `extenddb migrate --list` prints. **Every PostgreSQL and SQLite deployment created by a 0.1.x release runs `extenddb migrate --yes` once** before a server of this release starts on it: the server refuses a catalog with pending migrations. Migration 1 brings the catalog of every 0.1.x release to the v1.0 schema. Cassandra and MongoDB keep their version strings and both move to catalog 0.0.5 (Cassandra `0.0.4 -> 0.0.5` through `V006__backup_catalog_index.cql`, MongoDB `0.0.2 -> 0.0.5` through its step `002_backup_catalog_index`), the same change as migration 5: run `extenddb migrate --yes` once on each with every pre-upgrade server stopped, as [the section below](#cassandra-and-mongodb-catalog-005) says. -Configure `[backup.store]` before the release that stops creating in-catalog backups. In this release CreateBackup writes the in-catalog format whether or not a store is configured, and logs a deprecation warning naming `[backup.store]` on each call when none is; a later release's CreateBackup writes to the store and refuses without one. `kind = "disabled"` under `[backup.store]` is the opt-out that keeps a deployment on the in-catalog format on purpose and silences the warning. On file-backed SQLite an absent section is a filesystem store at `backups/` beside the database file, created `0700` at startup; that root must be owned by the extenddb user with mode `0700`, under directories owned by root or that user which others cannot write to. A default root that breaks the rule (a database file in a group-writable directory, say), or whose `extenddb-store.json` names another catalog (a catalog re-created by `destroy` and `init` under the old root), is logged at startup as a warning (`backup store: the default root ... failed its check`) and leaves the store unconfigured, not unhealthy, so the upgrade never stops a server that never configured backups and CreateBackup keeps working on it; the same root configured explicitly is a startup error or an unhealthy store. The warning applies only while the catalog has no backup rows for the default root (an upgrade, a fresh deployment, a catalog re-created by `destroy` and `init`): once a server has written rows for it, the default root is handled as a configured one, so its backups never vanish behind a warning (a failed identity check is an unhealthy store, reported on `/health`; a root that can no longer be opened is a startup error naming it). Instances that share a catalog must share one store root, an S3 bucket and prefix or a shared mount: a second instance pointed at a host-local directory under the same configured path starts with the store unhealthy (`/health` reports `backup_store` unhealthy; the admin guide's `[backup.store]` section says what the identity file and the manifest probe check). `extenddb destroy` leaves the store; `extenddb backup purge --confirm` is what deletes its backups, and a catalog re-created with `init` needs the old root's `extenddb-store.json` removed or a new root, since the file names the old catalog. +Configure `[backup.store]` before the release that stops creating in-catalog backups. From this release CreateBackup writes to the configured store on PostgreSQL, SQLite and MongoDB, and is asynchronous there as on the service: the response is `CREATING` and the backup becomes `AVAILABLE` once the upload ends, so a client that deletes a backup it just created has to poll DescribeBackup first (the previous release returned `AVAILABLE`). A store-backed backup is listed, described and deleted like any other, but its restore is a later release's: until the restore job lands, RestoreTableFromBackup of a backup in the configured store answers `ValidationException` ("is in a backup store; this release restores in-catalog backups only"), while in-catalog backups, those of the previous release included, restore as before; a deployment that needs to restore the backups it takes this release stays on the in-catalog format (`kind = "disabled"`, below) until then. Without a store CreateBackup writes the in-catalog format and logs a deprecation warning naming `[backup.store]` on each call; a later release's CreateBackup refuses without one. Cassandra writes the in-catalog format with or without a store. `kind = "disabled"` under `[backup.store]` is the opt-out that keeps a deployment on the in-catalog format on purpose and silences the warning. On file-backed SQLite an absent section is a filesystem store at `backups/` beside the database file, created `0700` at startup; that root must be owned by the extenddb user with mode `0700`, under directories owned by root or that user which others cannot write to. A default root that breaks the rule (a database file in a group-writable directory, say), or whose `extenddb-store.json` names another catalog (a catalog re-created by `destroy` and `init` under the old root), is logged at startup as a warning (`backup store: the default root ... failed its check`) and leaves the store unconfigured, not unhealthy, so the upgrade never stops a server that never configured backups and CreateBackup keeps working on it; the same root configured explicitly is a startup error or an unhealthy store. The warning applies only while the catalog has no backup rows for the default root (an upgrade, a fresh deployment, a catalog re-created by `destroy` and `init`): once a server has written rows for it, the default root is handled as a configured one, so its backups never vanish behind a warning (a failed identity check is an unhealthy store, reported on `/health`; a root that can no longer be opened is a startup error naming it). Instances that share a catalog must share one store root, an S3 bucket and prefix or a shared mount: a second instance pointed at a host-local directory under the same configured path starts with the store unhealthy (`/health` reports `backup_store` unhealthy; the admin guide's `[backup.store]` section says what the identity file and the manifest probe check). `extenddb destroy` leaves the store; `extenddb backup purge --confirm` is what deletes its backups, and a catalog re-created with `init` needs the old root's `extenddb-store.json` removed or a new root, since the file names the old catalog. A backup that a 0.1.x release took can restore without its global and local secondary indexes: a SQLite backup taken by 0.1.14 records only the table's key schema and attribute definitions, and its restore creates the table with its items and no index. Take a new backup of each table after upgrading to v1.0, and keep the 0.1.x backups for their items only. @@ -41,9 +41,11 @@ Exit status 6 means the migration lock, or a running server, was still held when A migration is additive when the previous release serves the catalog correctly with it applied, and breaking otherwise. Migrations 1 to 5 are breaking: do not run a 0.1.x server against a migrated catalog. A 0.1.x binary that writes to a migrated catalog (its `migrate --yes`, or a SQLite dev server) leaves it WrittenByOlderRelease. +A backup of a table that holds an `N` value the wire did not validate fails on the store-backed path where the previous release backed it up. Four write paths stored such a value (`1E126`, `abc`) unvalidated (#618: BatchWriteItem, TransactWriteItems, UpdateItem on an absent item in the key it creates, ImportTable), and the store-backed job validates every item with the rule the restore applies, so no backup that no restore can load is reported `AVAILABLE`. This meets file-backed SQLite without a `[backup.store]` section (the defaulted root) and any deployment once a store is configured. The client sees `CREATING` and then `BackupNotFound`; the reason reaches the operator only: `extenddb backup list --failed` prints the FAILED row with the count of refused items and the first one, and the server log names each refused item's key and attribute path beside the backup ARN (up to 1,000 per backup), which a tenant's support case can cite (every tenant-supplied text in the line is written as a JSON string's content, control characters and the Unicode line separators U+0085, U+2028 and U+2029 escaped, so a newline or a line separator in an attribute name or a number value cannot forge a second log record). The way out is to rewrite or delete each named item through the wire, which validates the value (a bad range key cannot be rewritten, since a PutItem with that key is refused; `DeleteItem` removes it); the in-catalog create of a server without a store still backs such a table up. + Migration 5 adds the catalog index of backups kept in a backup store (`backup_index`), turns each restore registration into a restore job record (`table_restores` gains its status, kind, claim and progress columns), adds the maintenance claim record and writes the catalog's random id. Stop every server built before this migration before running `extenddb migrate --yes`: such a server neither reads `backup_index` nor marks the restore registrations it writes, so a restore it ran during the upgrade would be owned by nobody. A restore in flight at that stop is left as a RUNNING job record whose owner is dead; the first reconciler pass of the new server (within `backup.reconcile_interval_secs`, default 60 s) marks it FAILED and removes its CREATING target, so the name is free again and the table has to be restored anew. -A rollback to the previous binary needs two things: the catalog restored from the pre-upgrade dump (the previous binary refuses a catalog carrying migration 5, exit status 4), and the `[backup]` section removed from the configuration file, which the previous binary refuses as an unknown field before it reaches the catalog (`unknown field 'backup'`). No backup store exists yet in this release, so the dump loses nothing a server wrote outside the catalog. +A rollback to the previous binary needs two things: the catalog restored from the pre-upgrade dump (the previous binary refuses a catalog carrying migration 5, exit status 4), and the `[backup]` section removed from the configuration file, which the previous binary refuses as an unknown field before it reaches the catalog (`unknown field 'backup'`). The dump predates every backup this release wrote to the store, so a restored catalog has no `backup_index` row for them while the store keeps their bytes under its root; the previous binary serves neither (it does not read the store), and a later release's `backup sync --adopt-store` re-adopts them (the store's identity file names the pre-rollback catalog, so the re-upgraded catalog finds the store unhealthy until it is adopted; PR 10). In-catalog backups taken after the dump are lost with the rows, as before. ### Cassandra and MongoDB: catalog 0.0.5 diff --git a/docs/manuals/11-deployment-guide.md b/docs/manuals/11-deployment-guide.md index ff20018b8..86709f8b5 100755 --- a/docs/manuals/11-deployment-guide.md +++ b/docs/manuals/11-deployment-guide.md @@ -251,6 +251,13 @@ User=extenddb Group=extenddb Restart=on-failure RestartSec=5 +# On a host that backs up tables of large items, one malloc arena keeps a +# backup job's high-water mark at the pipeline's terms whatever the number of +# large items it reads (the admin guide's [backup] max_concurrent_jobs row +# gives the figures). It costs the request path a few percent (3.5 to 9.2% +# on GetItem, Query and Scan on SQLite, about 2% on PostgreSQL (the scans, +# 2.1%)), so the line is left to the operator: +#Environment=MALLOC_ARENA_MAX=1 [Install] WantedBy=multi-user.target @@ -287,7 +294,7 @@ pool_size = 50 # Increase for higher concurrency Ensure PostgreSQL `max_connections` accommodates the total pool size plus overhead. -Backup and restore work is bounded per server by `[backup] max_concurrent_jobs` (default 4): `CreateBackup` and `RestoreTableFromBackup` jobs together, on every backend, and a request over the bound is refused at once with `LimitExceededException` and creates nothing (there is no wait). On PostgreSQL each running job holds one pooled connection for its copy plus one dedicated connection outside the pool for its claim, and startup raises `pool_size` and `catalog_pool_size` to at least `2 * max_concurrent_jobs + 10` with a warning naming both numbers (the admin guide's `[backup]` table). On SQLite a `CreateBackup` also holds its snapshot connection for its duration, and a server admits at most `pool_size / 2 - 1` of them at once (between 1 and 16; 4 for the default `pool_size` of 10), so that half the pool stays free for ordinary requests; a `CreateBackup` over that bound is refused at once with the same `LimitExceededException`. A MongoDB restore request that is not registered within 60 seconds answers an internal error while the restore goes on in the background; the MongoDB driver does not enforce `waitQueueTimeoutMS`, so the admission bound is what keeps a flood of restores from holding every pooled connection. +Backup and restore work is bounded per server by `[backup] max_concurrent_jobs` (default 4): `CreateBackup` and `RestoreTableFromBackup` jobs together, on every backend, and a request over the bound is refused at once with `LimitExceededException` and creates nothing (there is no wait). On PostgreSQL each running job holds one pooled connection for its copy plus one dedicated connection outside the pool for its claim, and startup raises `pool_size` and `catalog_pool_size` to at least `2 * max_concurrent_jobs + 10` with a warning naming both numbers (the admin guide's `[backup]` table). On SQLite a `CreateBackup` also holds its snapshot connection for its duration, and a server admits at most `pool_size / 2 - 1` of them at once (between 1 and 16; 4 for the default `pool_size` of 10), so that half the pool stays free for ordinary requests, and raises a smaller `pool_size` to `2 * (max_concurrent_jobs + 1)`, at most 34, so that bound refuses no job `max_concurrent_jobs` admits (the admin guide's `max_concurrent_jobs` row; above 16 jobs the slot refuses the seventeenth); a `CreateBackup` over that bound is refused at once with the same `LimitExceededException`. A MongoDB restore request that is not registered within 60 seconds answers an internal error while the restore goes on in the background; the MongoDB driver does not enforce `waitQueueTimeoutMS`, so the admission bound is what keeps a flood of restores from holding every pooled connection. ### PostgreSQL Tuning diff --git a/extenddb.sample.toml b/extenddb.sample.toml index 34d07a562..d4ff16b12 100755 --- a/extenddb.sample.toml +++ b/extenddb.sample.toml @@ -179,7 +179,7 @@ # Backup and restore jobs and the reconciler. Every key has a default; the # section may be left out. stale_job_after_secs must be at least three # heartbeat periods, the larger of heartbeat_interval_secs and the 15 s the -# jobs of this release heartbeat at (45 at the least); max_concurrent_jobs is +# in-catalog jobs heartbeat at (45 at the least); max_concurrent_jobs is # at most 1024; on PostgreSQL pool_size and catalog_pool_size are raised to # at least 2 * max_concurrent_jobs + 10. # data_file_target_bytes = 67108864 # Uncompressed bytes per data file (minimum 32 MiB) @@ -193,9 +193,10 @@ # # it defaults to backup-spool/ beside the database file # [backup.store] -# Where CreateBackup's backups will live, from a later release (this release's -# CreateBackup writes in-catalog backups with any store, and the store serves -# restores and deletes of the backups it holds). Absent on file-backed SQLite: a filesystem +# Where CreateBackup writes its backups on PostgreSQL, SQLite and MongoDB +# (CreateBackup answers CREATING and a job of the server uploads; Cassandra +# writes in-catalog backups with or without a store), and where restores and +# deletes of store-backed backups read. Absent on file-backed SQLite: a filesystem # store at backups/ beside the database file. Absent on PostgreSQL, MongoDB, # Cassandra and an in-memory SQLite database: no store, and CreateBackup # writes in-catalog backups with a deprecation warning. kind = "disabled" diff --git a/tests/rust/src/backup_arn_scoping.rs b/tests/rust/src/backup_arn_scoping.rs index 42e5a2905..6dc8714f6 100644 --- a/tests/rust/src/backup_arn_scoping.rs +++ b/tests/rust/src/backup_arn_scoping.rs @@ -12,8 +12,11 @@ //! //! These tests drive the ARN directly rather than provisioning a second account, //! so they exercise the resolution rule without needing a second set of -//! credentials. +//! credentials. Every backup they create is waited to AVAILABLE before it is +//! described or deleted (`backup_restore::create_available_backup`: a +//! store-backed ExtendDB creates asynchronously) and deleted at the end. +use crate::backup_restore::create_available_backup; use crate::test_base::*; use aws_sdk_dynamodb::types::{ AttributeDefinition, BillingMode, KeySchemaElement, KeyType, ScalarAttributeType, @@ -126,14 +129,7 @@ async fn describe_backup_with_own_arn_still_resolves() { let table = format!("BackupScopeOwn_{}", ts()); make_table(&table).await; - let create = c - .create_backup() - .table_name(&table) - .backup_name("scope-own") - .send() - .await - .unwrap(); - let arn = create.backup_details().unwrap().backup_arn().to_string(); + let arn = create_available_backup(c, &table, "scope-own").await; let resp = c.describe_backup().backup_arn(&arn).send().await.unwrap(); assert_eq!( @@ -159,16 +155,12 @@ async fn describe_backup_with_own_nonexistent_arn_is_backup_not_found() { let table = format!("BackupScopeMiss_{}", ts()); make_table(&table).await; - let create = c - .create_backup() - .table_name(&table) - .backup_name("scope-miss") - .send() - .await - .unwrap(); - let real = create.backup_details().unwrap().backup_arn().to_string(); + let real = create_available_backup(c, &table, "scope-miss").await; // Same account/region/table prefix, a backup id that was never issued. - let missing = format!("{}/00000000000000-deadbeef", real.rsplit_once('/').unwrap().0); + let missing = format!( + "{}/00000000000000-deadbeef", + real.rsplit_once('/').unwrap().0 + ); let err = c .describe_backup() @@ -194,18 +186,16 @@ async fn describe_backup_after_delete_is_backup_not_found() { let table = format!("BackupScopeDel_{}", ts()); make_table(&table).await; - let create = c - .create_backup() - .table_name(&table) - .backup_name("scope-del") - .send() - .await - .unwrap(); - let arn = create.backup_details().unwrap().backup_arn().to_string(); + let arn = create_available_backup(c, &table, "scope-del").await; c.delete_backup().backup_arn(&arn).send().await.unwrap(); - let err = c.describe_backup().backup_arn(&arn).send().await.unwrap_err(); + let err = c + .describe_backup() + .backup_arn(&arn) + .send() + .await + .unwrap_err(); assert_eq!( err_code(&err), Some("BackupNotFoundException"), @@ -214,6 +204,7 @@ async fn describe_backup_after_delete_is_backup_not_found() { c.delete_table().table_name(&table).send().await.ok(); } + #[tokio::test] async fn backup_ids_are_not_derived_from_the_timestamp_alone() { let c = client(); @@ -222,14 +213,7 @@ async fn backup_ids_are_not_derived_from_the_timestamp_alone() { let mut arns = Vec::new(); for i in 0..3 { - let create = c - .create_backup() - .table_name(&table) - .backup_name(format!("scope-id-{i}")) - .send() - .await - .unwrap(); - arns.push(create.backup_details().unwrap().backup_arn().to_string()); + arns.push(create_available_backup(c, &table, &format!("scope-id-{i}")).await); } for arn in &arns { diff --git a/tests/rust/src/backup_restore.rs b/tests/rust/src/backup_restore.rs index 65b2f24d3..c086a545a 100755 --- a/tests/rust/src/backup_restore.rs +++ b/tests/rust/src/backup_restore.rs @@ -2,12 +2,104 @@ // SPDX-License-Identifier: Apache-2.0 //! Backup and restore integration tests — mirrors Java `BackupRestoreTests`. +//! +//! The service creates a backup asynchronously; so does ExtendDB with a +//! configured `[backup.store]` (CreateBackup answers CREATING and a job +//! uploads it). A backup still CREATING refuses DeleteBackup +//! (`BackupInUseException`) and restore, so every test that goes on to +//! describe, delete or restore a backup it just created waits for AVAILABLE +//! first ([`wait_for_backup_available`]), and deletes the backups it made so +//! the server's store does not keep them. Where a backup lives decides what +//! RestoreTableFromBackup answers this release; the tests read it from +//! `EXTENDDB_TEST_BACKUP_STORE` ([`backup_store`]), as the Python suite does. use crate::test_base::*; use aws_sdk_dynamodb::types::{ AttributeDefinition, BillingMode, KeySchemaElement, KeyType, PointInTimeRecoverySpecification, ScalarAttributeType, }; +use aws_sdk_dynamodb::Client; +use std::time::{Duration, Instant}; + +/// Where the server under test writes its backups, from +/// `EXTENDDB_TEST_BACKUP_STORE`: `filesystem` or `s3` (a configured +/// `[backup.store]`, under which a backup is in the store and this release +/// refuses to restore it, PR 9a adding the restore) or `none` (the +/// in-catalog create, which restores). Against the service the variable is +/// not read (the restore runs). Against ExtendDB a test whose outcome depends +/// on it fails when it is unset, with the text the Python fixture prints, +/// rather than accept either outcome. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum BackupStore { + Filesystem, + S3, + None, +} + +pub fn backup_store() -> BackupStore { + const VALUES: [&str; 3] = ["filesystem", "s3", "none"]; + if is_real_dynamodb() { + return BackupStore::None; + } + let raw = std::env::var("EXTENDDB_TEST_BACKUP_STORE").unwrap_or_default(); + match raw.trim() { + "filesystem" => BackupStore::Filesystem, + "s3" => BackupStore::S3, + "none" => BackupStore::None, + "" => panic!( + "EXTENDDB_TEST_BACKUP_STORE is unset; set it to one of {VALUES:?} to say where the \ + server under test writes its backups" + ), + other => panic!( + "EXTENDDB_TEST_BACKUP_STORE is {other:?}; set it to one of {VALUES:?} to say where \ + the server under test writes its backups" + ), + } +} + +/// Poll DescribeBackup until the backup is AVAILABLE. Any other status than +/// CREATING is a failure: a backup that fails is BackupNotFound, never +/// FAILED, on the wire, so the describe itself errors then. +pub async fn wait_for_backup_available(c: &Client, arn: &str) { + let deadline = Instant::now() + Duration::from_secs(120); + loop { + let resp = c + .describe_backup() + .backup_arn(arn) + .send() + .await + .unwrap_or_else(|e| { + panic!("describe_backup of {arn} while waiting for AVAILABLE: {e:?}") + }); + let status = resp + .backup_description() + .and_then(|b| b.backup_details()) + .map(|d| d.backup_status().as_str().to_owned()) + .unwrap_or_default(); + if status == "AVAILABLE" { + return; + } + assert_eq!(status, "CREATING", "backup {arn} is {status}"); + assert!( + Instant::now() < deadline, + "backup {arn} not AVAILABLE in time" + ); + tokio::time::sleep(Duration::from_millis(250)).await; + } +} + +/// CreateBackup, then wait for AVAILABLE; the backup's ARN. +pub async fn create_available_backup(c: &Client, table: &str, backup_name: &str) -> String { + let create = create_backup_with_retry(c, table, backup_name).await; + let arn = create.backup_details().unwrap().backup_arn().to_string(); + wait_for_backup_available(c, &arn).await; + arn +} + +/// DeleteBackup after the test; a backup already gone is not a failure. +pub async fn delete_backup_quietly(c: &Client, arn: &str) { + c.delete_backup().backup_arn(arn).send().await.ok(); +} async fn make_table(name: &str) { let c = client(); @@ -71,8 +163,27 @@ async fn create_backup_happy_case() { let resp = create_backup_with_retry(c, &table, "test-backup-1").await; let details = resp.backup_details().unwrap(); assert!(!details.backup_arn().is_empty()); - assert_eq!(details.backup_status().as_str(), "AVAILABLE"); + // The in-catalog create answers AVAILABLE; the service and a store-backed + // ExtendDB answer CREATING and the backup reaches AVAILABLE later. + let status = details.backup_status().as_str(); + assert!( + status == "AVAILABLE" || status == "CREATING", + "CreateBackup answered {status}" + ); + let arn = details.backup_arn().to_string(); + wait_for_backup_available(c, &arn).await; + let desc = c.describe_backup().backup_arn(&arn).send().await.unwrap(); + assert_eq!( + desc.backup_description() + .unwrap() + .backup_details() + .unwrap() + .backup_status() + .as_str(), + "AVAILABLE" + ); + delete_backup_quietly(c, &arn).await; c.delete_table().table_name(&table).send().await.ok(); } @@ -101,13 +212,13 @@ async fn describe_backup() { let table = format!("DescBackup_{}", ts()); make_table(&table).await; - let create = create_backup_with_retry(c, &table, "desc-backup").await; - let arn = create.backup_details().unwrap().backup_arn().to_string(); + let arn = create_available_backup(c, &table, "desc-backup").await; let resp = c.describe_backup().backup_arn(&arn).send().await.unwrap(); let desc = resp.backup_description().unwrap(); assert_eq!(desc.backup_details().unwrap().backup_arn(), arn.as_str()); + delete_backup_quietly(c, &arn).await; c.delete_table().table_name(&table).send().await.ok(); } @@ -129,12 +240,14 @@ async fn list_backups() { let table = format!("ListBackup_{}", ts()); make_table(&table).await; - create_backup_with_retry(c, &table, "list-1").await; - create_backup_with_retry(c, &table, "list-2").await; + let first = create_available_backup(c, &table, "list-1").await; + let second = create_available_backup(c, &table, "list-2").await; let resp = c.list_backups().table_name(&table).send().await.unwrap(); assert!(resp.backup_summaries().len() >= 2); + delete_backup_quietly(c, &first).await; + delete_backup_quietly(c, &second).await; c.delete_table().table_name(&table).send().await.ok(); } @@ -156,8 +269,7 @@ async fn delete_backup() { let table = format!("DelBackup_{}", ts()); make_table(&table).await; - let create = create_backup_with_retry(c, &table, "del-backup").await; - let arn = create.backup_details().unwrap().backup_arn().to_string(); + let arn = create_available_backup(c, &table, "del-backup").await; let resp = c.delete_backup().backup_arn(&arn).send().await.unwrap(); let desc = resp.backup_description().unwrap(); @@ -185,31 +297,60 @@ async fn restore_table_from_backup() { .unwrap(); } - let create = create_backup_with_retry(c, &table, "restore-backup").await; - let arn = create.backup_details().unwrap().backup_arn().to_string(); + let arn = create_available_backup(c, &table, "restore-backup").await; let restored = format!("Restored_{}", ts()); - let resp = c + let restore = c .restore_table_from_backup() .target_table_name(&restored) .backup_arn(&arn) .send() - .await - .unwrap(); - assert_eq!( - resp.table_description() - .unwrap() - .table_status() - .unwrap() - .as_str(), - "CREATING" - ); - - wait_for_active(c, &restored).await; - let scan = c.scan().table_name(&restored).send().await.unwrap(); - assert_eq!(scan.count(), 5); + .await; + match backup_store() { + // A backup in the configured store is listed, described and deleted + // but not restored this release (the restore job is PR 9a's). + BackupStore::Filesystem | BackupStore::S3 => { + let err = restore.expect_err("a store-backed backup restored"); + assert_eq!(err_code(&err), Some("ValidationException"), "{err:?}"); + let msg = err_msg(&err); + assert!( + msg.contains("is in a backup store"), + "the refusal does not name the store: {msg}" + ); + println!("restore of a store-backed backup refused: {msg}"); + assert!( + c.describe_table() + .table_name(&restored) + .send() + .await + .is_err(), + "the refused restore created {restored}" + ); + } + BackupStore::None => { + let resp = restore.unwrap(); + assert_eq!( + resp.table_description() + .unwrap() + .table_status() + .unwrap() + .as_str(), + "CREATING" + ); + + wait_for_active(c, &restored).await; + let scan = c.scan().table_name(&restored).send().await.unwrap(); + assert_eq!(scan.count(), 5); + println!( + "restore of an in-catalog backup completed with {} items", + scan.count() + ); + + c.delete_table().table_name(&restored).send().await.ok(); + } + } - c.delete_table().table_name(&restored).send().await.ok(); + delete_backup_quietly(c, &arn).await; c.delete_table().table_name(&table).send().await.ok(); } @@ -282,7 +423,10 @@ async fn restore_table_to_point_in_time() { .use_latest_restorable_time(true) .send() .await; - assert!(err.is_err(), "RestoreTableToPointInTime should return an error (not yet supported)"); + assert!( + err.is_err(), + "RestoreTableToPointInTime should return an error (not yet supported)" + ); c.delete_table().table_name(&table).send().await.ok(); } diff --git a/tests/rust/src/restore_active_completeness.rs b/tests/rust/src/restore_active_completeness.rs index 83644ddc0..93ca7bcd5 100644 --- a/tests/rust/src/restore_active_completeness.rs +++ b/tests/rust/src/restore_active_completeness.rs @@ -23,12 +23,19 @@ //! The dataset is sized so the copy takes longer than the transition window on //! a loaded server. On very fast or idle hardware, raising `ITEMS` widens the //! window. +//! +//! On a server with a configured `[backup.store]` (`EXTENDDB_TEST_BACKUP_STORE` +//! at `filesystem` or `s3`) the restore of a backup this server created is +//! refused this release (the restore job is PR 9a's), so the race test pins +//! that refusal in place of the race. +use crate::backup_restore::{ + backup_store, delete_backup_quietly, wait_for_backup_available, BackupStore, +}; use crate::test_base::*; use aws_sdk_dynamodb::types::{ - AttributeDefinition, AttributeValue, BillingMode, GlobalSecondaryIndex, - KeySchemaElement, KeyType, Projection, ProjectionType, PutRequest, ScalarAttributeType, - Select, WriteRequest, + AttributeDefinition, AttributeValue, BillingMode, GlobalSecondaryIndex, KeySchemaElement, + KeyType, Projection, ProjectionType, PutRequest, ScalarAttributeType, Select, WriteRequest, }; use std::time::Duration; @@ -159,20 +166,36 @@ async fn restored_table_has_all_items_when_first_active() { .await .unwrap(); let arn = backup.backup_details().unwrap().backup_arn().to_string(); - // Wait until the backup is AVAILABLE. - for _ in 0..240 { - let d = c.describe_backup().backup_arn(&arn).send().await.unwrap(); - if d.backup_description() - .and_then(|b| b.backup_details()) - .map(|b| b.backup_status().as_str() == "AVAILABLE") - .unwrap_or(false) - { - break; - } - tokio::time::sleep(std::time::Duration::from_millis(250)).await; - } + wait_for_backup_available(c, &arn).await; let dst = format!("RestoreRaceDst_{}", ts()); + // A backup in the configured store is not restored this release (the + // restore job is PR 9a's): the race has nothing to observe, and the test + // pins the refusal instead. + if matches!(backup_store(), BackupStore::Filesystem | BackupStore::S3) { + let err = c + .restore_table_from_backup() + .target_table_name(&dst) + .backup_arn(&arn) + .send() + .await + .expect_err("a store-backed backup restored"); + assert_eq!(err_code(&err), Some("ValidationException"), "{err:?}"); + let msg = err_msg(&err); + assert!( + msg.contains("is in a backup store"), + "the refusal does not name the store: {msg}" + ); + println!("restore of a store-backed backup refused: {msg}"); + assert!( + c.describe_table().table_name(&dst).send().await.is_err(), + "the refused restore created {dst}" + ); + delete_backup_quietly(c, &arn).await; + c.delete_table().table_name(&src).send().await.ok(); + return; + } + println!("restore of an in-catalog backup: racing the observer"); // Race an observer against the in-flight restore: the moment it sees // ACTIVE, it counts. Returns None if ACTIVE never arrives within the // bound, which means the restore itself failed. @@ -254,6 +277,7 @@ async fn restored_table_has_all_items_when_first_active() { // above was still in flight (or just after, if the copy was fast). let observed = observer.await.unwrap(); + delete_backup_quietly(c, &arn).await; c.delete_table().table_name(&src).send().await.ok(); c.delete_table().table_name(&dst).send().await.ok(); diff --git a/tests/test_abac.py b/tests/test_abac.py index b56b85036..1e10a80b2 100755 --- a/tests/test_abac.py +++ b/tests/test_abac.py @@ -35,6 +35,7 @@ from conftest import wait_for_active, wait_for_deleted from management_helpers import ManagementClient +from test_backup_restore_fidelity import _delete_backup_when_available def _require_auth_env() -> tuple[str, str, str]: endpoint = os.environ.get("EXTENDDB_TEST_ENDPOINT", "").strip() admin_user = os.environ.get("EXTENDDB_ADMIN_USER", "").strip() @@ -1446,10 +1447,10 @@ def describe_backup(): "BackupDescription"]["BackupDetails"]["BackupArn"] == backup_arn finally: if backup_arn is not None: - try: - admin_client.delete_backup(BackupArn=backup_arn) - except Exception: - pass + # On a server with a backup store the backup is CREATING for + # the job's duration and DeleteBackup answers BackupInUse; + # deleting at once leaked a backup on every store-backed run. + _delete_backup_when_available(admin_client, backup_arn) try: admin_client.delete_table(TableName=table) except Exception: diff --git a/tests/test_auth_permissions.py b/tests/test_auth_permissions.py index 157f4cec3..3c2e33465 100755 --- a/tests/test_auth_permissions.py +++ b/tests/test_auth_permissions.py @@ -18,6 +18,7 @@ from __future__ import annotations import os +import time import uuid import boto3 @@ -1364,6 +1365,13 @@ def source_backup(self): pass if backup_arn is not None: try: + # A store-backed backup is CREATING until its job ends + # and refuses DeleteBackup until then. + deadline = time.monotonic() + 120 + while time.monotonic() < deadline and admin_client.describe_backup( + BackupArn=backup_arn + )["BackupDescription"]["BackupDetails"]["BackupStatus"] == "CREATING": + time.sleep(0.2) admin_client.delete_backup(BackupArn=backup_arn) except Exception: pass @@ -1438,6 +1446,14 @@ def test_restore_needs_the_item_actions_on_the_target(self, source_backup): """RestoreTableFromBackup on the backup plus the seven item actions on the target, each required; RestoreTableFromBackup on the target is neither required nor evaluated.""" + if os.environ.get("EXTENDDB_TEST_BACKUP_STORE", "").strip() in ("filesystem", "s3"): + # The denials hold on any server; the restores that must run are + # of a backup in the configured store, which RestoreTableFromBackup + # refuses until the restore job lands (PR 9a). + pytest.skip( + "RestoreTableFromBackup of a backup in the configured store is refused " + "until the restore job lands (PR 9a)" + ) admin_client, table, backup_arn = source_backup user = f"u-{uuid.uuid4().hex[:8]}" ak, sk = self._user_with_key(user) diff --git a/tests/test_authz_resource_arn.py b/tests/test_authz_resource_arn.py index 3d4accb65..291187c80 100644 --- a/tests/test_authz_resource_arn.py +++ b/tests/test_authz_resource_arn.py @@ -38,6 +38,7 @@ from conftest import wait_for_active from management_helpers import ManagementClient +from test_backup_restore_fidelity import _delete_backup_when_available def _require_auth_env() -> tuple[str, str, str]: @@ -122,7 +123,13 @@ def _user_with_policy(mgmt: ManagementClient, account_id: str, policy: dict) -> @pytest.fixture(scope="module") def table(auth_env, mgmt, account_id, region): """A table with a GSI, a stream, one item, and a backup. Returns the - canonical table, index, stream, and backup ARNs.""" + canonical table, index, stream, and backup ARNs. The cells deny + DeleteBackup and DeleteTable to other users, so the backup and the table + are deleted here once the module is done: on a server with a configured + `[backup.store]` the backup is a complete set of objects under the store, + and one that is left behind stays until someone deletes it (CI runners + die with theirs; a developer's server keeps them). The backup is deleted + once AVAILABLE, since a backup still CREATING refuses DeleteBackup.""" endpoint = auth_env[0] ak, sk = _user_with_policy(mgmt, account_id, ALLOW_ALL) admin = _client(endpoint, ak, sk, region) @@ -147,13 +154,18 @@ def table(auth_env, mgmt, account_id, region): desc = admin.describe_table(TableName=name)["Table"] table_arn = desc["TableArn"] backup_arn = admin.create_backup(TableName=name, BackupName="arnbk")["BackupDetails"]["BackupArn"] - return { + yield { "name": name, "table_arn": table_arn, "index_arn": table_arn + "/index/gsi1", "stream_arn": desc.get("LatestStreamArn"), "backup_arn": backup_arn, } + _delete_backup_when_available(admin, backup_arn) + try: + admin.delete_table(TableName=name) + except ClientError: + pass class TestExplicitDenyOnResourceArn: diff --git a/tests/test_backup_job.py b/tests/test_backup_job.py new file mode 100644 index 000000000..6bca868a9 --- /dev/null +++ b/tests/test_backup_job.py @@ -0,0 +1,575 @@ +# SPDX-License-Identifier: Apache-2.0 + +"""The store-backed CreateBackup job at the server's boundary. + +Each test starts its own `extenddb serve --foreground` on an isolated +deployment with a filesystem backup store (SQLite: the default root beside +the database file; PostgreSQL: a root under the test's directory) and reads +the API's answers, the server's log, `extenddb backup list --failed` and the +store's root: + +- a second CreateBackup while one job runs with `max_concurrent_jobs = 1` is + LimitExceededException naming the bound, and the slot is free once the + first backup is AVAILABLE; +- a table holding an `N` value the wire let in (#618: `1E126` or `abc` in a + non-key attribute, `1E126` in the range key of a key-ordered table) written + through each of the four paths that store it unvalidated (BatchWriteItem, + TransactWriteItems, UpdateItem on an absent item in the key it creates, + ImportTable) gives a FAILED row, no manifest under the root, three log + lines naming each item's key and attribute, and a FAILED reason giving the + count and the first item and naming #618; +- an item over `max_item_size_bytes` (stored under the default limit, the + server restarted with a lower one) gives a reason that does not name #618; +- a refused item with a UUID hash key and a ten-digit epoch `N` range key is + named by its whole key, and the wire's reason for its value, in the log + line and in `backup list --failed`. + +The binary at `EXTENDDB_BINARY` decides which deployment runs: a SQLite build +or a PostgreSQL build (with `EXTENDDB_TEST_PG_CONNECTION_STRING`). Excluded +from the backend-agnostic suite; run by name. +""" + +from __future__ import annotations + +import json +import os +import re +import threading +import time +import uuid +from concurrent.futures import ThreadPoolExecutor + +import boto3 +import pytest +import requests +from botocore.auth import SigV4Auth +from botocore.awsrequest import AWSRequest +from botocore.exceptions import ClientError + +import lifecycle_helpers as lh +from test_backup_store_config import ( + ADMIN_PASSWORD, + EXTENDDB_BINARY, + _append, + _client, + _pg_build, + _provision, + _Server, + _sqlite_build, + _SqliteDeployment, + _wait_backup_available, + _wait_health, +) + +ACCOUNT = "123456789012" +# The job's bound in the LimitExceeded cell: items of this count and padding +# keep the first job running for longer than the second request's round trip +# (0.4 s for 30,000 400-byte items on a SQLite build of this host). A refusal +# is itself the evidence that the bound held; when the second CreateBackup is +# accepted instead, the cell reads the first backup's status to tell a broken +# bound (the first still CREATING) from a probe that came too late (the first +# already AVAILABLE, where it asks for more items). +BUSY_ITEMS = int(os.environ.get("EXTENDDB_TEST_BACKUP_BUSY_ITEMS", "30000")) +LIMIT_TEXT = ( + "Too many backup and restore jobs are running on this server " + "(at most 1 at once, the [backup] max_concurrent_jobs bound); retry later" +) + + +class _Deployment: + """A deployment with a filesystem store, its config and its root.""" + + def __init__(self, backend: str, config: str, port: int, root: str, cert: str, cwd: str): + self.backend = backend + self.config = config + self.port = port + self.root = root + self.cert = cert + self.cwd = cwd + + def run(self, *args): + import subprocess + + return subprocess.run( + [EXTENDDB_BINARY, args[0], "--config", self.config, *args[1:]], + cwd=self.cwd, + capture_output=True, + text=True, + stdin=subprocess.DEVNULL, + timeout=180, + ) + + def set_limit(self, key: str, value: str) -> None: + """Set a key of the init template's `[limits]` table.""" + with open(self.config) as f: + text = f.read() + assert text.count("[limits]\n") == 1, text + text = text.replace("[limits]\n", f"[limits]\n{key} = {value}\n", 1) + with open(self.config, "w") as f: + f.write(text) + + +@pytest.fixture() +def deployment(request, tmp_path) -> _Deployment: + """The build's deployment: SQLite with the defaulted root, or PostgreSQL + with a filesystem root under the test's directory. Both read + `reconcile_interval_secs = 1`, `max_concurrent_jobs = 1` and import from + the test's directory (ImportTable reads `//`).""" + base = os.path.realpath(str(tmp_path)) + if _sqlite_build(): + d = _SqliteDeployment(str(tmp_path)) + config, port, cert = d.config, d.port, d.cert + root = os.path.join(base, "backups") + store = "" + backend = "sqlite" + elif _pg_build(): + cli_env = request.getfixturevalue("cli_env") + result = lh._run_extenddb( + "init", *lh._init_args(cli_env), config=cli_env["config_path"], + env_override={"EXTENDDB_ADMIN_PASSWORD": ADMIN_PASSWORD}, + ) + assert result.returncode == 0, result.stdout + result.stderr + lh._patch_config_port(cli_env["config_path"], cli_env["port"]) + config, port = cli_env["config_path"], cli_env["port"] + cert = os.path.join(cli_env["tls_dir"], "cert.pem") + root = os.path.join(base, "backups") + os.mkdir(root, 0o700) + store = f'[backup.store]\nkind = "filesystem"\n[backup.store.filesystem]\npath = "{root}"\n' + backend = "postgres" + else: + pytest.skip("EXTENDDB_BINARY is neither a SQLite nor a PostgreSQL build") + os.mkdir(os.path.join(base, ACCOUNT)) + _append( + config, + "[backup]\nreconcile_interval_secs = 1\nmax_concurrent_jobs = 1\n" + + store + + f'[import]\npaths = ["{base}"]\n', + ) + return _Deployment(backend, config, port, root, cert, base) + + +def _started(d: _Deployment, log_path: str) -> tuple[_Server, object, dict]: + """The server started on the deployment, a client and its credentials.""" + server = _Server(d.config, log_path) + try: + _wait_health( + d.port, + lambda h: h.get("components", {}).get("backup_store", {}).get("status") == "healthy", + server=server, + ) + creds = _provision(d.port, d.cert, ADMIN_PASSWORD) + except BaseException: + # A server whose store never came up (or whose provisioning failed) + # would otherwise keep running after the test's failure. + server.stop() + raise + return server, _client(d.port, creds), creds + + +def _signed_post(d: _Deployment, creds: dict, operation: str, body: dict) -> requests.Response: + """A hand-signed request for an operation botocore's model would refuse + (ImportTable's `FileSource`).""" + body_bytes = json.dumps(body).encode("utf-8") + headers = { + "X-Amz-Target": f"DynamoDB_20120810.{operation}", + "Content-Type": "application/x-amz-json-1.0", + } + endpoint = f"https://127.0.0.1:{d.port}" + frozen = boto3.Session( + aws_access_key_id=creds["AWS_ACCESS_KEY_ID"], + aws_secret_access_key=creds["AWS_SECRET_ACCESS_KEY"], + ).get_credentials().get_frozen_credentials() + aws_req = AWSRequest(method="POST", url=endpoint, data=body_bytes, headers=headers) + SigV4Auth(frozen, "dynamodb", "us-east-1").add_auth(aws_req) + return requests.post(endpoint, data=body_bytes, headers=dict(aws_req.headers), verify=False) + + +def _wait_active(ddb, name: str, timeout: float = 60.0) -> None: + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + if ddb.describe_table(TableName=name)["Table"]["TableStatus"] == "ACTIVE": + return + time.sleep(0.05) + raise AssertionError(f"{name} did not become ACTIVE") + + +def _create_composite_table(ddb, name: str, gsi: bool = False) -> None: + """A key-ordered table: `pk` S hash key, `sk` N range key.""" + attrs = [ + {"AttributeName": "pk", "AttributeType": "S"}, + {"AttributeName": "sk", "AttributeType": "N"}, + ] + kwargs: dict = {} + if gsi: + attrs.append({"AttributeName": "g", "AttributeType": "S"}) + kwargs["GlobalSecondaryIndexes"] = [ + { + "IndexName": "by-g", + "KeySchema": [{"AttributeName": "g", "KeyType": "HASH"}], + "Projection": {"ProjectionType": "ALL"}, + } + ] + ddb.create_table( + TableName=name, + KeySchema=[ + {"AttributeName": "pk", "KeyType": "HASH"}, + {"AttributeName": "sk", "KeyType": "RANGE"}, + ], + AttributeDefinitions=attrs, + BillingMode="PAY_PER_REQUEST", + **kwargs, + ) + _wait_active(ddb, name) + + +def _wait_backup_gone(ddb, arn: str, timeout: float = 60.0) -> None: + """Poll DescribeBackup until the backup is BackupNotFound (its row FAILED + or removed); an AVAILABLE backup fails the test.""" + deadline = time.monotonic() + timeout + while True: + try: + status = ddb.describe_backup(BackupArn=arn)["BackupDescription"]["BackupDetails"]["BackupStatus"] + except ClientError as e: + assert e.response["Error"]["Code"] == "BackupNotFoundException", e.response["Error"] + return + assert status == "CREATING", f"backup {arn} is {status}" + assert time.monotonic() < deadline, f"backup {arn} still CREATING after {timeout} s" + time.sleep(0.1) + + +_RUST_ESCAPES = {"n": "\n", "r": "\r", "t": "\t", "0": "\0", "\\": "\\", '"': '"', "'": "'"} + + +def _rust_debug_string(quoted: str) -> str: + """The string a Rust `{:?}` of a `String` printed: the quotes removed and + its escapes undone (`\\n`, `\\"`, `\\\\` and `\\u{hex}` for a character + `Debug` does not print as itself). `json.loads` refuses the `\\u{..}` + form, so a reason holding such a character (a key the client wrote with + one) is decoded here.""" + assert quoted.startswith('"') and quoted.endswith('"'), quoted + out: list[str] = [] + i, body = 0, quoted[1:-1] + while i < len(body): + c = body[i] + if c != "\\": + out.append(c) + i += 1 + continue + e = body[i + 1] + if e == "u": + close = body.index("}", i + 3) + out.append(chr(int(body[i + 3 : close], 16))) + i = close + 1 + else: + out.append(_RUST_ESCAPES[e]) + i += 2 + return "".join(out) + + +def _failed_rows(d: _Deployment) -> dict[str, str]: + """`extenddb backup list --failed`: each FAILED row's ARN to its reason.""" + res = d.run("backup", "list", "--failed") + assert res.returncode == 0, res.stdout + res.stderr + rows: dict[str, str] = {} + for line in res.stdout.splitlines(): + m = re.match(r"^FAILED\s.*?\s(arn:\S+)\s.*failure_reason=(\".*\")$", line) + if m: + rows[m.group(1)] = _rust_debug_string(m.group(2)) + return rows + + +def _prefix(d: _Deployment, table: str, arn: str) -> str: + return os.path.join(d.root, ACCOUNT, table, arn.rsplit("/", 1)[1]) + + +def _assert_no_manifest(d: _Deployment, table: str, arn: str) -> None: + prefix = _prefix(d, table, arn) + if os.path.isdir(prefix): + left = os.listdir(prefix) + assert "extenddb-manifest.json" not in left, left + + +def _restored_item_lines(log: str, arn: str) -> list[str]: + return [ + line for line in log.splitlines() if f"backup {arn}: item cannot be restored: " in line + ] + + +def test_second_create_backup_over_the_job_bound_is_limit_exceeded(deployment, tmp_path): + """With `max_concurrent_jobs = 1`, a CreateBackup while one job runs is + LimitExceededException naming the bound (a second CreateBackup accepted + while the first backup is still CREATING fails the cell; one accepted + once the first is AVAILABLE means the probe came too late and asks for + more items); the first job finishes AVAILABLE with every item and a GSI + in its SourceTableFeatureDetails, and a third CreateBackup is then + accepted (the slot was freed).""" + d = deployment + server, ddb, creds = _started(d, str(tmp_path / "serve.log")) + try: + name = f"busy-{uuid.uuid4().hex[:8]}" + _create_composite_table(ddb, name, gsi=True) + _load(d, creds, name, BUSY_ITEMS) + first = ddb.create_backup(TableName=name, BackupName="first")["BackupDetails"] + assert first["BackupStatus"] == "CREATING", first + try: + ddb.create_backup(TableName=name, BackupName="second") + except ClientError as e: + refused = e.response["Error"] + else: + refused = None + status_after = ddb.describe_backup(BackupArn=first["BackupArn"])["BackupDescription"]["BackupDetails"]["BackupStatus"] + if refused is None: + assert status_after == "AVAILABLE", ( + f"the second CreateBackup was accepted while the first job was still running " + f"(the first backup is {status_after}): the max_concurrent_jobs bound did not hold" + ) + pytest.fail( + "the first backup was already AVAILABLE when the second CreateBackup was sent, " + "so the probe checked nothing; raise EXTENDDB_TEST_BACKUP_BUSY_ITEMS" + ) + assert refused["Code"] == "LimitExceededException", refused + assert refused["Message"] == LIMIT_TEXT, refused + _wait_backup_available(ddb, first["BackupArn"], timeout=120) + desc = ddb.describe_backup(BackupArn=first["BackupArn"])["BackupDescription"] + assert desc["SourceTableDetails"]["ItemCount"] == BUSY_ITEMS, desc["SourceTableDetails"] + gsis = desc["SourceTableFeatureDetails"]["GlobalSecondaryIndexes"] + assert [g["IndexName"] for g in gsis] == ["by-g"], gsis + assert os.path.isfile(os.path.join(_prefix(d, name, first["BackupArn"]), "extenddb-manifest.json")) + third = ddb.create_backup(TableName=name, BackupName="third")["BackupDetails"] + assert third["BackupStatus"] == "CREATING", third + _wait_backup_available(ddb, third["BackupArn"], timeout=120) + listed = sorted(b["BackupName"] for b in ddb.list_backups(TableName=name)["BackupSummaries"]) + assert listed == ["first", "third"], listed + finally: + server.stop() + + +def _load(d: _Deployment, creds: dict, name: str, n: int) -> None: + """Write n composite-key items with BatchWriteItem from 8 threads, one + client each: a boto3 client shared across threads fails at random with + TLS record-layer errors that look like server faults.""" + local = threading.local() + + def write(start: int) -> None: + if not hasattr(local, "client"): + local.client = _client(d.port, creds) + requests_ = [ + { + "PutRequest": { + "Item": { + "pk": {"S": f"p{i % 500:04d}"}, + "sk": {"N": str(i)}, + "g": {"S": f"g{i % 50}"}, + "pad": {"S": "x" * 400}, + } + } + } + for i in range(start, min(start + 25, n)) + ] + pending = {name: requests_} + while pending: + pending = local.client.batch_write_item(RequestItems=pending).get("UnprocessedItems") or {} + if pending: + time.sleep(0.05) + + with ThreadPoolExecutor(max_workers=8) as pool: + list(pool.map(write, range(0, n, 25))) + + +# The three items each path stores: `1E126` and `abc` in a non-key attribute +# and `1E126` in the range key (an `abc` range key is refused by the number +# column on PostgreSQL and SQLite, so no path stores one). UpdateItem +# validates the values its expression sets, so its three items carry the bad +# value in the key it creates. +BAD_ITEMS = [ + {"pk": {"S": "a"}, "sk": {"N": "1"}, "amount": {"N": "1E126"}}, + {"pk": {"S": "a"}, "sk": {"N": "2"}, "note": {"N": "abc"}}, + {"pk": {"S": "b"}, "sk": {"N": "1E126"}}, +] +BAD_KEYS = [ + {"pk": {"S": "a"}, "sk": {"N": "1E126"}}, + {"pk": {"S": "b"}, "sk": {"N": "1E127"}}, + {"pk": {"S": "c"}, "sk": {"N": "1E128"}}, +] +GOOD_ITEMS = [{"pk": {"S": "z"}, "sk": {"N": str(i)}, "amount": {"N": str(i)}} for i in range(5)] + + +def _write_batch(d, ddb, creds, name): + _create_composite_table(ddb, name) + ddb.batch_write_item( + RequestItems={name: [{"PutRequest": {"Item": i}} for i in BAD_ITEMS + GOOD_ITEMS]} + ) + return BAD_ITEMS + + +def _write_transact(d, ddb, creds, name): + _create_composite_table(ddb, name) + ddb.transact_write_items( + TransactItems=[{"Put": {"TableName": name, "Item": i}} for i in BAD_ITEMS + GOOD_ITEMS] + ) + return BAD_ITEMS + + +def _write_update(d, ddb, creds, name): + _create_composite_table(ddb, name) + for key in BAD_KEYS: + ddb.update_item( + TableName=name, + Key=key, + UpdateExpression="SET note = :v", + ExpressionAttributeValues={":v": {"S": "created from the key"}}, + ) + for item in GOOD_ITEMS: + ddb.put_item(TableName=name, Item=item) + return BAD_KEYS + + +def _write_import(d, ddb, creds, name): + path = os.path.join(d.cwd, ACCOUNT, f"{name}.json") + with open(path, "w") as f: + for item in BAD_ITEMS + GOOD_ITEMS: + f.write(json.dumps({"Item": item}) + "\n") + resp = _signed_post( + d, + creds, + "ImportTable", + { + "FileSource": {"Path": path}, + "InputFormat": "DYNAMODB_JSON", + "TableCreationParameters": { + "TableName": name, + "AttributeDefinitions": [ + {"AttributeName": "pk", "AttributeType": "S"}, + {"AttributeName": "sk", "AttributeType": "N"}, + ], + "KeySchema": [ + {"AttributeName": "pk", "KeyType": "HASH"}, + {"AttributeName": "sk", "KeyType": "RANGE"}, + ], + "BillingMode": "PAY_PER_REQUEST", + }, + }, + ) + assert resp.status_code == 200, resp.text + assert resp.json()["ImportTableDescription"]["ImportStatus"] == "COMPLETED", resp.text + _wait_active(ddb, name) + return BAD_ITEMS + + +WRITE_PATHS = { + "batch_write_item": _write_batch, + "transact_write_items": _write_transact, + "update_item": _write_update, + "import_table": _write_import, +} + + +def _key_json(item: dict) -> str: + return json.dumps({"pk": item["pk"], "sk": item["sk"]}, separators=(",", ":")) + + +@pytest.mark.parametrize("path", sorted(WRITE_PATHS)) +def test_an_item_the_wire_let_in_fails_the_backup(deployment, tmp_path, path): + """Three items with an `N` value #618 let in, stored through one write + path, and five that restore: CreateBackup answers CREATING, the job + validates every item (three log lines, one per item, with its key and + attribute path), the row is FAILED with a reason giving the count and + the first item and naming #618, DescribeBackup is BackupNotFound, + ListBackups omits it, and no manifest is under the root.""" + d = deployment + server, ddb, creds = _started(d, str(tmp_path / "serve.log")) + try: + name = f"n618-{path.replace('_', '')}-{uuid.uuid4().hex[:6]}" + bad = WRITE_PATHS[path](d, ddb, creds, name) + stored = ddb.scan(TableName=name, ConsistentRead=True)["Items"] + assert len(stored) == len(bad) + len(GOOD_ITEMS), stored + details = ddb.create_backup(TableName=name, BackupName="nightly")["BackupDetails"] + assert details["BackupStatus"] == "CREATING", details + arn = details["BackupArn"] + _wait_backup_gone(ddb, arn) + assert ddb.list_backups(TableName=name)["BackupSummaries"] == [] + _assert_no_manifest(d, name, arn) + reason = _failed_rows(d)[arn] + first = sorted(bad, key=lambda i: (i["pk"]["S"], float(i["sk"]["N"])))[0] + assert reason.startswith(f"{len(bad)} item(s) cannot be restored; first: key {_key_json(first)}, attribute "), reason + assert reason.endswith(" (#618)"), reason + if "sk" in first and first["sk"]["N"] == "1E126": + assert ", attribute sk: " in reason, reason + else: + assert ", attribute amount: " in reason, reason + log = server.log_text() + lines = _restored_item_lines(log, arn) + assert len(lines) == len(bad), lines + logged_keys = {re.search(r"key (\{.*?\}\}), attribute", line).group(1) for line in lines} + assert logged_keys == {_key_json(i) for i in bad}, logged_keys + assert all(re.search(r"attribute (sk|amount|note): ", line) for line in lines), lines + assert f"backup {arn}: FAILED ({reason}); its objects were deleted" in log, log + assert "more item(s) cannot be restored" not in log + finally: + server.stop() + + +def test_a_refused_item_with_a_uuid_and_epoch_key_is_named_whole(deployment, tmp_path): + """One refused item with a realistic key, a UUID hash key and a ten-digit + epoch range key, and `amount: abc`: the log line and the FAILED reason + carry the whole key as DynamoDB JSON (`{"pk":{"S":""},"sk":{"N": + "1761955200"}}`), not a shortened form, and the wire's reason for the + value, so an operator can find the item from either.""" + d = deployment + server, ddb, _ = _started(d, str(tmp_path / "serve.log")) + try: + name = f"n618-key-{uuid.uuid4().hex[:6]}" + _create_composite_table(ddb, name) + item = {"pk": {"S": str(uuid.uuid4())}, "sk": {"N": "1761955200"}, "amount": {"N": "abc"}} + ddb.batch_write_item(RequestItems={name: [{"PutRequest": {"Item": i}} for i in [item] + GOOD_ITEMS]}) + arn = ddb.create_backup(TableName=name, BackupName="nightly")["BackupDetails"]["BackupArn"] + _wait_backup_gone(ddb, arn) + key = _key_json(item) + assert f'"pk":{{"S":"{item["pk"]["S"]}"}}' in key and '"sk":{"N":"1761955200"}' in key, key + value_reason = "attribute amount: The parameter cannot be converted to a numeric value: abc" + reason = _failed_rows(d)[arn] + assert reason == f"1 item(s) cannot be restored; first: key {key}, {value_reason} (#618)", reason + lines = _restored_item_lines(server.log_text(), arn) + assert len(lines) == 1, lines + assert f"backup {arn}: item cannot be restored: key {key}, {value_reason}" in lines[0], lines[0] + finally: + server.stop() + + +def test_an_item_over_max_item_size_bytes_fails_the_backup_without_naming_618(deployment, tmp_path): + """A 300 KB item stored under the default `max_item_size_bytes`, the + server restarted with `[limits] max_item_size_bytes = 65536`: the backup + is FAILED with a reason naming the item's size and the limit and not + #618, and the server's own PutItem refuses such an item.""" + d = deployment + name = f"big-{uuid.uuid4().hex[:8]}" + server, ddb, _ = _started(d, str(tmp_path / "serve.log")) + try: + _create_composite_table(ddb, name) + ddb.put_item(TableName=name, Item={"pk": {"S": "big"}, "sk": {"N": "1"}, "pad": {"S": "x" * 300000}}) + ddb.put_item(TableName=name, Item={"pk": {"S": "small"}, "sk": {"N": "1"}, "pad": {"S": "x"}}) + # The same table backs up whole under the default limit. + arn = ddb.create_backup(TableName=name, BackupName="under-default")["BackupDetails"]["BackupArn"] + _wait_backup_available(ddb, arn) + finally: + server.stop() + d.set_limit("max_item_size_bytes", "65536") + server, ddb, _ = _started(d, str(tmp_path / "serve2.log")) + try: + with pytest.raises(ClientError) as refused: + ddb.put_item(TableName=name, Item={"pk": {"S": "big2"}, "sk": {"N": "1"}, "pad": {"S": "x" * 100000}}) + assert refused.value.response["Error"]["Code"] == "ValidationException" + arn = ddb.create_backup(TableName=name, BackupName="under-lower")["BackupDetails"]["BackupArn"] + _wait_backup_gone(ddb, arn) + _assert_no_manifest(d, name, arn) + reason = _failed_rows(d)[arn] + assert reason.startswith('1 item(s) cannot be restored; first: key {"pk":{"S":"big"},"sk":{"N":"1"}}: item size '), reason + assert reason.endswith(" bytes exceeds the limit of 65536 bytes"), reason + assert "#618" not in reason, reason + lines = _restored_item_lines(server.log_text(), arn) + assert len(lines) == 1 and "exceeds the limit of 65536 bytes" in lines[0], lines + # The first backup, taken under the default limit, is untouched. + listed = [b["BackupName"] for b in ddb.list_backups(TableName=name)["BackupSummaries"]] + assert listed == ["under-default"], listed + finally: + server.stop() diff --git a/tests/test_backup_restore_fidelity.py b/tests/test_backup_restore_fidelity.py index 35032df5e..8d152b607 100644 --- a/tests/test_backup_restore_fidelity.py +++ b/tests/test_backup_restore_fidelity.py @@ -24,6 +24,7 @@ import boto3 import pytest +import requests from botocore.config import Config from botocore.exceptions import ClientError, ReadTimeoutError @@ -41,13 +42,98 @@ BACKUP_TIMEOUT_S = 900.0 RESTORE_TIMEOUT_S = 1800.0 +# Where the server under test writes its backups: `filesystem` or `s3` (a +# configured `[backup.store]`, under which CreateBackup is asynchronous and +# backs up vector tables) or `none` (the in-catalog create). The CI jobs set +# it beside EXTENDDB_TEST_ENDPOINT; the one test whose expected outcome +# depends on it fails when it is unset rather than accept either outcome. +BACKUP_STORE = os.environ.get("EXTENDDB_TEST_BACKUP_STORE", "").strip() or None +BACKUP_STORE_VALUES = ("filesystem", "s3", "none") + +# RestoreTableFromBackup of a backup written to a configured store answers +# ValidationException ("is in a backup store; this release restores +# in-catalog backups only") until the restore job of PR 9a lands. The cells +# that restore a backup this server created skip on such a server and come +# back with PR 9a; the in-catalog create's backups (`none`, and the service) +# restore as before. The marker is gated on the target as well as the +# variable, so a stale `EXTENDDB_TEST_BACKUP_STORE=filesystem` in the +# environment of a real-DynamoDB run (which does not require the variable) +# cannot skip the 24 cells against the service. +restores_a_backup_of_this_server = pytest.mark.skipif( + BACKUP_STORE in ("filesystem", "s3") and not _is_real_dynamodb(), + reason=( + f"EXTENDDB_TEST_BACKUP_STORE={BACKUP_STORE}: RestoreTableFromBackup of a backup " + "in the configured store is refused until the restore job lands (PR 9a)" + ), +) + + +@pytest.fixture(scope="session") +def backup_store_component(endpoint_url): + """The server's `backup_store` health component, read once from + `/health`, checked against EXTENDDB_TEST_BACKUP_STORE: a configured store + (`filesystem`, `s3`) must be reported with that kind and `healthy`, and + `none` must report no component. A variable the harness cannot observe + is not a parameter: a check that only reads it asserts nothing about the + server, and without this, `s3` against a server with a filesystem store + passed every cell.""" + if BACKUP_STORE not in BACKUP_STORE_VALUES: + pytest.fail( + f"EXTENDDB_TEST_BACKUP_STORE is {'unset' if BACKUP_STORE is None else repr(BACKUP_STORE)}; " + f"set it to one of {BACKUP_STORE_VALUES} to say where the server under test writes " + "its backups" + ) + assert endpoint_url, "the backup store component is ExtendDB's; no endpoint is set" + health = requests.get(f"{endpoint_url.rstrip('/')}/health", verify=False, timeout=10).json() + component = health.get("components", {}).get("backup_store") + if BACKUP_STORE == "none": + assert component is None, ( + f"EXTENDDB_TEST_BACKUP_STORE=none but the server reports a backup store: {component}" + ) + else: + assert component is not None, ( + f"EXTENDDB_TEST_BACKUP_STORE={BACKUP_STORE} but the server reports no backup store: " + f"{health}" + ) + assert component.get("kind") == BACKUP_STORE, ( + f"EXTENDDB_TEST_BACKUP_STORE={BACKUP_STORE} but the server's store is {component}" + ) + assert component.get("status") == "healthy", ( + f"the server's {BACKUP_STORE} store is not healthy: {component}" + ) + print(f"backup store under test: {component if component else 'none'}") + return component + + +def wait_for_backup_available(client, backup_arn: str, timeout: float = BACKUP_TIMEOUT_S) -> None: + """Poll DescribeBackup until the backup is AVAILABLE. + + The service creates a backup asynchronously; so does ExtendDB with a + configured `[backup.store]` (CreateBackup answers CREATING and the job + uploads). A backup still CREATING refuses DeleteBackup (BackupInUse) and + restore, so every caller that goes on to delete, restore or read the + size of a backup it just created waits here first. Any other status is a + failure: a backup that fails is BackupNotFound, never FAILED, on the wire. + """ + deadline = time.monotonic() + timeout + while True: + status = client.describe_backup(BackupArn=backup_arn)["BackupDescription"][ + "BackupDetails" + ]["BackupStatus"] + if status == "AVAILABLE": + return + assert status == "CREATING", f"backup {backup_arn} is {status}" + assert time.monotonic() < deadline, f"backup {backup_arn} not AVAILABLE in time" + time.sleep(_poll_interval()) + def _create_backup(client, table_name: str) -> str: """CreateBackup, then wait for the backup to be AVAILABLE. The service creates a backup asynchronously and refuses to restore one - that is still CREATING; ExtendDB returns it AVAILABLE. A just-created - table can also refuse CreateBackup for a short while, which is retried. + that is still CREATING; so does ExtendDB with a configured store. A + just-created table can also refuse CreateBackup for a short while, which + is retried. """ deadline = time.monotonic() + 60 while True: @@ -62,26 +148,25 @@ def _create_backup(client, table_name: str) -> str: if not retryable or time.monotonic() >= deadline: raise time.sleep(0.5) - deadline = time.monotonic() + BACKUP_TIMEOUT_S try: - while True: - status = client.describe_backup(BackupArn=arn)["BackupDescription"][ - "BackupDetails" - ]["BackupStatus"] - if status == "AVAILABLE": - return arn - assert status == "CREATING", f"backup {arn} is {status}" - assert time.monotonic() < deadline, f"backup {arn} not AVAILABLE in time" - time.sleep(_poll_interval()) + wait_for_backup_available(client, arn) + return arn except BaseException: # The caller never learns the ARN, so the backup is removed here. - try: - client.delete_backup(BackupArn=arn) - except ClientError: - pass + _delete_backup_when_available(client, arn) raise +def _delete_backup_when_available(client, backup_arn: str) -> None: + """DeleteBackup once the backup is no longer CREATING; a backup that is + gone (failed or deleted) is left as it is.""" + try: + wait_for_backup_available(client, backup_arn) + client.delete_backup(BackupArn=backup_arn) + except (ClientError, AssertionError): + pass + + def _backup_creation_time(client, backup_arn: str): return client.describe_backup(BackupArn=backup_arn)["BackupDescription"][ "BackupDetails" @@ -256,7 +341,11 @@ def _index_items(client, table: str, index: str, hash_attr: str, values: list) - def _cleanup(client, backup_arn: str | None, *tables: str) -> None: - """Delete tables and the backup; each step runs even if an earlier one fails.""" + """Delete tables and the backup; each step runs even if an earlier one fails. + + A backup still CREATING (the asynchronous create) is waited for first: + DeleteBackup of one is BackupInUseException. + """ if tables: try: _drop(client, tables[0]) @@ -265,6 +354,7 @@ def _cleanup(client, backup_arn: str | None, *tables: str) -> None: return if backup_arn: try: + wait_for_backup_available(client, backup_arn) client.delete_backup(BackupArn=backup_arn) except client.exceptions.BackupNotFoundException: pass @@ -338,11 +428,13 @@ def _round_trip(client, sort_kind: str | None, hash_kind: str = "S") -> None: _cleanup(client, backup_arn, restored, source) +@restores_a_backup_of_this_server @pytest.mark.parametrize("hash_kind", ["S", "N", "B"]) def test_restore_hash_only_table(dynamodb_client, hash_kind): _round_trip(dynamodb_client, None, hash_kind) +@restores_a_backup_of_this_server @pytest.mark.parametrize("sort_kind", ["S", "N", "B"]) def test_restore_composite_key_table(dynamodb_client, sort_kind): _round_trip(dynamodb_client, sort_kind) @@ -359,6 +451,7 @@ def _index_by_name(indexes: list[dict]) -> dict: return {i["IndexName"]: i for i in indexes} +@restores_a_backup_of_this_server def test_restore_preserves_indexes_and_provisioned_throughput(dynamodb_client): """GSIs and LSIs, their key schemas, projections, and throughput, the table's provisioned throughput, and every index's contents come back.""" @@ -546,6 +639,7 @@ def ranked(table: str) -> list[str]: _cleanup(client, backup_arn, restored, source) +@restores_a_backup_of_this_server def test_restore_normalizes_stale_throughput_after_on_demand_switch(dynamodb_client): """A backup taken after an on-demand switch does not restore stale capacity.""" client = dynamodb_client @@ -620,6 +714,7 @@ def test_restore_normalizes_stale_throughput_after_on_demand_switch(dynamodb_cli _cleanup(client, backup_arn, restored, source) +@restores_a_backup_of_this_server def test_restore_override_pay_per_request_to_provisioned(dynamodb_client): client = dynamodb_client source = f"restore-override-ppr-{uuid.uuid4().hex[:10]}" @@ -658,6 +753,7 @@ def test_restore_override_pay_per_request_to_provisioned(dynamodb_client): _cleanup(client, backup_arn, restored, source) +@restores_a_backup_of_this_server def test_restore_override_provisioned_to_pay_per_request(dynamodb_client): client = dynamodb_client source = f"restore-override-prov-{uuid.uuid4().hex[:10]}" @@ -692,6 +788,7 @@ def test_restore_override_provisioned_to_pay_per_request(dynamodb_client): _cleanup(client, backup_arn, restored, source) +@restores_a_backup_of_this_server def test_provisioned_restore_override_requiring_gsi_override_leaves_no_target( dynamodb_client, ): @@ -934,6 +1031,7 @@ def large_backup(dynamodb_client): _cleanup(client, backup_arn, source) +@restores_a_backup_of_this_server def test_restore_response_arrives_before_its_copy_finishes(dynamodb_client, large_backup): """The CREATING response reaches a client with a 10 s read timeout and no retries before the copy finishes, and the table then reaches ACTIVE with @@ -996,6 +1094,7 @@ def _table_appears(client, name: str, timeout: float) -> bool: time.sleep(_poll_interval() * 5) +@restores_a_backup_of_this_server def test_restore_survives_a_client_that_disconnects_before_the_response( dynamodb_client, large_backup ): @@ -1051,6 +1150,7 @@ def test_restore_survives_a_client_that_disconnects_before_the_response( _cleanup(dynamodb_client, None, target) +@restores_a_backup_of_this_server def test_first_read_after_active_succeeds(dynamodb_client, large_backup): """A read on the restore target while it is CREATING is ResourceNotFoundException, and the first read after DescribeTable reports @@ -1107,6 +1207,7 @@ def test_first_read_after_active_succeeds(dynamodb_client, large_backup): _cleanup(client, None, target) +@restores_a_backup_of_this_server def test_create_backup_of_a_creating_table_is_table_not_found(dynamodb_client, large_backup): """CreateBackup on a table that is still CREATING is TableNotFoundException "Table not found: " (measured on DynamoDB), @@ -1136,6 +1237,7 @@ def test_create_backup_of_a_creating_table_is_table_not_found(dynamodb_client, l _cleanup(client, None, target) +@restores_a_backup_of_this_server def test_restore_into_a_creating_target_and_a_second_restore_of_its_backup_are_refused( dynamodb_client, large_backup ): @@ -1178,6 +1280,7 @@ def test_restore_into_a_creating_target_and_a_second_restore_of_its_backup_are_r _cleanup(client, None, target, other) +@restores_a_backup_of_this_server def test_target_and_backup_are_held_while_the_copy_runs(dynamodb_client, large_backup): """DeleteTable of a CREATING restore target is ResourceInUseException, and (ExtendDB) DeleteBackup of its source is BackupInUseException. @@ -1628,6 +1731,7 @@ def test_backup_arn_shape_and_account_are_checked(dynamodb_client): _cleanup(client, backup_arn, source) +@restores_a_backup_of_this_server def test_empty_index_overrides_restore_no_indexes_and_sse_override_is_applied(dynamodb_client): client = dynamodb_client source = f"restore-noop-{uuid.uuid4().hex[:10]}" @@ -1678,6 +1782,7 @@ def test_empty_index_overrides_restore_no_indexes_and_sse_override_is_applied(dy _cleanup(client, backup_arn, plain, restored, source) +@restores_a_backup_of_this_server @pytest.mark.parametrize("billing", ["PROVISIONED", "PAY_PER_REQUEST"]) def test_index_overrides_listing_the_backups_indexes_restore_them(dynamodb_client, billing): """GlobalSecondaryIndexOverride and LocalSecondaryIndexOverride built from @@ -1848,6 +1953,7 @@ def test_index_overrides_listing_the_backups_indexes_restore_them(dynamodb_clien _cleanup(client, backup_arn, changed, restored, source) +@restores_a_backup_of_this_server def test_on_demand_throughput_override_is_applied(dynamodb_client): """OnDemandThroughputOverride is the restored table's on-demand throughput, equal to the backup's or changed (measured on DynamoDB). @@ -1903,8 +2009,23 @@ def test_on_demand_throughput_override_is_applied(dynamodb_client): @extenddb_only -def test_create_backup_of_a_vector_table_is_refused(dynamodb_client): - """Refused at CreateBackup, not at restore time.""" +def test_create_backup_of_a_vector_table_is_refused(dynamodb_client, backup_store_component): + """CreateBackup of a table with vector indexes: the outcome of the + server's configured path, read from EXTENDDB_TEST_BACKUP_STORE and + checked against the server's `/health` (`backup_store_component`). + + `none` (no `[backup.store]`, or the in-catalog create): refused at + CreateBackup with the ValidationException main gives, and no backup is + created. `filesystem` or `s3`: the store-backed create backs the table + up (the manifest records its vector indexes): CreateBackup answers + CREATING, DescribeBackup reaches AVAILABLE with the table's features, + ListBackups lists it, and RestoreTableFromBackup of it answers the + ValidationException naming the store, which PR 9a's restore job + replaces with the restore (its refusal then narrows to a target without + vector support). The test fails rather than accept either outcome when + the variable is unset (the fixture fails first), and prints which branch + ran. + """ from test_vector_index_query_scan import _error_message, _signed_post client = dynamodb_client @@ -1913,9 +2034,19 @@ def test_create_backup_of_a_vector_table_is_refused(dynamodb_client): "CreateTable", { "TableName": name, - "AttributeDefinitions": [{"AttributeName": "pk", "AttributeType": "S"}], + "AttributeDefinitions": [ + {"AttributeName": "pk", "AttributeType": "S"}, + {"AttributeName": "g", "AttributeType": "S"}, + ], "KeySchema": [{"AttributeName": "pk", "KeyType": "HASH"}], "BillingMode": "PAY_PER_REQUEST", + "GlobalSecondaryIndexes": [ + { + "IndexName": "by-g", + "KeySchema": [{"AttributeName": "g", "KeyType": "HASH"}], + "Projection": {"ProjectionType": "KEYS_ONLY"}, + } + ], "VectorIndexes": [ { "IndexName": "vidx", @@ -1930,25 +2061,66 @@ def test_create_backup_of_a_vector_table_is_refused(dynamodb_client): if resp.status_code == 400 and "not supported" in _error_message(resp): pytest.skip("target backend does not support vector indexes") assert resp.status_code == 200, resp.text + backup_arn = None try: wait_for_active(client, name, timeout=180) vector = _signed_post("DescribeTable", {"TableName": name}).json()["Table"] if not vector.get("VectorIndexes"): pytest.skip("target accepted CreateTable but did not create the vector index") + client.put_item( + TableName=name, + Item={"pk": {"S": "a"}, "g": {"S": "g1"}, "emb": {"L": [{"N": "0.1"}] * 4}}, + ) + if BACKUP_STORE == "none": + print("branch: the in-catalog create refuses a vector table") + # A plain call, so a create that is accepted against expectation + # leaves its ARN for the cleanup rather than a backup nobody + # deletes. + try: + accepted = client.create_backup(TableName=name, BackupName=f"{name}-bkp") + except ClientError as e: + error = e.response["Error"] + else: + backup_arn = accepted["BackupDetails"]["BackupArn"] + pytest.fail( + f"the in-catalog create accepted a vector table: {accepted['BackupDetails']}" + ) + assert error["Code"] == "ValidationException" + assert error["Message"] == ( + f"table {name} has 1 vector index(es); restoring a table with vector " + "indexes is not supported by this storage backend" + ) + listed = client.list_backups(TableName=name)["BackupSummaries"] + assert listed == [], listed + return + print(f"branch: the {BACKUP_STORE} store backs a vector table up") + details = client.create_backup(TableName=name, BackupName=f"{name}-bkp")["BackupDetails"] + backup_arn = details["BackupArn"] + assert details["BackupStatus"] == "CREATING", details + wait_for_backup_available(client, backup_arn) + desc = client.describe_backup(BackupArn=backup_arn)["BackupDescription"] + assert desc["BackupDetails"]["BackupStatus"] == "AVAILABLE", desc + assert desc["SourceTableDetails"]["TableName"] == name + assert desc["SourceTableDetails"]["ItemCount"] == 1, desc["SourceTableDetails"] + gsis = desc["SourceTableFeatureDetails"]["GlobalSecondaryIndexes"] + assert [g["IndexName"] for g in gsis] == ["by-g"], desc["SourceTableFeatureDetails"] + listed = client.list_backups(TableName=name)["BackupSummaries"] + assert [b["BackupArn"] for b in listed] == [backup_arn], listed + # Until PR 9a's restore job, the restore of a backup in the store is + # refused; PR 9a replaces this assertion with the round trip. with pytest.raises(ClientError) as exc_info: - client.create_backup(TableName=name, BackupName=f"{name}-bkp") + client.restore_table_from_backup(TargetTableName=f"{name}-r", BackupArn=backup_arn) error = exc_info.value.response["Error"] - assert error["Code"] == "ValidationException" + assert error["Code"] == "ValidationException", error assert error["Message"] == ( - f"table {name} has 1 vector index(es); restoring a table with vector " - "indexes is not supported by this storage backend" - ) - listed = client.list_backups(TableName=name)["BackupSummaries"] - assert listed == [], listed + f"Backup {backup_arn} is in a backup store; this release restores in-catalog " + "backups only" + ), error finally: - _cleanup(client, None, name) + _cleanup(client, backup_arn, name) +@restores_a_backup_of_this_server def test_update_table_on_a_restored_table_reports_no_restore_summary(dynamodb_client): """Once a restored table is ACTIVE, no description-returning response carries RestoreSummary: UpdateTable's no more than DescribeTable's. @@ -1981,6 +2153,7 @@ def test_update_table_on_a_restored_table_reports_no_restore_summary(dynamodb_cl _cleanup(client, backup_arn, restored, source) +@restores_a_backup_of_this_server @extenddb_only def test_a_provisioned_gsi_without_throughput_restores_with_the_table_throughput( dynamodb_client, @@ -2349,6 +2522,7 @@ def test_create_backup_accepts_the_table_arn(dynamodb_client): _cleanup(client, backup_arn, source) +@restores_a_backup_of_this_server def test_table_class_is_not_restored(dynamodb_client): """A STANDARD_INFREQUENT_ACCESS source restores as STANDARD (measured on DynamoDB); DescribeBackup still reports nothing of the diff --git a/tests/test_backup_store_config.py b/tests/test_backup_store_config.py index bf7926444..1f71f22bd 100644 --- a/tests/test_backup_store_config.py +++ b/tests/test_backup_store_config.py @@ -10,11 +10,11 @@ with a filesystem store, `checking` while an S3 store's first check is in flight, unhealthy (status `degraded`) while an S3 endpoint refuses connections or stalls, and healthy again once it answers, without a restart; -- the restore and delete of a store-backed backup refused with a - ValidationException naming the store's kind and the problem's category and - not its root while the store is checking or unhealthy; CreateBackup served - throughout (this release's create writes the in-catalog format, which - touches no store); +- the restore and delete of a store-backed backup, and CreateBackup (which + writes to the store), refused with a ValidationException naming the + store's kind and the problem's category and not its root while the store + is checking or unhealthy, and CreateBackup writing a backup under the + store's root once it is healthy; - a stalled S3 endpoint holding the listener for no longer than the server's own startup, and each reconciler cycle for no longer than the check's bound; - the configuration errors that stop startup with the key, path or bucket in @@ -41,6 +41,7 @@ import json import os +import re import signal import socket import ssl @@ -174,6 +175,18 @@ def _client(port: int, creds: dict): ) +def _wait_backup_available(ddb, arn: str, timeout: float = 60.0) -> None: + """Poll DescribeBackup to AVAILABLE; any other status fails.""" + deadline = time.monotonic() + timeout + while True: + status = ddb.describe_backup(BackupArn=arn)["BackupDescription"]["BackupDetails"]["BackupStatus"] + if status == "AVAILABLE": + return + assert status == "CREATING", f"backup {arn} is {status}" + assert time.monotonic() < deadline, f"backup {arn} not AVAILABLE in {timeout} s" + time.sleep(0.1) + + def _create_table(ddb, name: str) -> None: ddb.create_table( TableName=name, @@ -426,14 +439,38 @@ def test_default_root_is_created_private_and_reported_healthy(self, sqlite_deplo # The console banner names the store beside the storage line. assert " storage: sqlite" in log, log assert f" backup store: filesystem path {root} (the default beside the database file)" in log, log - # CreateBackup takes the in-catalog path and says the store is configured. + # CreateBackup writes to the store: CREATING answered, the job's + # objects under the root, the row AVAILABLE once the manifest is + # written, and no deprecation warning. creds = _provision(d.port, d.cert, ADMIN_PASSWORD) ddb = _client(d.port, creds) _create_table(ddb, "Music") + ddb.put_item(TableName="Music", Item={"pk": {"S": "a"}, "n": {"N": "1"}}) details = ddb.create_backup(TableName="Music", BackupName="nightly")["BackupDetails"] - assert details["BackupStatus"] in ("CREATING", "AVAILABLE") - assert "writing the in-catalog backup format; the configured filesystem backup store" in server.log_text() - assert "configure [backup.store]" not in server.log_text() + assert details["BackupStatus"] == "CREATING", details + arn = details["BackupArn"] + _wait_backup_available(ddb, arn) + backup_id = arn.rsplit("/", 1)[1] + prefix = os.path.join(root, "123456789012", "Music", backup_id) + assert sorted(os.listdir(prefix)) == [ + "data", "extenddb-manifest.json", "extenddb-manifest.sha256", + "manifest-files.json", "manifest-summary.json", + ], os.listdir(prefix) + assert (os.stat(os.path.join(prefix, "extenddb-manifest.json")).st_mode & 0o777) == 0o600 + with open(os.path.join(prefix, "extenddb-manifest.json")) as f: + manifest = json.load(f) + assert manifest["backup_arn"] == arn and manifest["source_backend"] == "sqlite", manifest + described = ddb.describe_backup(BackupArn=arn)["BackupDescription"]["BackupDetails"] + assert described["BackupSizeBytes"] == manifest["backup_size_bytes"] > 0, described + log = server.log_text() + assert f"backup {arn}: started, " in log and "in the table as last counted by the stats pass, key-ordered" in log, log + # The completion line carries the backup's own count, and its + # bytes count the documents beside the data files. + written = re.search(rf"backup {re.escape(arn)}: AVAILABLE, (\d+) items, (\d+) bytes written in", log) + assert written and int(written.group(1)) == 1, log + assert int(written.group(2)) > manifest["backup_size_bytes"], log + assert "in-catalog backup format" not in log, log + assert "configure [backup.store]" not in log, log finally: server.stop() @@ -551,10 +588,13 @@ def test_an_unreachable_s3_endpoint_starts_the_server_unhealthy_and_recovers( creds = _provision(d.port, d.cert, ADMIN_PASSWORD) ddb = _client(d.port, creds) _create_table(ddb, "Music") - # The in-catalog create touches no store: it runs on an unhealthy - # one, as it does on a server with none. - details = ddb.create_backup(TableName="Music", BackupName="nightly")["BackupDetails"] - assert details["BackupStatus"] in ("CREATING", "AVAILABLE") + # CreateBackup writes to the store, so it is refused while the + # store is unhealthy, naming the kind and the category, not the + # bucket; no row is created. + with pytest.raises(ClientError) as refused: + ddb.create_backup(TableName="Music", BackupName="nightly") + _assert_store_refusal(refused.value, "s3", bucket, category="unreachable") + assert ddb.list_backups(TableName="Music")["BackupSummaries"] == [] log = server.log_text() assert "is unhealthy: the store could not be reached" in log, log assert bucket in log, "the operator's line names the bucket" @@ -566,11 +606,25 @@ def test_an_unreachable_s3_endpoint_starts_the_server_unhealthy_and_recovers( ) assert health["status"] == "healthy" assert "is healthy again" in server.log_text() - details = ddb.create_backup(TableName="Music", BackupName="nightly2")["BackupDetails"] - assert details["BackupStatus"] in ("CREATING", "AVAILABLE") - # The identity file was created on the first passing check. + # The identity file was created on the first passing check, and + # CreateBackup now writes a backup into the bucket. body = _s3_client().get_object(Bucket=bucket, Key="extenddb-store.json")["Body"].read() assert json.loads(body)["backend"] == "sqlite" + ddb.put_item(TableName="Music", Item={"pk": {"S": "a"}, "n": {"N": "1"}}) + details = ddb.create_backup(TableName="Music", BackupName="nightly2")["BackupDetails"] + assert details["BackupStatus"] == "CREATING", details + arn = details["BackupArn"] + _wait_backup_available(ddb, arn) + keys = sorted( + o["Key"].rsplit("/", 1)[1] + for o in _s3_client().list_objects_v2( + Bucket=bucket, Prefix=f"123456789012/Music/{arn.rsplit('/', 1)[1]}/" + )["Contents"] + ) + assert keys == [ + "000001.json.gz", "extenddb-manifest.json", "extenddb-manifest.sha256", + "manifest-files.json", "manifest-summary.json", + ], keys # And degrades again once the endpoint is gone. forwarder.stop() @@ -579,8 +633,9 @@ def test_an_unreachable_s3_endpoint_starts_the_server_unhealthy_and_recovers( d.port, lambda h: h["components"]["backup_store"]["status"] == "unhealthy", timeout=30, server=server ) assert health["status"] == "degraded" - details = ddb.create_backup(TableName="Music", BackupName="again")["BackupDetails"] - assert details["BackupStatus"] in ("CREATING", "AVAILABLE"), "the in-catalog create is not gated" + with pytest.raises(ClientError) as refused: + ddb.create_backup(TableName="Music", BackupName="again") + _assert_store_refusal(refused.value, "s3", bucket, category="unreachable") finally: if forwarder is not None: forwarder.stop() @@ -590,9 +645,9 @@ def test_a_stalled_s3_endpoint_holds_neither_the_listener_nor_the_reconciler(sel """An endpoint that accepts connections and never answers. Checked: the server binds and serves before the store's first check, which runs after the listener is up (`/health` answers well within the SDK's - attempt timeout with the component `checking`); the in-catalog - CreateBackup is served meanwhile; the restore of a seeded store-backed - row is refused naming the pending check; the check is abandoned at its + attempt timeout with the component `checking`); CreateBackup and the + restore of a seeded store-backed row are refused naming the pending + check; the check is abandoned at its 30 s bound, the component turns `unhealthy` (`unreachable`) without a restart and the refusal names that; and the reconciler keeps cycling while the endpoint stalls (the next cycle's check opens a new @@ -645,14 +700,16 @@ def accept_and_hold(): log = server.log_text() assert "listening on" in log and "run after the listener is up" in log, log assert "backup store: s3" in log and "tarpit-bucket" in log, log - # Requests are served while the check hangs: the in-catalog create - # runs, and the seeded row's restore is refused naming the pending - # check and the store's kind, not the bucket. + # Requests are served while the check hangs: the table is + # created, and CreateBackup and the seeded row's restore are + # refused naming the pending check and the store's kind, not the + # bucket. creds = _provision(d.port, d.cert, ADMIN_PASSWORD) ddb = _client(d.port, creds) _create_table(ddb, "Music") - details = ddb.create_backup(TableName="Music", BackupName="nightly")["BackupDetails"] - assert details["BackupStatus"] in ("CREATING", "AVAILABLE") + with pytest.raises(ClientError) as refused: + ddb.create_backup(TableName="Music", BackupName="nightly") + _assert_store_refusal(refused.value, "s3", "tarpit-bucket", category="first check is pending") with pytest.raises(ClientError) as refused: ddb.restore_table_from_backup(TargetTableName="Restored", BackupArn=arn) _assert_store_refusal(refused.value, "s3", "tarpit-bucket", category="first check is pending") @@ -684,8 +741,9 @@ def accept_and_hold(): assert accepted_at[1] - first < 45.0, f"the second connection came {accepted_at[1] - first:.1f} s after the first" assert "is unhealthy" in log and log.count("is unhealthy: the store could not be reached") == 1, \ "the transition is logged once" - details = ddb.create_backup(TableName="Music", BackupName="nightly2")["BackupDetails"] - assert details["BackupStatus"] in ("CREATING", "AVAILABLE") + with pytest.raises(ClientError) as refused: + ddb.create_backup(TableName="Music", BackupName="nightly2") + _assert_store_refusal(refused.value, "s3", "tarpit-bucket", category="unreachable") finally: server.stop() tarpit.close() @@ -739,10 +797,11 @@ def test_a_default_root_with_rows_is_unhealthy_rather_than_unconfigured(self, sq with pytest.raises(ClientError) as refused: ddb.delete_backup(BackupArn=arn) _assert_store_refusal(refused.value, "filesystem", root, category="manifest probe failed") - # The in-catalog create is served on the unhealthy store. + # CreateBackup is refused on the unhealthy store too. _create_table(ddb, "Music") - details = ddb.create_backup(TableName="Music", BackupName="nightly")["BackupDetails"] - assert details["BackupStatus"] in ("CREATING", "AVAILABLE") + with pytest.raises(ClientError) as refused: + ddb.create_backup(TableName="Music", BackupName="nightly") + _assert_store_refusal(refused.value, "filesystem", root, category="manifest probe failed") # The manifest put back: the next cycle recovers the store. for level in (os.path.dirname(os.path.dirname(manifest_dir)), os.path.dirname(manifest_dir), manifest_dir): if not os.path.isdir(level): @@ -819,8 +878,8 @@ def test_a_catalog_re_created_after_destroy_warns_about_the_old_default_root(sel """`destroy` then `init --overwrite` on the same directory: the default root's identity file names the old catalog. The server starts with the store unconfigured and a warning naming the file (not degraded, not a - startup error), CreateBackup keeps the in-catalog path, and `destroy`'s - output said to remove the file.""" + startup error), CreateBackup takes the in-catalog path with its + deprecation warning, and `destroy`'s output said to remove the file.""" d = sqlite_deployment server = _Server(d.config, str(tmp_path / "serve.log")) try: