diff --git a/.github/workflows/integration-cassandra.yml b/.github/workflows/integration-cassandra.yml index 703372dd0..8bad0fbb5 100644 --- a/.github/workflows/integration-cassandra.yml +++ b/.github/workflows/integration-cassandra.yml @@ -92,8 +92,10 @@ jobs: - name: Direct storage-trait integration tests # Ported from the plug-in repo's tests/rust suite. Parallel-safe: # each test provisions its own account and keyspace; the shared - # control-plane setting test restores what it mutates. - run: cargo test -p extenddb-storage-cassandra --test direct_integration + # control-plane setting test restores what it mutates. The + # test-hooks feature compiles the tests that pause an operation or + # shorten a heartbeat (crates/storage-cassandra/src/test_hooks.rs). + run: cargo test -p extenddb-storage-cassandra --features test-hooks --test direct_integration env: EXTENDDB_TEST_CASSANDRA: required diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 202d6441b..c3b4d96c8 100755 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -291,26 +291,27 @@ change is code in `crates/storage-mongodb/src/bootstrapper.rs`. ### Example: a nullable column on `tables` -Migration 5, `table_note`, adds a nullable `note TEXT` column to the catalog's -`tables` table on SQLite and PostgreSQL. +Migration 6, `table_note` (6 being the next free id at the time of writing; +migration 5 is `backup_catalog_index`), adds a nullable `note TEXT` column to +the catalog's `tables` table on SQLite and PostgreSQL. In `crates/storage/src/migrations/registry.rs`, at the end of the `migration_ids!` list: ```rust /// Adds a nullable `note` column to the `tables` catalog table. - TableNote = 5, "table_note", + TableNote = 6, "table_note", "add a nullable note column to the tables catalog"; ``` PostgreSQL has `ADD COLUMN IF NOT EXISTS`, so its body is a file, -`crates/storage-postgres/migrations/0005_table_note.sql`: +`crates/storage-postgres/migrations/0006_table_note.sql`: ```sql -- Copyright 2026 ExtendDB contributors -- SPDX-License-Identifier: Apache-2.0 -- --- Migration 5, table_note, PostgreSQL catalog database: a nullable note +-- Migration 6, table_note, PostgreSQL catalog database: a nullable note -- column on tables. IF NOT EXISTS makes a second run change nothing. ALTER TABLE tables ADD COLUMN IF NOT EXISTS note TEXT; @@ -325,7 +326,7 @@ In `crates/storage-postgres/src/migrations/mod.rs`, the arm in `migration()`: bodies: vec![file_body!( StoreKind::Catalog, "migrations", - "0005_table_note.sql" + "0006_table_note.sql" )], }, ``` @@ -336,19 +337,20 @@ and the id in the transactional arm of `runs_in()`: MigrationId::Baseline | MigrationId::ResourceTagConditionKeys | MigrationId::SortKeyDefinitions + | MigrationId::BackupCatalogIndex | MigrationId::TableNote => RunsIn::Transaction, ``` SQLite has no `ADD COLUMN IF NOT EXISTS`, and a file with a plain `ADD COLUMN` fails when it runs a second time, so the SQLite body is code that checks for the column first. -`crates/storage-sqlite/src/migrations/m0005_table_note.rs`: +`crates/storage-sqlite/src/migrations/m0006_table_note.rs`: ```rust // Copyright 2026 ExtendDB contributors // SPDX-License-Identifier: Apache-2.0 -//! Migration 5, `table_note`, SQLite: a nullable `note` column on `tables`. +//! Migration 6, `table_note`, SQLite: a nullable `note` column on `tables`. //! //! SQLite has no `ADD COLUMN IF NOT EXISTS`, so the body reads the table's //! columns first and adds the column only when it is missing. A second run @@ -360,10 +362,10 @@ use extenddb_storage::migrations::{CodeBody, Done, MigrationError}; use super::SqliteStores; /// This file, whose SHA-256 the checksum test pins. -pub(crate) const SOURCE: &str = include_str!("m0005_table_note.rs"); +pub(crate) const SOURCE: &str = include_str!("m0006_table_note.rs"); /// Recorded as this migration's checksum. Changes only with a new migration. -const FINGERPRINT: &str = "0005_table_note/sqlite/1"; +const FINGERPRINT: &str = "0006_table_note/sqlite/1"; pub(crate) struct TableNote; @@ -402,7 +404,7 @@ beside the others: ```rust #[rustfmt::skip] #[allow(clippy::all, deprecated)] -mod m0005_table_note; +mod m0006_table_note; ``` and the arm in `migration()`: @@ -413,8 +415,8 @@ and the arm in `migration()`: convergent: true, bodies: vec![StoreBody::code( StoreKind::Catalog, - &m0005_table_note::TableNote, - m0005_table_note::SOURCE, + &m0006_table_note::TableNote, + m0006_table_note::SOURCE, )], }, ``` @@ -426,7 +428,7 @@ rows of `tables`: ```rust #[tokio::test] -async fn migration_5_adds_the_note_column_to_tables() { +async fn migration_6_adds_the_note_column_to_tables() { let db = temp_db("m5"); load_dump(&db, DEV_0_1_14).await; run(&db).await.expect("migrate"); @@ -437,12 +439,12 @@ async fn migration_5_adds_the_note_column_to_tables() { ) .await; text.push_str(&dump(&db, "SELECT table_name, note FROM tables ORDER BY table_name").await); - golden("0005_table_note", &text); + golden("0006_table_note", &text); } ``` -`EXTENDDB_BLESS_EXPECTED=1 cargo test -p extenddb-storage-sqlite --lib migration_5` -writes `crates/storage-sqlite/tests/fixtures/golden/0005_table_note.txt`, one +`EXTENDDB_BLESS_EXPECTED=1 cargo test -p extenddb-storage-sqlite --lib migration_6` +writes `crates/storage-sqlite/tests/fixtures/golden/0006_table_note.txt`, one tab-separated line per row under each query: ``` @@ -457,15 +459,15 @@ The PostgreSQL body is a file, so it has no golden test. The example declares `Kind::Breaking`. A nullable column can be additive by the rule in step 4, but declaring it additive needs the old-binary job, which this example does not add. As a breaking migration it makes a 1.x binary -without migration 5 refuse a catalog that has it, and a rollback restores the +without migration 6 refuse a catalog that has it, and a rollback restores the backup. The same PR changes the code that reads and writes `tables.note` in each backend's catalog store, and commits the `docs/schema/sqlite.txt` and `docs/schema/postgres.txt` that the integration jobs regenerate. It changes no per-table DDL, because `tables` is a catalog table, and adds no `RELEASED` -entry. `extenddb migrate --list` then shows migration 5, and on a catalog at -migration 4 `extenddb migrate` lists migration 5 as pending and exits 3. +entry. `extenddb migrate --list` then shows migration 6, and on a catalog at +migration 5 `extenddb migrate` lists migration 6 as pending and exits 3. ### Running the migration tests diff --git a/Cargo.lock b/Cargo.lock index ffe80fb18..443c6c8fa 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1817,6 +1817,7 @@ dependencies = [ "extenddb-auth", "extenddb-core", "extenddb-storage", + "futures", "hex", "hmac 0.12.1", "serde", diff --git a/crates/app/src/init_helpers.rs b/crates/app/src/init_helpers.rs index 406db8b0e..15e700f3f 100755 --- a/crates/app/src/init_helpers.rs +++ b/crates/app/src/init_helpers.rs @@ -300,6 +300,20 @@ key_path = "{tls_key}" # max_import_item_count = 10000000 # max_export_item_count = 10000000 +# [backup] +# Backup and restore jobs. stale_job_after_secs must be at least three +# heartbeat periods, the larger of heartbeat_interval_secs and the 15 s the +# jobs of this release heartbeat at (45 at the least); pool_size is raised to +# at least 2 * max_concurrent_jobs + 10 on PostgreSQL; max_concurrent_jobs +# is at most 1024. +# data_file_target_bytes = 67108864 # Uncompressed bytes per data file (minimum 32 MiB) +# heartbeat_interval_secs = 15 # Job heartbeat timer +# stale_job_after_secs = 300 # Heartbeat age after which a job's owner is dead +# failed_row_retention_secs = 604800 # FAILED tombstones kept for the operator (7 days) +# reconcile_interval_secs = 60 # Reconciler cycle +# max_concurrent_jobs = 4 # Backup and restore jobs running at once on this instance +# spool_path = "/var/lib/extenddb/spool" # Restore download area for an S3 store + # [import] # paths = [] # Allowed directories for import operations diff --git a/crates/config/src/backup.rs b/crates/config/src/backup.rs new file mode 100644 index 000000000..92339fa4f --- /dev/null +++ b/crates/config/src/backup.rs @@ -0,0 +1,317 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! The `[backup]` section: the knobs backup and restore jobs read. +//! +//! ```toml +//! [backup] +//! data_file_target_bytes = 67108864 # uncompressed bytes per data file; minimum 32 MiB +//! heartbeat_interval_secs = 15 # job heartbeat timer +//! stale_job_after_secs = 300 # heartbeat age after which a job's owner is dead +//! failed_row_retention_secs = 604800 # FAILED tombstones kept for the operator, 7 days +//! reconcile_interval_secs = 60 # reconciler cycle +//! max_concurrent_jobs = 4 # backup and restore jobs running at once on this instance +//! # spool_path = "/var/lib/extenddb/spool" # restore download area for an S3 store +//! ``` +//! +//! `stale_job_after_secs` must be at least three heartbeat periods: a job +//! checks before each store write that its latest heartbeat confirmed its +//! claim within one period, and the margin is what makes that detection +//! precede a reconciler's deletion of the job's prefix. The period the floor +//! is computed from is the one the jobs run at: the larger of +//! `heartbeat_interval_secs` and [`JOB_HEARTBEAT_INTERVAL`] (15 s), since +//! until a later release wires the knob to the jobs every backend heartbeats +//! at that constant, and a floor on a shorter configured interval would let +//! the reconciler declare a live job dead between two of its heartbeats. The +//! floor is a floor: on SQLite a heartbeat waits for the engine write lock +//! like any writer, so a long write transaction delays it, and a +//! `stale_job_after_secs` near three periods on a busy file can declare a +//! live restore dead. +//! +//! `max_concurrent_jobs` is at most [`MAX_CONCURRENT_JOBS`]: a value over +//! the runtime's permit ceiling would stop the server at startup rather than +//! at load, and the PostgreSQL pool rule makes anything over a few hundred +//! meaningless anyway. +//! +//! The store itself (`[backup.store]`) is configured separately and is not +//! part of this struct. + +use extenddb_storage::backup_catalog::JOB_HEARTBEAT_INTERVAL; +use serde::Deserialize; + +/// The smallest `data_file_target_bytes`: the cap on one line of a data +/// file, so a file closes at most one line past the knob. +pub const MIN_DATA_FILE_TARGET_BYTES: u64 = 32 * 1024 * 1024; + +/// The smallest ratio of `stale_job_after_secs` to the heartbeat period +/// ([`BackupConfig::effective_heartbeat_interval_secs`]). +pub const MIN_STALE_HEARTBEATS: u64 = 3; + +/// The largest `max_concurrent_jobs`. +pub const MAX_CONCURRENT_JOBS: usize = 1024; + +#[derive(Debug, Clone, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct BackupConfig { + /// Uncompressed bytes per data file; a file closes once it reaches this, + /// so it exceeds it by less than one line. + #[serde(default = "default_data_file_target_bytes")] + pub data_file_target_bytes: u64, + /// The job heartbeat timer. + #[serde(default = "default_heartbeat_interval_secs")] + pub heartbeat_interval_secs: u64, + /// Heartbeat age after which a job's owner is dead, and the delay before + /// the reconciler deletes a dead job's prefix. + #[serde(default = "default_stale_job_after_secs")] + pub stale_job_after_secs: u64, + /// How long FAILED tombstones are kept for the operator. + #[serde(default = "default_failed_row_retention_secs")] + pub failed_row_retention_secs: u64, + /// The reconciler's cycle. + #[serde(default = "default_reconcile_interval_secs")] + pub reconcile_interval_secs: u64, + /// Backup and restore jobs running at once on this instance, the + /// in-catalog creates and restores included; a further CreateBackup or + /// RestoreTableFromBackup is `LimitExceededException`. + #[serde(default = "default_max_concurrent_jobs")] + pub max_concurrent_jobs: usize, + /// Restore download area for an S3 store. Defaults beside the database + /// file on SQLite; required on the other backends once a store is + /// configured. + #[serde(default)] + pub spool_path: Option, +} + +impl Default for BackupConfig { + fn default() -> Self { + Self { + data_file_target_bytes: default_data_file_target_bytes(), + heartbeat_interval_secs: default_heartbeat_interval_secs(), + stale_job_after_secs: default_stale_job_after_secs(), + failed_row_retention_secs: default_failed_row_retention_secs(), + reconcile_interval_secs: default_reconcile_interval_secs(), + max_concurrent_jobs: default_max_concurrent_jobs(), + spool_path: None, + } + } +} + +impl BackupConfig { + /// The period the jobs heartbeat at: the larger of + /// `heartbeat_interval_secs` and [`JOB_HEARTBEAT_INTERVAL`], which is + /// what every backend runs at until the knob reaches the jobs. + #[must_use] + pub fn effective_heartbeat_interval_secs(&self) -> u64 { + self.heartbeat_interval_secs + .max(JOB_HEARTBEAT_INTERVAL.as_secs()) + } + + /// The cross-field rules, checked at load. + /// + /// # Errors + /// + /// A message naming the key and the rule it breaks. + pub fn validate(&self) -> Result<(), String> { + if self.heartbeat_interval_secs == 0 { + return Err("backup.heartbeat_interval_secs must be at least 1".to_owned()); + } + let period = self.effective_heartbeat_interval_secs(); + let floor = period.saturating_mul(MIN_STALE_HEARTBEATS); + if self.stale_job_after_secs < floor { + return Err(format!( + "backup.stale_job_after_secs = {} is under {MIN_STALE_HEARTBEATS} * the job \ + heartbeat period of {period} s = {floor} (the larger of \ + backup.heartbeat_interval_secs = {} and the {} s the jobs heartbeat at in \ + this release); a reconciler could declare a running job dead between two \ + of its heartbeats", + self.stale_job_after_secs, + self.heartbeat_interval_secs, + JOB_HEARTBEAT_INTERVAL.as_secs() + )); + } + if self.data_file_target_bytes < MIN_DATA_FILE_TARGET_BYTES { + return Err(format!( + "backup.data_file_target_bytes = {} is under the minimum of \ + {MIN_DATA_FILE_TARGET_BYTES} (32 MiB, the cap on one line of a data file)", + self.data_file_target_bytes + )); + } + if self.reconcile_interval_secs == 0 { + return Err("backup.reconcile_interval_secs must be at least 1".to_owned()); + } + if self.max_concurrent_jobs == 0 { + return Err("backup.max_concurrent_jobs must be at least 1".to_owned()); + } + if self.max_concurrent_jobs > MAX_CONCURRENT_JOBS { + return Err(format!( + "backup.max_concurrent_jobs = {} is over the maximum of {MAX_CONCURRENT_JOBS}", + self.max_concurrent_jobs + )); + } + Ok(()) + } +} + +fn default_data_file_target_bytes() -> u64 { + 64 * 1024 * 1024 +} +fn default_heartbeat_interval_secs() -> u64 { + 15 +} +fn default_stale_job_after_secs() -> u64 { + 300 +} +fn default_failed_row_retention_secs() -> u64 { + 7 * 24 * 3600 +} +fn default_reconcile_interval_secs() -> u64 { + 60 +} +fn default_max_concurrent_jobs() -> usize { + 4 +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn the_defaults_are_the_documented_ones() { + let c = BackupConfig::default(); + assert_eq!(c.data_file_target_bytes, 67_108_864); + assert_eq!(c.heartbeat_interval_secs, 15); + assert_eq!(c.stale_job_after_secs, 300); + assert_eq!(c.failed_row_retention_secs, 604_800); + assert_eq!(c.reconcile_interval_secs, 60); + assert_eq!(c.max_concurrent_jobs, 4); + assert_eq!(c.spool_path, None); + c.validate().unwrap(); + let parsed: BackupConfig = toml::from_str("").unwrap(); + assert_eq!(parsed, c); + } + + #[test] + fn an_unknown_key_is_refused() { + let e = toml::from_str::("store_kind = \"s3\"\n").unwrap_err(); + assert!(e.to_string().contains("store_kind"), "{e}"); + } + + #[test] + fn a_stale_window_under_three_heartbeats_is_refused() { + let c = BackupConfig { + heartbeat_interval_secs: 20, + stale_job_after_secs: 59, + ..BackupConfig::default() + }; + let e = c.validate().unwrap_err(); + assert!(e.contains("stale_job_after_secs = 59"), "{e}"); + assert!(e.contains("= 60"), "{e}"); + let ok = BackupConfig { + stale_job_after_secs: 60, + ..c + }; + ok.validate().unwrap(); + } + + /// The floor is computed from the period the jobs run at, not from a + /// configured interval shorter than it: `heartbeat_interval_secs = 1` + /// with `stale_job_after_secs = 3` would let the reconciler remove a live + /// restore between two 15 s heartbeats, and is refused naming the 15 s + /// period and the 45 s floor. + #[test] + fn the_stale_floor_is_three_times_the_period_the_jobs_heartbeat_at() { + let c = BackupConfig { + heartbeat_interval_secs: 1, + stale_job_after_secs: 3, + ..BackupConfig::default() + }; + assert_eq!(c.effective_heartbeat_interval_secs(), 15); + let e = c.validate().unwrap_err(); + assert!(e.contains("stale_job_after_secs = 3 "), "{e}"); + assert!(e.contains("period of 15 s = 45"), "{e}"); + assert!(e.contains("heartbeat_interval_secs = 1 "), "{e}"); + let under = BackupConfig { + stale_job_after_secs: 44, + ..c.clone() + }; + assert!(under.validate().is_err()); + let ok = BackupConfig { + stale_job_after_secs: 45, + ..c + }; + ok.validate().unwrap(); + // A configured interval over the constant is the period. + let long = BackupConfig { + heartbeat_interval_secs: 40, + stale_job_after_secs: 120, + ..BackupConfig::default() + }; + assert_eq!(long.effective_heartbeat_interval_secs(), 40); + long.validate().unwrap(); + } + + #[test] + fn max_concurrent_jobs_has_an_upper_bound() { + let at_most = BackupConfig { + max_concurrent_jobs: MAX_CONCURRENT_JOBS, + ..BackupConfig::default() + }; + at_most.validate().unwrap(); + let over = BackupConfig { + max_concurrent_jobs: MAX_CONCURRENT_JOBS + 1, + ..BackupConfig::default() + }; + let e = over.validate().unwrap_err(); + assert!(e.contains("max_concurrent_jobs = 1025"), "{e}"); + assert!(e.contains("1024"), "{e}"); + let huge = BackupConfig { + max_concurrent_jobs: usize::MAX, + ..BackupConfig::default() + }; + assert!(huge.validate().is_err()); + } + + #[test] + fn the_other_floors_are_checked() { + let small_file = BackupConfig { + data_file_target_bytes: MIN_DATA_FILE_TARGET_BYTES - 1, + ..BackupConfig::default() + }; + assert!( + small_file + .validate() + .unwrap_err() + .contains("data_file_target_bytes") + ); + let no_jobs = BackupConfig { + max_concurrent_jobs: 0, + ..BackupConfig::default() + }; + assert!( + no_jobs + .validate() + .unwrap_err() + .contains("max_concurrent_jobs") + ); + let no_heartbeat = BackupConfig { + heartbeat_interval_secs: 0, + ..BackupConfig::default() + }; + assert!( + no_heartbeat + .validate() + .unwrap_err() + .contains("heartbeat_interval_secs") + ); + let no_cycle = BackupConfig { + reconcile_interval_secs: 0, + ..BackupConfig::default() + }; + assert!( + no_cycle + .validate() + .unwrap_err() + .contains("reconcile_interval_secs") + ); + } +} diff --git a/crates/config/src/lib.rs b/crates/config/src/lib.rs index 4cc5cf123..5b1cb603a 100644 --- a/crates/config/src/lib.rs +++ b/crates/config/src/lib.rs @@ -29,6 +29,9 @@ pub struct AppConfig { pub limits: LimitsConfig, #[serde(default)] pub logging: LoggingConfig, + /// The knobs backup and restore jobs read ([`backup::BackupConfig`]). + #[serde(default)] + pub backup: backup::BackupConfig, /// Import configuration. Lists allowed source directories for import /// operations. If empty or absent, imports are denied (secure default). #[serde(default, rename = "import")] @@ -92,8 +95,8 @@ impl Default for ServerConfig { impl ServerConfig { /// Operator-facing warnings for server settings that work but cost - /// something: today a `region` outside `[a-z0-9-]`, which makes the - /// backup ARNs the server issues unresolvable (see + /// something: today a `region` outside `[a-z0-9-]`, under which + /// `CreateBackup` is refused (see /// [`extenddb_storage::config::server_region_warning`]). Shown in the /// `serve` banner; the same warning is logged when the storage components /// are created. @@ -417,7 +420,13 @@ pub fn load(config_path: &str) -> anyhow::Result { .add_source(config::File::with_name(config_path).required(false)) .add_source(config::Environment::with_prefix("EXTENDDB").separator("__")) .build()?; - Ok(config.try_deserialize()?) + validated(config.try_deserialize()?) +} + +/// The cross-field checks a deserialized config still has to pass. +fn validated(config: AppConfig) -> anyhow::Result { + config.backup.validate().map_err(|e| anyhow::anyhow!(e))?; + Ok(config) } /// Load `AppConfig` entirely from built-in defaults and environment variables, @@ -448,7 +457,7 @@ pub fn load_builtin_defaults() -> anyhow::Result { )) .add_source(config::Environment::with_prefix("EXTENDDB").separator("__")) .build()?; - Ok(config.try_deserialize()?) + validated(config.try_deserialize()?) } /// Redact password from a connection string for safe logging (REQ-LOG-002). @@ -495,6 +504,7 @@ pub fn pid_file_path_default(port: u16) -> PathBuf { pid_file_path(&ServerConfig::default().run_dir, port) } +pub mod backup; mod display; pub use display::{MASK, build_config_entries, should_redact}; diff --git a/crates/core/src/backup_format/layout.rs b/crates/core/src/backup_format/layout.rs index 7dd912bae..5c2f87ff2 100644 --- a/crates/core/src/backup_format/layout.rs +++ b/crates/core/src/backup_format/layout.rs @@ -53,9 +53,9 @@ pub const DATA_PREFIX: &str = "data/"; /// the engine accepted is a valid ARN component. The engine's /// `max_table_name_length` can be raised above 255; a table with a longer /// name has a backup ARN `check_backup_arn` accepts and this type refuses, -/// so the off-host backup path cannot write it (the design has `CreateBackup` -/// refuse it up front). -const MAX_TABLE_NAME_LEN: usize = 255; +/// so the off-host backup path cannot write it (`CreateBackup` refuses it up +/// front through [`check_store_prefix_table_name`]). +pub const MAX_TABLE_NAME_LEN: usize = 255; /// The key of the `index`-th data file within a backup prefix: /// `data/000001.json.gz` for index 1. @@ -195,11 +195,41 @@ fn ascii_word(s: &str, allowed: &[u8]) -> bool { } /// The region shape a backup ARN may carry: non-empty `[a-z0-9-]`. The -/// manifest checks a table ARN's region with the same predicate. -pub(super) fn is_region(s: &str) -> bool { +/// manifest checks a table ARN's region with the same predicate, the engine +/// checks `server.region` with it before forming a backup ARN at +/// `CreateBackup`, and the startup warning about `server.region` speaks for +/// it. +#[must_use] +pub fn is_region(s: &str) -> bool { ascii_word(s, b"-") && !s.bytes().any(|b| b.is_ascii_uppercase()) } +/// The store-prefix rules on a table name, the two inputs [`check_backup_arn`] +/// accepts that a [`BackupArn`] refuses: a name over [`MAX_TABLE_NAME_LEN`] +/// bytes, or made only of dots (`.` and `..` are path components; a longer +/// run of dots is a legal `DynamoDB` name but is refused to keep the rule +/// simple). The engine applies them at `CreateBackup`, before the ARN is +/// formed, so the refusal names the rule whatever the name's length. +/// +/// # Errors +/// +/// [`FormatError::InvalidArn`] naming the rule the name breaks. +pub fn check_store_prefix_table_name(table_name: &str) -> Result<(), FormatError> { + if table_name.len() > MAX_TABLE_NAME_LEN { + return Err(FormatError::InvalidArn(format!( + "table name of {} bytes is longer than {MAX_TABLE_NAME_LEN}", + table_name.len() + ))); + } + if table_name.bytes().all(|b| b == b'.') { + return Err(FormatError::InvalidArn(format!( + "table name {:?} is only dots", + truncated(table_name) + ))); + } + Ok(()) +} + /// Strip surrounding ASCII whitespace from a `BackupArn`. Measured on /// `DynamoDB`: an ARN with a trailing newline or CRLF resolves, and one with a /// leading space or tab is refused with `Invalid Backup ARN`. This parser @@ -377,22 +407,12 @@ impl BackupArn { }) } - /// The store-prefix rules over already-checked parts: the two rules - /// `check_backup_arn` does not apply, and the only two inputs it accepts - /// that this type refuses. + /// The store-prefix rules over already-checked parts + /// ([`check_store_prefix_table_name`]): the two rules `check_backup_arn` + /// does not apply, and the only two inputs it accepts that this type + /// refuses. fn from_parts(parts: &BackupArnParts<'_>) -> Result { - if parts.table_name.len() > MAX_TABLE_NAME_LEN { - return Err(FormatError::InvalidArn(format!( - "table name of {} bytes is longer than {MAX_TABLE_NAME_LEN}", - parts.table_name.len() - ))); - } - if parts.table_name.bytes().all(|b| b == b'.') { - return Err(FormatError::InvalidArn(format!( - "table name {:?} is only dots", - truncated(parts.table_name) - ))); - } + check_store_prefix_table_name(parts.table_name)?; Ok(Self { partition: parts.partition.to_owned(), region: parts.region.to_owned(), @@ -752,8 +772,22 @@ mod tests { assert!(BackupArn::new(" aws", "us-east-1", "1234", "Music", ID).is_err()); assert!(BackupArn::new("aws", "us-east-1", "1234", "Music", &format!("{ID}\n")).is_err()); assert!(BackupArn::new("aws", "us-east-1", "1234", "Music", &format!("{ID} ")).is_err()); - // Dots inside names are fine; DynamoDB allows them. - BackupArn::new("aws", "us-east-1", "1234", "a.b.c", ID).unwrap(); + // Dots inside names are fine; DynamoDB allows them. So are a leading + // dot, a trailing dot and a run of dots beside other characters: the + // rule refuses a name made only of dots, not one that starts with + // them. + for dotted in ["a.b.c", ".cfg", "a.", "a..b", "..a", "a.."] { + BackupArn::new("aws", "us-east-1", "1234", dotted, ID) + .unwrap_or_else(|e| panic!("{dotted}: {e}")); + check_store_prefix_table_name(dotted).unwrap_or_else(|e| panic!("{dotted}: {e}")); + } + for only_dots in [".", "..", "...", "...."] { + let err = check_store_prefix_table_name(only_dots).unwrap_err(); + assert!( + err.to_string().contains("is only dots"), + "{only_dots}: {err}" + ); + } } #[test] diff --git a/crates/core/src/backup_format/mod.rs b/crates/core/src/backup_format/mod.rs index 9287a9124..ee88a6c93 100644 --- a/crates/core/src/backup_format/mod.rs +++ b/crates/core/src/backup_format/mod.rs @@ -85,9 +85,9 @@ pub use key_order::{KeyOrder, hash_key_text}; pub use layout::{ BACKUP_ID_MILLIS_DIGITS, BackupArn, BackupArnError, BackupArnParts, DATA_PREFIX, EXTENDDB_MANIFEST_FILE, FILES_MANIFEST_FILE, MANIFEST_CHECKSUM_FILE, MAX_BACKUP_ARN_LEN, - SUMMARY_MANIFEST_FILE, backup_creation_date_time, backup_id_epoch_millis, check_backup_arn, - data_file_key, format_backup_arn, format_backup_id, is_backup_id, parse_backup_arn, - trim_backup_arn, + MAX_TABLE_NAME_LEN, SUMMARY_MANIFEST_FILE, backup_creation_date_time, backup_id_epoch_millis, + check_backup_arn, check_store_prefix_table_name, data_file_key, format_backup_arn, + format_backup_id, is_backup_id, is_region, parse_backup_arn, trim_backup_arn, }; pub use manifest::{ CURRENT_FORMAT_VERSION, DataFileEntry, DataFileReader, ExtenddbManifest, ItemOrder, KeyRange, diff --git a/crates/core/src/metrics/types.rs b/crates/core/src/metrics/types.rs index 39d9d07f4..85811deaa 100755 --- a/crates/core/src/metrics/types.rs +++ b/crates/core/src/metrics/types.rs @@ -129,6 +129,8 @@ pub enum QuerySource { ControlPlane, /// Background: login attempt cleanup. LoginAttemptCleanup, + /// Background: the backup reconciler. + BackupReconciler, /// Management CLI commands. Management, } @@ -149,6 +151,7 @@ impl std::fmt::Display for QuerySource { Self::TableSizeRefresh => f.write_str("TableSizeRefresh"), Self::ControlPlane => f.write_str("ControlPlane"), Self::LoginAttemptCleanup => f.write_str("LoginAttemptCleanup"), + Self::BackupReconciler => f.write_str("BackupReconciler"), Self::Management => f.write_str("Management"), } } diff --git a/crates/engine/Cargo.toml b/crates/engine/Cargo.toml index 3634c9152..9f58bea6e 100755 --- a/crates/engine/Cargo.toml +++ b/crates/engine/Cargo.toml @@ -21,3 +21,4 @@ hmac = "0.12" sha2 = "0.10" hex = "0.4" tokio = { workspace = true } +futures = { workspace = true } diff --git a/crates/engine/src/backup.rs b/crates/engine/src/backup.rs index 67114a013..8b7301ad7 100755 --- a/crates/engine/src/backup.rs +++ b/crates/engine/src/backup.rs @@ -16,6 +16,7 @@ use std::sync::Arc; +use extenddb_core::backup_format::{BackupArn, check_store_prefix_table_name, is_region}; use extenddb_core::error::{DynamoDbError, ErrorMessageKey, error_message}; use extenddb_core::limits::LimitsConfig; use extenddb_core::types::{ @@ -23,11 +24,19 @@ use extenddb_core::types::{ LocalSecondaryIndexInfo, ProvisionedThroughput, RestoreSummary, SourceTableFeatureDetails, SseDescription, TableDescription, TableStatus, }; -use extenddb_storage::BackupEngine; +use extenddb_storage::backup_catalog::{ + BackupKey, BackupRow, BackupStatus, DeleteStart, TimeBounds, is_backup_not_found, +}; use extenddb_storage::error::StorageError; -use extenddb_storage::{BackupRestoreSource, RestoreRequest, RestoreTableOverrides}; +use extenddb_storage::{ + BackupCatalog, BackupRestoreSource, LegacyBackupReader, RestoreRequest, RestoreTableOverrides, + TableEngine, +}; +use futures::TryStreamExt; use serde_json::{Value, json}; +pub use runtime::BackupRuntime; + use crate::{OperationContext, serialize_output}; /// `ListBackups` page size when the request gives no `Limit`, and the largest @@ -84,20 +93,37 @@ fn backup_arn_field( /// The spellings of one `BackupArn` the engine looks a backup up under: /// the ARN this deployment issues for the backup the caller named /// ([`canonical_backup_arn`]) and, when it differs, the ARN as the caller -/// wrote it, trimmed. +/// wrote it, trimmed. The strings are what the legacy reader is addressed +/// by, built as main built them; each is also parsed once, here, into the +/// [`BackupArn`] the catalog index is addressed by. `BackupArn` applies two +/// store-prefix rules `check_backup_arn` does not (a table name over 255 +/// bytes, or made only of dots), which a legacy backup can break: main's +/// CreateTable and CreateBackup accept `...`, and an operator can raise +/// `limits.max_table_name_length`. A spelling that fails them (or one built +/// under a `server.region` outside the ARN alphabet, which the legacy path +/// never checked) has no parsed form and no catalog-index read (no index row +/// can carry such an ARN, since `check_backup_arn_fits` refuses the name at +/// CreateBackup), and the legacy reader is tried under its string as on +/// main. #[derive(Debug, Clone, PartialEq, Eq)] pub(crate) struct BackupArnSpellings { canonical: String, as_written: Option, + canonical_arn: Option, + as_written_arn: Option, } impl BackupArnSpellings { fn new(parts: &extenddb_storage::util::BackupArnParts<'_>, region: &str) -> Self { let canonical = canonical_backup_arn(parts, region); let as_written = (parts.arn != canonical).then(|| parts.arn.to_owned()); + let canonical_arn = BackupArn::parse(&canonical).ok(); + let as_written_arn = as_written.as_deref().and_then(|a| BackupArn::parse(a).ok()); Self { canonical, as_written, + canonical_arn, + as_written_arn, } } @@ -105,11 +131,57 @@ impl BackupArnSpellings { fn canonical(&self) -> &str { &self.canonical } + + /// The parsed spellings in lookup order: canonical, then as written; + /// none for a name the store-prefix rules refuse. + fn arns(&self) -> impl Iterator { + self.canonical_arn + .iter() + .chain(self.as_written_arn.as_ref()) + } +} + +/// The catalog row one of `arn`'s spellings names, read in [`resolve_backup`]'s +/// order, with the spelling that resolved. A row read by account and backup +/// id is returned by the backend only when its account, table and ARN equal +/// the spelling's, so an ARN of account B carrying the id of account A's +/// backup finds nothing (and never A's row). `None` when neither spelling +/// has a row: the caller falls back to the legacy reader. +async fn read_catalog_row( + storage: &S, + arn: &BackupArnSpellings, +) -> Result, StorageError> { + for spelling in arn.arns() { + if let Some(row) = storage.read_backup_row(spelling).await? { + return Ok(Some((spelling.clone(), row))); + } + } + Ok(None) +} + +/// The error for a backup the catalog index has no describable row for. +fn backup_not_found(arn: &str) -> DynamoDbError { + DynamoDbError::BackupNotFoundException(format!( + "{}: {arn}", + extenddb_storage::backup_catalog::BACKUP_NOT_FOUND + )) +} + +/// The error for a CREATING row, or one whose backup a CREATING restore +/// target still reads: `BackupInUseException`, as the service answers. +fn backup_in_use(arn: &str, restoring_into: Option<&str>) -> DynamoDbError { + DynamoDbError::BackupInUseException(match restoring_into { + Some(table) => extenddb_storage::backup_catalog::backup_in_use_message(table, arn), + None => format!("Backup is being created: {arn}"), + }) } -/// Whether a backend error says the backup is not there. -fn is_backup_not_found(e: &StorageError) -> bool { - matches!(e, StorageError::Validation(m) if m.contains("Backup not found")) +/// The error for a row of a store other than the configured one. Names no +/// root: an API error never does. +fn backup_in_another_store(arn: &str) -> DynamoDbError { + DynamoDbError::ValidationException(format!( + "Backup {arn} is in a backup store other than the one this server is configured with" + )) } /// Look a backup up by `lookup`, first under the canonical ARN, then, if that @@ -136,8 +208,8 @@ where } } -/// [`resolve_backup`]'s lookup for one `BackupEngine` method taking the ARN: -/// `by!(storage, account_id, method)`. +/// [`resolve_backup`]'s lookup for one [`LegacyBackupReader`] method taking +/// the ARN: `by!(storage, account_id, method)`. macro_rules! by { ($storage:expr, $account_id:expr, $method:ident) => {{ let storage = $storage; @@ -255,8 +327,8 @@ fn table_name_bounds(limits: &LimitsConfig) -> (usize, usize) { (limits.min_table_name_length, limits.max_table_name_length) } -/// The `TableName` of `CreateBackup`, which the service accepts as the -/// table's name or its ARN (measured on DynamoDB; the model +/// The `TableName` of `CreateBackup` and `ListBackups`, which the service +/// accepts as the table's name or its ARN (measured on DynamoDB; the model /// gives the member the `TableName` shape, max 255, so an SDK built from it /// refuses the ARN form client-side and only a hand-built request sends it). /// An ARN is `arn:aws:dynamodb:::table/`: another @@ -313,13 +385,7 @@ pub(crate) async fn handle_create_backup( let table_name = create_backup_table_name(&body, &ctx.account_id, &ctx.limits)?; let backup_name = required_str(&body, "BackupName", "backupName")?; validate_name_member(backup_name, "backupName", BACKUP_NAME_BOUNDS)?; - - // A backup of a table with vector indexes could never be restored (the - // restore refuses it), so refuse it here with the restore's own reason - // rather than hand back an AVAILABLE backup the customer finds out about - // at restore time. A lookup failure is left to create_backup, which - // reports a missing table in its own terms. - if let Ok(desc) = ctx + let probe = ctx .storage .describe_table( &ctx.account_id, @@ -327,18 +393,108 @@ pub(crate) async fn handle_create_backup( table_name: table_name.to_owned(), }, ) - .await - { - refuse_backup_of_vector_table(&desc)?; - } + .await; + let details = create_backup( + &*ctx.storage, + &ctx.backup, + &ctx.account_id, + &ctx.region, + table_name, + backup_name, + probe, + ) + .await?; + serialize_output(&json!({ "BackupDetails": details })) +} - let details = ctx - .storage - .create_backup(&ctx.account_id, table_name, backup_name) +/// One CreateBackup after its members are checked, in the service's order, +/// every check before the permit: the table exists and is ACTIVE for the +/// account (`TableNotFoundException`, "Table not found: ", otherwise, +/// whether or not a permit is free); the server's region and the table name +/// fit a backup ARN and its store prefix (`check_backup_arn_fits`: a +/// `server.region` outside `[a-z0-9-]` is a `ValidationException` naming the +/// setting, a name over 255 bytes or made only of dots one naming the rule); +/// the table has no vector index +/// (the in-catalog format cannot restore one); then the job permit +/// (`LimitExceededException` when none is free). `probe` is the +/// `DescribeTable` of the table, which the handler ran. +/// +/// This is the in-catalog create, the path while no `[backup.store]` is +/// configured; each call logs that the format is deprecated. +async fn create_backup( + storage: &S, + runtime: &BackupRuntime, + account_id: &str, + region: &str, + table_name: &str, + backup_name: &str, + probe: Result, +) -> Result { + let desc = match probe { + Ok(desc) if desc.table_status == TableStatus::Active => desc, + Ok(_) => { + return Err(create_backup_storage_err_to_dynamo( + StorageError::TableNotActive(table_name.to_owned()), + table_name, + )); + } + Err(e) => return Err(create_backup_storage_err_to_dynamo(e, table_name)), + }; + check_backup_arn_fits(region, account_id, table_name)?; + refuse_backup_of_vector_table(&desc)?; + let permit = runtime.take_permit()?; + tracing::warn!( + "CreateBackup of {table_name} (account {account_id}) is writing the in-catalog backup \ + format, which a later release stops creating; configure [backup.store] to write \ + backups to a backup store" + ); + storage + .create(account_id, table_name, backup_name, permit) .await - .map_err(|e| create_backup_storage_err_to_dynamo(e, table_name))?; + .map_err(|e| create_backup_storage_err_to_dynamo(e, table_name)) +} - serialize_output(&json!({ "BackupDetails": details })) +/// The backup ARN rules a table name has to fit before a backup of it can be +/// created, each refusal naming its cause: first the server's own region +/// (`server.region` outside `[a-z0-9-]` forms no backup ARN whatever the +/// name, so the message names the setting, not the table), then the two +/// store-prefix rules on the name (over 255 bytes, or made only of dots; +/// [`check_store_prefix_table_name`] names the limit whatever the name's +/// length, where forming the ARN first would answer "not a backup ARN" for a +/// name that takes the ARN over 1024 bytes), then the ARN itself. The id is +/// a placeholder of the right shape; only the region and the name are being +/// checked. +fn check_backup_arn_fits( + region: &str, + account_id: &str, + table_name: &str, +) -> Result<(), DynamoDbError> { + if !is_region(region) { + return Err(DynamoDbError::ValidationException(format!( + "CreateBackup is refused on this server: server.region {region:?} is outside \ + [a-z0-9-], so no backup ARN can be formed; set server.region to a value in \ + that class" + ))); + } + check_store_prefix_table_name(table_name).map_err(|e| { + DynamoDbError::ValidationException(format!( + "TableName cannot be backed up: a backup ARN cannot be formed for it: {e}" + )) + })?; + let placeholder = extenddb_core::backup_format::format_backup_id(0, 0); + BackupArn::new( + extenddb_storage::util::ARN_PARTITION, + region, + account_id, + table_name, + &placeholder, + ) + .map(drop) + .map_err(|e| { + DynamoDbError::ValidationException(format!( + "TableName cannot be backed up: a backup ARN cannot be formed for it: {e}" + )) + }) } /// [`storage_err_to_dynamo`] for CreateBackup, where a table that does not @@ -379,21 +535,33 @@ pub(crate) async fn handle_describe_backup( serialize_output(&json!({ "BackupDescription": desc })) } -/// The DescribeBackup body: the backend's description, with +/// The DescribeBackup body. The catalog index is consulted first: a row of +/// one of the ARN's spellings answers on its own, derived from its recorded +/// source table ([`BackupRow::description`]), with DELETING reported as +/// DELETED and a DELETED or FAILED row `BackupNotFoundException`. With no +/// row, the legacy reader's description, with /// `SourceTableDetails.ProvisionedThroughput` and `SourceTableFeatureDetails` /// filled from the backup's recorded source when the backup is AVAILABLE. /// The backup is looked up under the ARN's spellings in [`resolve_backup`]'s /// order; the source is then read under the spelling that resolved. -async fn describe_backup( +async fn describe_backup( storage: &S, account_id: &str, region: &str, backup_arn: &BackupArnSpellings, ) -> Result { - let (backup_arn, mut desc) = - resolve_backup(backup_arn, by!(storage, account_id, describe_backup)) - .await - .map_err(storage_err_to_dynamo)?; + if let Some((arn, row)) = read_catalog_row(storage, backup_arn) + .await + .map_err(storage_err_to_dynamo)? + { + if !row.status.is_described() { + return Err(backup_not_found(&arn.to_arn_string())); + } + return row.description().map_err(storage_err_to_dynamo); + } + let (backup_arn, mut desc) = resolve_backup(backup_arn, by!(storage, account_id, describe)) + .await + .map_err(storage_err_to_dynamo)?; let backup_arn = backup_arn.as_str(); // A CREATING or DELETED backup has no restore source to read; it is @@ -408,9 +576,9 @@ async fn describe_backup( // answer for a database failure, while RestoreTableFromBackup on the same // read is a 500. let source = if desc.backup_details.backup_status == "AVAILABLE" { - match storage.backup_restore_source(account_id, backup_arn).await { + match storage.restore_source(account_id, backup_arn).await { Ok(source) => Some(source), - Err(StorageError::Validation(m)) if m.contains("Backup not found") => None, + Err(e) if is_backup_not_found(&e) => None, Err(StorageError::Unsupported(m)) => { tracing::warn!("DescribeBackup {backup_arn}: definition not described: {m}"); None @@ -524,10 +692,14 @@ pub(crate) async fn handle_list_backups( serialize_output(&page) } -/// `TableName` here is a table name only: unlike `CreateBackup`, the ARN form -/// is refused with the pattern clause (the service's behaviour for -/// `ListBackups` with an ARN was not measured). -async fn list_backups( +/// `TableName` is a table name or a table ARN, as `CreateBackup` accepts it +/// (the service accepts both, measured on DynamoDB). The listing merges the +/// catalog index's rows (CREATING and AVAILABLE ones, summarized from their +/// recorded source table) with the legacy reader's summaries into one list, +/// which is then filtered and paged in (`BackupCreationDateTime`, ARN) order: +/// `created_at` is the id's instant on both sides, so a cursor places itself +/// in the merged order the same way whichever side issued it. +async fn list_backups( storage: &S, account_id: &str, region: &str, @@ -560,10 +732,7 @@ async fn list_backups( }; let table_name = match body.get("TableName") { None | Some(Value::Null) => None, - Some(Value::String(name)) => { - validate_name_member(name, "tableName", table_name_bounds(limits))?; - Some(name.as_str()) - } + Some(Value::String(_)) => Some(create_backup_table_name(body, account_id, limits)?), Some(_) => { return Err(DynamoDbError::SerializationException( "Start of structure or map found where not expected: TableName".to_owned(), @@ -584,10 +753,41 @@ async fn list_backups( }; let filter = ListBackupsFilter::parse(body)?; - let summaries = storage - .list_backups(account_id, table_name) + let bounds = TimeBounds { + lower: filter.lower, + upper: filter.upper, + }; + // A row whose summary does not derive (a `source_table_details_json` + // this build cannot read) is logged and left out, so one such row does + // not turn every ListBackups of the account into an internal error; a + // failure to read the rows propagates. + let mut summaries: Vec = storage + .list_account_backup_rows(account_id, table_name, &bounds) + .try_filter_map(|row| async move { + if !row.status.is_listed() { + return Ok(None); + } + match row.summary() { + Ok(summary) => Ok(Some(summary)), + Err(e) => { + tracing::warn!( + "ListBackups (account {account_id}): backup {} left out of the \ + listing: {e}", + row.backup_arn + ); + Ok(None) + } + } + }) + .try_collect() .await .map_err(storage_err_to_dynamo)?; + summaries.extend( + storage + .list(account_id, table_name) + .await + .map_err(storage_err_to_dynamo)?, + ); let start = start.map(|cursor| cursor.resolve(&summaries)).transpose()?; let summaries = summaries.into_iter().filter(|s| filter.keeps(s)).collect(); let (page, last) = page_backups( @@ -812,7 +1012,14 @@ pub(crate) async fn handle_delete_backup( ctx: &OperationContext, ) -> Result { let backup_arn = backup_arn_field(&body, &ctx.account_id, &ctx.region)?; - let desc = delete_backup(&*ctx.storage, &ctx.account_id, &ctx.region, &backup_arn).await?; + let desc = delete_backup( + &*ctx.storage, + &ctx.backup, + &ctx.account_id, + &ctx.region, + &backup_arn, + ) + .await?; serialize_output(&json!({ "BackupDescription": desc })) } @@ -820,6 +1027,9 @@ pub(crate) async fn handle_delete_backup( /// as it was, with `BackupStatus` `DELETED` (measured on DynamoDB). `SourceTableDetails.ProvisionedThroughput` is a required member /// of the shape and a typed SDK can fail to unmarshal a response without it. /// +/// The catalog index is consulted first ([`delete_catalog_backup`]); with no +/// row of either spelling, the legacy reader's delete. +/// /// The restore source is read before the delete, since it is gone after, /// under the ARN spelling that resolves ([`resolve_backup`]); the delete then /// names that spelling. A backup the read does not find under either @@ -832,14 +1042,21 @@ pub(crate) async fn handle_delete_backup( /// means a damaged `PROVISIONED` backup is described with on-demand `{0, 0}` /// (ExtendDB's choice: deletability over the throughput member); a failure to /// reach the database propagates, as the delete itself would fail on it. -async fn delete_backup( +async fn delete_backup( storage: &S, + runtime: &BackupRuntime, account_id: &str, region: &str, backup_arn: &BackupArnSpellings, ) -> Result { + if let Some((arn, row)) = read_catalog_row(storage, backup_arn) + .await + .map_err(storage_err_to_dynamo)? + { + return delete_catalog_backup(storage, runtime, &arn, &row).await; + } let (resolved, source) = - match resolve_backup(backup_arn, by!(storage, account_id, backup_restore_source)).await { + match resolve_backup(backup_arn, by!(storage, account_id, restore_source)).await { Ok((resolved, source)) => (Some(resolved), Some(source)), Err(e) if is_backup_not_found(&e) => (None, None), Err(StorageError::Unsupported(m) | StorageError::Internal(m)) => { @@ -852,8 +1069,8 @@ async fn delete_backup( Err(e) => return Err(storage_err_to_dynamo(e)), }; let mut desc = match resolved { - Some(resolved) => storage.delete_backup(account_id, &resolved).await, - None => resolve_backup(backup_arn, by!(storage, account_id, delete_backup)) + Some(resolved) => storage.delete(account_id, &resolved).await, + None => resolve_backup(backup_arn, by!(storage, account_id, delete)) .await .map(|(_, desc)| desc), } @@ -863,6 +1080,78 @@ async fn delete_backup( Ok(desc) } +/// DeleteBackup of a backup the catalog index has a row for. In the order +/// the request's checks run, the same order as a restore's +/// ([`check_catalog_row_for_restore`]): a DELETED or FAILED row is +/// `BackupNotFoundException`; a CREATING row is `BackupInUseException`; a +/// backup a CREATING restore target still reads is `BackupInUseException` +/// naming that table ([`BackupCatalog::backup_in_use`]); a row of a store +/// other than the configured one is a `ValidationException` naming no root; +/// then the in-use check runs again with the AVAILABLE-to-DELETING write in +/// one transaction under the row's lock +/// ([`BackupCatalog::start_backup_delete`]), so a restore's registration and +/// this order wholly one before the other: a registration that won leaves +/// this `BackupInUseException` naming its target. A DELETING row (a retried +/// DeleteBackup) resumes the deletion. The request returns the description +/// with `BackupStatus` DELETED; the store's objects are deleted by the +/// duty a configured store brings, and the row stays DELETING until a +/// listing of its prefix returns nothing. +/// +/// The description is built before the DELETING write: a row whose +/// `source_table_details_json` this build cannot read is answered as an +/// internal error with the row still AVAILABLE, rather than moved to DELETING +/// and then refused on every retry. +async fn delete_catalog_backup( + storage: &S, + runtime: &BackupRuntime, + arn: &BackupArn, + row: &BackupRow, +) -> Result { + let arn_text = arn.to_arn_string(); + match row.status { + BackupStatus::Deleted | BackupStatus::Failed => return Err(backup_not_found(&arn_text)), + BackupStatus::Creating => return Err(backup_in_use(&arn_text, None)), + BackupStatus::Available | BackupStatus::Deleting => {} + } + let key = BackupKey::from_arn(arn); + if let Some(table) = storage + .backup_in_use(&key) + .await + .map_err(storage_err_to_dynamo)? + { + return Err(backup_in_use(&arn_text, Some(&table))); + } + if !runtime.is_configured_store(&row.store) { + return Err(backup_in_another_store(&arn_text)); + } + let mut desc = row.description().map_err(storage_err_to_dynamo)?; + match storage + .start_backup_delete(&key) + .await + .map_err(storage_err_to_dynamo)? + { + DeleteStart::Deleting | DeleteStart::Status(BackupStatus::Deleting) => {} + DeleteStart::InUse(table) => return Err(backup_in_use(&arn_text, Some(&table))), + DeleteStart::Status(BackupStatus::Creating) => { + return Err(backup_in_use(&arn_text, None)); + } + DeleteStart::Status(BackupStatus::Deleted | BackupStatus::Failed) => { + return Err(backup_not_found(&arn_text)); + } + // The contract says an AVAILABLE row becomes `Deleting`; a backend + // that answers its status instead has broken it, which is reported + // as the fault it is rather than as an absent backup. + DeleteStart::Status(BackupStatus::Available) => { + return Err(storage_err_to_dynamo(StorageError::Internal(format!( + "start_backup_delete of {arn_text} answered Status(AVAILABLE), which the \ + contract excludes" + )))); + } + } + desc.backup_details.backup_status = BackupStatus::Deleted.as_str().to_owned(); + Ok(desc) +} + /// Overrides refused outright. `VectorIndexOverride` is not in the botocore /// 1.33.13 model; newer models and SDKs send it. The index overrides and /// `OnDemandThroughputOverride` are applied as the service applies them; see @@ -969,6 +1258,7 @@ pub(crate) async fn handle_restore_table_from_backup( let auth_cache = ctx.auth_cache.clone(); restore_table_from_backup_detached( Arc::clone(&ctx.storage), + Arc::clone(&ctx.backup), Arc::clone(&ctx.account_id), Arc::clone(&ctx.region), Arc::clone(&ctx.limits), @@ -1001,6 +1291,7 @@ pub(crate) async fn handle_restore_table_from_backup( /// the request before the backend registered the restore, and no restore ran. async fn restore_table_from_backup_detached( storage: Arc, + runtime: Arc, account_id: Arc, region: Arc, limits: Arc, @@ -1008,13 +1299,14 @@ async fn restore_table_from_backup_detached( after: F, ) -> Result where - S: BackupEngine + ?Sized + Send + Sync + 'static, + S: BackupCatalog + LegacyBackupReader + TableEngine + ?Sized + Send + Sync + 'static, F: FnOnce(Arc, String) -> Fut + Send + 'static, Fut: std::future::Future + Send, { tokio::spawn(async move { let (target_table_name, response) = - restore_table_from_backup(&*storage, &account_id, ®ion, &limits, &body).await?; + restore_table_from_backup(&*storage, &runtime, &account_id, ®ion, &limits, &body) + .await?; after(account_id, target_table_name).await; Ok(response) }) @@ -1029,17 +1321,30 @@ where /// order: (1) parse the overrides and check their shape, with no storage /// read; (2) check `BackupArn`: missing or malformed is a /// `ValidationException`, another account's is an `AccessDeniedException`; -/// (3) check `TargetTableName`; (4) read the backup's restore source; +/// (3) check `TargetTableName`: its syntax, then the name under the account +/// through `describe_table` ([`check_restore_target_name`]: a table of that +/// name is `TableAlreadyExistsException`, a CREATING restore target +/// `TableInUseException` naming its source backup), so a client retrying its +/// own restore is answered at once whatever the backup's state and whether +/// or not a permit is free; (4) read the backup: the catalog index's row +/// first ([`check_catalog_row_for_restore`]: a CREATING row or one in use is +/// `BackupInUseException`, a DELETING, DELETED or FAILED row +/// `BackupNotFoundException`, a row of another store a +/// `ValidationException`), else the legacy reader's restore source; /// (5) refuse what no backend restores faithfully (multi-part key, vector /// indexes); (6) check the throughput override against the backup's billing /// mode, build the target's `CreateTableInput`, and apply the index and /// on-demand throughput overrides to it, refusing what the service refuses -/// ([`overrides`]); (7) validate that input as CreateTable does; (8) ask the -/// backend to create and register the target; (9) map its errors, a taken -/// name being `TableAlreadyExistsException`, a name being restored into -/// `TableInUseException`, a backup being restored from `BackupInUseException`. -async fn restore_table_from_backup( +/// ([`overrides`]); (7) validate that input as CreateTable does; (8) take the +/// job permit, last among the checks (`LimitExceededException` when none is +/// free), and ask the backend to create and register the target; (9) map +/// its errors, a taken name being `TableAlreadyExistsException`, a name +/// being restored into `TableInUseException` (the backend checks the name +/// again, fenced, under the registration), a backup being restored from +/// `BackupInUseException`. +async fn restore_table_from_backup( storage: &S, + runtime: &BackupRuntime, account_id: &str, region: &str, limits: &LimitsConfig, @@ -1056,14 +1361,22 @@ async fn restore_table_from_backup( // first (measured on DynamoDB with two invalid inputs). let backup_arn = backup_arn_field(body, account_id, region)?; - // 3. + // 3. The name's syntax, then the name itself, before the backup is read. let target_table_name = required_str(body, "TargetTableName", "targetTableName")?; extenddb_core::validation::validate_table_name(target_table_name, limits)?; + check_restore_target_name(storage, account_id, target_table_name).await?; - // 4. Under the ARN spelling that resolves; the restore is registered - // against that spelling, the one the backend stored. + // 4. The catalog index first, then, with no row, the legacy reader under + // the ARN spelling that resolves; the restore is registered against that + // spelling, the one the backend stored. + if let Some((arn, row)) = read_catalog_row(storage, &backup_arn) + .await + .map_err(storage_err_to_dynamo)? + { + check_catalog_row_for_restore(storage, runtime, &arn, &row).await?; + } let (backup_arn, source) = - resolve_backup(&backup_arn, by!(storage, account_id, backup_restore_source)) + resolve_backup(&backup_arn, by!(storage, account_id, restore_source)) .await .map_err(storage_err_to_dynamo)?; @@ -1082,15 +1395,17 @@ async fn restore_table_from_backup( extenddb_core::validation::validate_create_table(&input, limits)?; let on_demand = input.billing_mode == Some(BillingMode::PayPerRequest); - // 8. + // 8. The permit last: every refusal above needs no slot, and a job + // that waited for one would hold what it opened while it waited. let request = RestoreRequest { backup_arn: backup_arn.clone(), source_table_arn: source.source_table_arn.clone(), restore_date_time: source.backup_creation_date_time, input, }; + let permit = runtime.take_permit()?; let desc = storage - .restore_table_from_backup(account_id, request) + .restore(account_id, request, permit) .await // 9. .map_err(restore_storage_err_to_dynamo)?; @@ -1099,6 +1414,84 @@ async fn restore_table_from_backup( Ok((target_table_name.to_owned(), response)) } +/// Step 4 of a restore for a backup the catalog index has a row for, before +/// the permit and reading no store: a CREATING row is `BackupInUseException`; +/// a DELETING, DELETED or FAILED row is `BackupNotFoundException`; a backup +/// a CREATING restore target still reads is `BackupInUseException` naming +/// that table ([`BackupCatalog::backup_in_use`]); a row of a store other +/// than the configured one is a `ValidationException` naming no root. An +/// AVAILABLE row of the configured store is a backup in a store, whose +/// restore this release does not run. +async fn check_catalog_row_for_restore( + storage: &S, + runtime: &BackupRuntime, + arn: &BackupArn, + row: &BackupRow, +) -> Result<(), DynamoDbError> { + let arn_text = arn.to_arn_string(); + match row.status { + BackupStatus::Creating => return Err(backup_in_use(&arn_text, None)), + BackupStatus::Deleting | BackupStatus::Deleted | BackupStatus::Failed => { + return Err(backup_not_found(&arn_text)); + } + BackupStatus::Available => {} + } + if let Some(table) = storage + .backup_in_use(&BackupKey::from_arn(arn)) + .await + .map_err(storage_err_to_dynamo)? + { + return Err(backup_in_use(&arn_text, Some(&table))); + } + if !runtime.is_configured_store(&row.store) { + return Err(backup_in_another_store(&arn_text)); + } + Err(DynamoDbError::ValidationException(format!( + "Backup {arn_text} is in a backup store; this release restores in-catalog backups only" + ))) +} + +/// Step 3 of a restore, the target name under the account, read through +/// `describe_table` before the backup and the permit: a CREATING table with a +/// `RestoreSummary` is a restore target, `TableInUseException` naming its +/// source backup (the text the backend's own registration answers with, +/// [`StorageError::TableBeingRestored`]); any other table of that name is +/// `TableAlreadyExistsException`; no table passes. Unfenced: the backend +/// checks the name again under its registration, so two restores into one +/// free name still have one winner. +async fn check_restore_target_name( + storage: &S, + account_id: &str, + target_table_name: &str, +) -> Result<(), DynamoDbError> { + let probe = storage + .describe_table( + account_id, + extenddb_core::types::DescribeTableInput { + table_name: target_table_name.to_owned(), + }, + ) + .await; + match probe { + Ok(desc) => { + let source = desc + .restore_summary + .as_ref() + .filter(|_| desc.table_status == TableStatus::Creating) + .and_then(|s| s.source_backup_arn.clone()); + Err(restore_storage_err_to_dynamo(match source { + Some(backup_arn) => StorageError::TableBeingRestored { + table_name: target_table_name.to_owned(), + backup_arn, + }, + None => StorageError::TableAlreadyExists(target_table_name.to_owned()), + })) + } + Err(StorageError::TableNotFound(_)) => Ok(()), + Err(e) => Err(storage_err_to_dynamo(e)), + } +} + /// The RestoreTableFromBackup response body for the backend's description. /// /// The service returns the target CREATING with `RestoreSummary` @@ -1235,7 +1628,7 @@ pub(crate) async fn handle_update_continuous_backups( /// faked a restore by snapshotting the current table state (ignoring /// `RestoreDateTime`), which violates tenet 1 (fidelity over features). /// Until real PITR is implemented, return an error. There is no storage -/// method behind this; it was removed from `BackupEngine` as unreachable. +/// method behind this; it was removed from the storage trait as unreachable. pub(crate) async fn handle_restore_table_to_point_in_time( _body: Value, _ctx: &OperationContext, @@ -1264,7 +1657,7 @@ fn storage_err_to_dynamo(e: StorageError) -> DynamoDbError { // A missing (or deleted) backup surfaces from the backend as a // Validation error carrying "Backup not found"; DynamoDB reports // this as BackupNotFoundException, not ResourceNotFoundException. - if msg.contains("Backup not found") { + if msg.contains(extenddb_storage::backup_catalog::BACKUP_NOT_FOUND) { DynamoDbError::BackupNotFoundException(msg) } else { DynamoDbError::ValidationException(msg) @@ -1286,5 +1679,7 @@ fn storage_err_to_dynamo(e: StorageError) -> DynamoDbError { } mod overrides; +pub mod reconciler; +mod runtime; #[cfg(test)] mod tests; diff --git a/crates/engine/src/backup/reconciler.rs b/crates/engine/src/backup/reconciler.rs new file mode 100644 index 000000000..a9c649714 --- /dev/null +++ b/crates/engine/src/backup/reconciler.rs @@ -0,0 +1,315 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! The reconciler: the periodic worker that resolves catalog rows whose +//! owner is dead. +//! +//! One cycle runs every `interval`. It first checks the catalog-level +//! maintenance claim; while a maintenance command holds it the reconciler +//! runs only the duties that resolve the states the command waits on, which +//! today are every duty it has, so the check is what decides when the other +//! duties are added. The duties, over the records `list_dead_restore_jobs` +//! returns: +//! +//! - A RUNNING restore job record whose `heartbeat_at` is older than +//! `stale_job_after` (on the catalog's clock) has a dead owner. One of kind +//! `legacy` is marked FAILED and its target removed through +//! `remove_restore_target`, never resumed: an in-catalog copy has no data +//! files or progress to resume from. One of kind `manifest` is left for the +//! duty that resumes it, and logged once per record for as long as this +//! process runs. +//! - A FAILED restore job record whose table is still a CREATING restore +//! target, the FAILED-target duty: the FAILED write landed (the job's own, +//! or an earlier cycle's) and the target's removal did not (an error +//! between the two statements, or the process stopped between them). The +//! backend lists it once the record is stale, and the target is removed +//! through `remove_restore_target` with no second FAILED write. +//! - An in-catalog CREATING backup row whose create is dead (reported by the +//! backend's `abandoned_creating_backups`) is removed through the reader's +//! `delete`, which takes its items with it. +//! +//! A cycle that fails logs and waits for the next; each cycle records the +//! worker metrics the other workers record (`WorkerLastSuccess` and +//! `WorkerCycleLatency` on success, `WorkerErrorCount` on failure, under +//! `QuerySource::BackupReconciler`); the worker stops at the shutdown token. + +use std::collections::HashSet; +use std::sync::Arc; +use std::time::Duration; + +use extenddb_core::metrics::{MetricsCollector, QuerySource}; +use extenddb_storage::backup_catalog::{JobStatus, RestoreJobRow, RestoreKind, RestoreTargetKey}; +use extenddb_storage::error::StorageError; +use extenddb_storage::{BackupCatalog, CancellationToken, LegacyBackupReader, sleep_or_shutdown}; + +/// The two durations a reconciler reads from `[backup]`. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct ReconcilerSettings { + /// `reconcile_interval_secs`. + pub interval: Duration, + /// `stale_job_after_secs`. + pub stale_job_after: Duration, +} + +/// The reason written on a restore job record whose owner died. +pub const OWNER_DEAD: &str = "owner dead: no heartbeat within stale_job_after_secs"; + +/// What one cycle did. +#[derive(Debug, Default, Clone, PartialEq, Eq)] +pub struct CycleReport { + /// Whether the maintenance claim was held during the cycle. + pub maintenance_held: bool, + /// Legacy restore targets failed and removed, by table name. + pub legacy_restores_removed: Vec, + /// Targets of FAILED records removed (the FAILED-target duty), by table + /// name. + pub failed_targets_removed: Vec, + /// Targets a removal found no longer a CREATING restore target (whether + /// this cycle wrote the FAILED or found it), left to their current owner, + /// by table name. + pub targets_left_to_owner: Vec, + /// Manifest restore jobs with a dead owner, left for their resumer. + pub manifest_restores_seen: Vec, + /// Those of `manifest_restores_seen` this cycle logged: the ones no + /// earlier cycle of this process had seen. + pub manifest_restores_logged: Vec, + /// In-catalog CREATING backup rows removed, by ARN. + pub legacy_backups_removed: Vec, +} + +/// What a reconciler carries from one cycle to the next: the manifest +/// records it has already logged, so a record it cannot act on is reported +/// once rather than every cycle. +#[derive(Debug, Default)] +pub struct ReconcilerState { + logged_manifest_restores: HashSet, +} + +/// Run cycles every `settings.interval` until `shutdown` fires. One cycle +/// runs at once at startup, so a restart resolves what the stopped process +/// left without waiting an interval. +pub async fn run( + storage: Arc, + settings: ReconcilerSettings, + metrics: Arc, + shutdown: CancellationToken, +) where + S: BackupCatalog + LegacyBackupReader + ?Sized, +{ + let mut state = ReconcilerState::default(); + loop { + let started = std::time::Instant::now(); + match cycle(&*storage, settings.stale_job_after, &mut state).await { + Ok(report) => { + #[allow(clippy::cast_precision_loss)] + let cycle_us = started.elapsed().as_micros() as f64; + metrics.record_worker_success(QuerySource::BackupReconciler, cycle_us); + if !report.legacy_restores_removed.is_empty() + || !report.failed_targets_removed.is_empty() + || !report.legacy_backups_removed.is_empty() + { + tracing::info!( + "backup reconciler: removed {} abandoned restore target(s) and {} \ + abandoned in-catalog backup(s)", + report.legacy_restores_removed.len() + report.failed_targets_removed.len(), + report.legacy_backups_removed.len() + ); + } + } + Err(e) => { + metrics.record_worker_error(QuerySource::BackupReconciler); + tracing::error!("backup reconciler cycle failed: {e}"); + } + } + if !sleep_or_shutdown(&shutdown, settings.interval).await { + return; + } + } +} + +/// One cycle: the maintenance check, then the duties. +/// +/// # Errors +/// +/// A catalog read that fails ends the cycle; a failed write on one row is +/// logged and the cycle goes on to the next row. +pub async fn cycle( + storage: &S, + stale_job_after: Duration, + state: &mut ReconcilerState, +) -> Result +where + S: BackupCatalog + LegacyBackupReader + ?Sized, +{ + let mut report = CycleReport { + maintenance_held: storage.check_maintenance_claim().await?, + ..CycleReport::default() + }; + if report.maintenance_held { + tracing::info!( + "backup reconciler: a maintenance claim is held; running only the duties that \ + resolve rows whose owner is dead" + ); + } + + for job in storage.list_dead_restore_jobs(stale_job_after).await? { + match (job.status, job.kind) { + // The FAILED-target duty, whatever the record's kind: the job is + // over and only its target is left. + (JobStatus::Failed, _) => { + match remove_restore_target( + storage, + &job, + "was FAILED when the reconciler reached it", + ) + .await + { + Removal::Removed => report + .failed_targets_removed + .push(job.target.table_name.clone()), + Removal::LeftToOwner => report + .targets_left_to_owner + .push(job.target.table_name.clone()), + Removal::Failed => {} + } + } + (_, RestoreKind::Legacy) => match remove_dead_legacy_restore(storage, &job).await { + Some(Removal::Removed) => report + .legacy_restores_removed + .push(job.target.table_name.clone()), + Some(Removal::LeftToOwner) => report + .targets_left_to_owner + .push(job.target.table_name.clone()), + Some(Removal::Failed) | None => {} + }, + (_, RestoreKind::Manifest) => { + if state.logged_manifest_restores.insert(job.target.clone()) { + tracing::info!( + "restore of {} into {} (account {}) has no live owner; a manifest \ + restore is resumed by a later release's reconciler (reported once)", + job.source_backup_arn, + job.target.table_name, + job.target.account_id + ); + report + .manifest_restores_logged + .push(job.target.table_name.clone()); + } + report + .manifest_restores_seen + .push(job.target.table_name.clone()); + } + } + } + + for backup in storage.abandoned_creating_backups(stale_job_after).await? { + match storage.delete(&backup.account_id, &backup.backup_arn).await { + Ok(_) => { + tracing::warn!( + "removed in-catalog backup {} (account {}): its CreateBackup did not finish \ + and no process owns it", + backup.backup_arn, + backup.account_id + ); + report.legacy_backups_removed.push(backup.backup_arn); + } + Err(e) => tracing::error!( + "could not remove the abandoned in-catalog backup {} (account {}): {e}", + backup.backup_arn, + backup.account_id + ), + } + } + Ok(report) +} + +/// Mark a dead legacy restore FAILED and remove its target. `None` when the +/// FAILED write did not match (the record is no longer RUNNING and belongs to +/// whoever moved it) or errored; otherwise the removal's outcome. +async fn remove_dead_legacy_restore( + storage: &S, + job: &RestoreJobRow, +) -> Option { + let target = &job.target; + match storage.fail_restore_job(target, OWNER_DEAD).await { + Ok(true) => {} + Ok(false) => { + tracing::info!( + "restore into {} (account {}) was no longer RUNNING when the reconciler \ + reached it; left to its owner", + target.table_name, + target.account_id + ); + return None; + } + Err(e) => { + tracing::error!( + "could not mark the abandoned restore into {} (account {}) FAILED: {e}", + target.table_name, + target.account_id + ); + return None; + } + } + Some(remove_restore_target(storage, job, "was marked FAILED by the reconciler").await) +} + +/// What [`remove_restore_target`] found. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Removal { + /// The CREATING target was removed. + Removed, + /// The target was no longer a CREATING restore target with this id; it + /// belongs to whoever moved it and the record is left as it is. + LeftToOwner, + /// The removal errored; the FAILED record and the target wait for a + /// later cycle. + Failed, +} + +/// Remove the CREATING target of a record that is FAILED (`how` says, for +/// the log, whether this cycle wrote the FAILED or found it). A target that +/// is no longer a CREATING restore target belongs to whoever moved it; a +/// removal that errors leaves the FAILED record and the target for a later +/// cycle, which lists the pair again once the record is stale. +async fn remove_restore_target( + storage: &S, + job: &RestoreJobRow, + how: &str, +) -> Removal { + let target = &job.target; + match storage.remove_restore_target(target).await { + Ok(true) => { + tracing::warn!( + "removed abandoned restore target {} ({}) in account {}, restoring from {}: \ + its restore did not finish and no process owns it", + target.table_name, + target.table_id, + target.account_id, + job.source_backup_arn + ); + Removal::Removed + } + Ok(false) => { + tracing::warn!( + "the abandoned restore into {} (account {}) {how} but its target ({}) was no \ + longer a CREATING restore target; left to its current owner", + target.table_name, + target.account_id, + target.table_id + ); + Removal::LeftToOwner + } + Err(e) => { + tracing::error!( + "the abandoned restore into {} (account {}) {how} but its target ({}) could \ + not be removed yet; a later cycle retries once the FAILED record is older \ + than stale_job_after_secs: {e}", + target.table_name, + target.account_id, + target.table_id + ); + Removal::Failed + } + } +} diff --git a/crates/engine/src/backup/runtime.rs b/crates/engine/src/backup/runtime.rs new file mode 100644 index 000000000..c4f816bbb --- /dev/null +++ b/crates/engine/src/backup/runtime.rs @@ -0,0 +1,79 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! The per-process state of the backup operations: the job slots every +//! backup and restore job holds one of, and the store this instance is +//! configured to write. + +use std::sync::Arc; + +use extenddb_core::error::DynamoDbError; +use extenddb_storage::JobPermit; +use extenddb_storage::backup_catalog::{JobBound, StoreIdentity, limit_exceeded_message}; +use tokio::sync::Semaphore; + +/// The configuration key of the job bound the runtime enforces, as the +/// refusal names it. +const MAX_CONCURRENT_JOBS_KEY: &str = "[backup] max_concurrent_jobs"; + +/// One per server process, shared by every request through the operation +/// context. +#[derive(Debug)] +pub struct BackupRuntime { + slots: Arc, + max_concurrent_jobs: usize, + /// The store this instance writes new backups to and deletes from; + /// `None` while no store is configured, when every catalog row is a row + /// of another store. + configured_store: Option, +} + +impl BackupRuntime { + /// `max_concurrent_jobs` slots and no configured store. + #[must_use] + pub fn new(max_concurrent_jobs: usize) -> Self { + Self { + slots: Arc::new(Semaphore::new(max_concurrent_jobs)), + max_concurrent_jobs, + configured_store: None, + } + } + + /// The same runtime with `store` configured. + #[must_use] + pub fn with_store(mut self, store: StoreIdentity) -> Self { + self.configured_store = Some(store); + self + } + + /// The slots jobs take permits from. + #[must_use] + pub fn slots(&self) -> &Arc { + &self.slots + } + + /// Permits no job holds right now. + #[must_use] + pub fn free_slots(&self) -> usize { + self.slots.available_permits() + } + + /// Take a job slot without waiting, or `LimitExceededException` when every + /// slot is held: a queued job would hold a snapshot or a transaction + /// while it waited. The text is the one every refusal over a job bound + /// answers ([`limit_exceeded_message`]), naming this one. + pub(crate) fn take_permit(&self) -> Result { + JobPermit::try_acquire(&self.slots).ok_or_else(|| { + DynamoDbError::LimitExceededException(limit_exceeded_message(JobBound::Counted { + name: MAX_CONCURRENT_JOBS_KEY, + limit: self.max_concurrent_jobs, + })) + }) + } + + /// Whether `store` is the one this instance is configured with. With no + /// store configured, no row's store is. + pub(crate) fn is_configured_store(&self, store: &StoreIdentity) -> bool { + self.configured_store.as_ref() == Some(store) + } +} diff --git a/crates/engine/src/backup/tests.rs b/crates/engine/src/backup/tests.rs index f7bd86281..adb0fc616 100644 --- a/crates/engine/src/backup/tests.rs +++ b/crates/engine/src/backup/tests.rs @@ -1,24 +1,36 @@ // Copyright 2026 ExtendDB contributors // SPDX-License-Identifier: Apache-2.0 -//! Engine backup rules, against a `BackupEngine` double that records what the -//! engine asked of it. +//! Engine backup rules, against a double of the catalog index and the legacy +//! reader that records what the engine asked of it. use std::collections::HashMap; use std::sync::Mutex; +use std::time::Duration; +use extenddb_core::backup_format::BackupArn; use extenddb_core::error::DynamoDbError; use extenddb_core::limits::LimitsConfig; use extenddb_core::types::{ AttributeDefinition, BackupDescription, BackupDetails, BackupSummary, BillingMode, - BillingModeSummary, ContinuousBackupsDescription, GsiDescription, GsiInput, KeySchemaElement, - KeyType, LsiInput, OnDemandThroughput, Projection, ProjectionType, ProvisionedThroughput, - ProvisionedThroughputDescription, ScalarAttributeType, SourceTableDetails, StreamSpecification, - StreamViewType, TableDescription, TableStatus, TimeToLiveDescription, TimeToLiveStatus, + BillingModeSummary, ContinuousBackupsDescription, CreateTableInput, DeleteTableInput, + DescribeTableInput, GsiDescription, GsiInput, IndexInfo, KeySchemaElement, KeyType, + ListTablesInput, ListTablesOutput, LsiInput, OnDemandThroughput, Projection, ProjectionType, + ProvisionedThroughput, ProvisionedThroughputDescription, RestoreSummary, ScalarAttributeType, + SourceTableDetails, StreamSpecification, StreamViewType, TableDescription, TableKeyInfo, + TableStatus, TimeToLiveDescription, TimeToLiveStatus, UpdateTableInput, +}; +use extenddb_storage::backup_catalog::{ + AbandonedLegacyBackup, BackupKey, BackupRow, BackupStatus, DeleteStart, JobClaim, + RestoreJobRow, RestoreTargetKey, StoreIdentity, StoreKind, TimeBounds, }; use extenddb_storage::backup_definition::{BACKUP_DEFINITION_VERSION, BackupTableDefinition}; use extenddb_storage::error::StorageError; -use extenddb_storage::{BackupEngine, BackupRestoreSource, BoxedFuture, RestoreRequest}; +use extenddb_storage::{ + BackupCatalog, BackupRestoreSource, BoxedFuture, JobPermit, LegacyBackupReader, RestoreRequest, + TableEngine, +}; +use futures::stream::BoxStream; use serde_json::{Value, json}; use super::*; @@ -43,13 +55,50 @@ struct Fake { descriptions: Mutex>, summaries: Mutex>, restore_error: Mutex>, - /// Returned (once) by `backup_restore_source` instead of the stored source. + /// Returned (once) by `restore_source` instead of the stored source. source_error: Mutex>, restores: Mutex>, + /// The permits the legacy restore and create were handed, held as the + /// backend would hold them for the job's life. + permits: Mutex>, + creates: Mutex>, source_reads: Mutex, listed_for: Mutex>>, - /// When set, `backup_restore_source` waits for a permit before answering. + /// When set, `restore_source` waits for a permit before answering. source_gate: Option>, + /// The catalog index: rows by ARN. + rows: Mutex>, + /// `backup_in_use` answers: the CREATING target's name, by backup ARN. + in_use: Mutex>, + /// What `start_backup_delete` answers after the row check, by ARN. + delete_starts: Mutex>, + delete_starts_seen: Mutex>, + row_reads: Mutex>, + dead_jobs: Mutex>, + failed_jobs: Mutex>, + removed_targets: Mutex>, + /// Whether `remove_restore_target` answers `true` (the target was a + /// CREATING restore target with the key's id). + target_still_creating: Mutex, + /// Whether `fail_restore_job` answers `true` (the record was RUNNING); + /// separate from `target_still_creating` so the FAILED write matching + /// while the removal finds the target moved is a reachable state. + record_still_running: Mutex, + abandoned_backups: Mutex>, + deleted: Mutex>, + maintenance_held: Mutex, + /// How many times `check_maintenance_claim` ran: one per reconciler + /// cycle, so a test can see that a cycle ran. + maintenance_checks: Mutex, + /// When set, `check_maintenance_claim` fails with it (every call). + maintenance_error: Mutex>, + /// The `TimeBounds` `list_account_backup_rows` received. + bounds_seen: Mutex>, + /// The account's tables, by name, as `describe_table` reports them; a + /// name absent is `TableNotFound`. + tables: Mutex>, + /// The names `describe_table` was asked for. + table_reads: Mutex>, } impl Fake { @@ -73,17 +122,299 @@ fn unused() -> BoxedFuture<'static, Result> Box::pin(async { Err(StorageError::Internal("not used by these tests".to_owned())) }) } -impl BackupEngine for Fake { - fn create_backup( +/// The runtime the tests run with: four slots, no configured store. +fn runtime() -> BackupRuntime { + BackupRuntime::new(4) +} + +fn store() -> StoreIdentity { + StoreIdentity { + store_kind: StoreKind::Filesystem, + store_root: "/var/lib/extenddb/backups".to_owned(), + store_id: "store-1".to_owned(), + } +} + +/// A catalog row for `arn`, with the manifest fixture's source table (the +/// `Music` table of account 123456789012). +fn row(arn: &str, status: BackupStatus) -> BackupRow { + let parsed = BackupArn::parse(arn).unwrap(); + let manifest: Value = serde_json::from_str(include_str!( + "../../../core/testdata/extenddb-manifest/v1.json" + )) + .unwrap(); + BackupRow { + backup_id: parsed.backup_id().to_owned(), + account_id: parsed.account_id().to_owned(), + table_name: parsed.table_name().to_owned(), + backup_arn: arn.to_owned(), + backup_name: "music-nightly".to_owned(), + status, + store: store(), + size_bytes: 61828, + item_count: 100_000, + created_at: extenddb_storage::util::backup_creation_date_time(arn).unwrap(), + completed_at: None, + failure_reason: None, + source_table_details_json: manifest["source_table"].clone(), + owner_generation: 7, + heartbeat_at: 1.0, + } +} + +impl BackupCatalog for Fake { + fn catalog_id(&self) -> BoxedFuture<'_, Result> { + Box::pin(async { Ok("catalog-1".to_owned()) }) + } + + fn claim_backup<'a>( + &'a self, + _row: &'a BackupRow, + ) -> BoxedFuture<'a, Result, StorageError>> { + unused() + } + + fn list_backup_rows<'a>( + &'a self, + _status: Option, + _store: &'a StoreIdentity, + ) -> BoxStream<'a, Result> { + Box::pin(futures::stream::empty()) + } + + fn list_account_backup_rows<'a>( + &'a self, + account_id: &'a str, + table_name: Option<&'a str>, + bounds: &'a TimeBounds, + ) -> BoxStream<'a, Result> { + self.bounds_seen.lock().unwrap().push(*bounds); + let rows: Vec = self + .rows + .lock() + .unwrap() + .values() + .filter(|r| r.account_id == account_id) + .filter(|r| table_name.is_none_or(|t| r.table_name == t)) + .filter(|r| bounds.contains(r.created_at)) + .cloned() + .collect(); + Box::pin(futures::stream::iter(rows.into_iter().map(Ok))) + } + + /// The backend's rule: the row of (account, id), and only when its + /// account, table and ARN equal the request's. + fn read_backup_row<'a>( + &'a self, + arn: &'a BackupArn, + ) -> BoxedFuture<'a, Result, StorageError>> { + self.row_reads.lock().unwrap().push(arn.to_arn_string()); + let found = self + .rows + .lock() + .unwrap() + .values() + .find(|r| r.account_id == arn.account_id() && r.backup_id == arn.backup_id()) + .filter(|r| r.table_name == arn.table_name() && r.backup_arn == arn.to_arn_string()) + .cloned(); + Box::pin(async move { Ok(found) }) + } + + fn start_backup_delete<'a>( + &'a self, + key: &'a BackupKey, + ) -> BoxedFuture<'a, Result> { + self.delete_starts_seen.lock().unwrap().push(key.clone()); + let answer = self + .delete_starts + .lock() + .unwrap() + .get(&key.backup_arn) + .cloned(); + let status = self + .rows + .lock() + .unwrap() + .get(&key.backup_arn) + .map(|r| r.status); + Box::pin(async move { + match (answer, status) { + (Some(answer), _) => Ok(answer), + (None, Some(BackupStatus::Available)) => Ok(DeleteStart::Deleting), + (None, Some(status)) => Ok(DeleteStart::Status(status)), + (None, None) => Err(StorageError::Validation("Backup not found: x".to_owned())), + } + }) + } + + fn set_backup_deleted<'a>( + &'a self, + _key: &'a BackupKey, + ) -> BoxedFuture<'a, Result> { + unused() + } + + fn remove_backup_row<'a>( + &'a self, + _key: &'a BackupKey, + ) -> BoxedFuture<'a, Result> { + unused() + } + + fn backup_in_use<'a>( + &'a self, + key: &'a BackupKey, + ) -> BoxedFuture<'a, Result, StorageError>> { + let found = self.in_use.lock().unwrap().get(&key.backup_arn).cloned(); + Box::pin(async move { Ok(found) }) + } + + fn remove_restore_target<'a>( + &'a self, + key: &'a RestoreTargetKey, + ) -> BoxedFuture<'a, Result> { + self.removed_targets.lock().unwrap().push(key.clone()); + let answer = *self.target_still_creating.lock().unwrap(); + Box::pin(async move { Ok(answer) }) + } + + fn list_dead_restore_jobs( + &self, + _stale_after: Duration, + ) -> BoxedFuture<'_, Result, StorageError>> { + let jobs = self.dead_jobs.lock().unwrap().clone(); + Box::pin(async move { Ok(jobs) }) + } + + fn fail_restore_job<'a>( + &'a self, + key: &'a RestoreTargetKey, + reason: &'a str, + ) -> BoxedFuture<'a, Result> { + self.failed_jobs + .lock() + .unwrap() + .push((key.clone(), reason.to_owned())); + let answer = *self.record_still_running.lock().unwrap(); + Box::pin(async move { Ok(answer) }) + } + + fn check_maintenance_claim(&self) -> BoxedFuture<'_, Result> { + *self.maintenance_checks.lock().unwrap() += 1; + let held = *self.maintenance_held.lock().unwrap(); + let error = self.maintenance_error.lock().unwrap().clone(); + Box::pin(async move { + match error { + Some(e) => Err(StorageError::Internal(e)), + None => Ok(held), + } + }) + } +} + +impl TableEngine for Fake { + fn create_table( + &self, + _account_id: &str, + _input: CreateTableInput, + ) -> BoxedFuture<'_, Result> { + unused() + } + + fn delete_table( + &self, + _account_id: &str, + _input: DeleteTableInput, + ) -> BoxedFuture<'_, Result> { + unused() + } + + /// The account's table of that name, or `TableNotFound`. + fn describe_table( + &self, + _account_id: &str, + input: DescribeTableInput, + ) -> BoxedFuture<'_, Result> { + self.table_reads + .lock() + .unwrap() + .push(input.table_name.clone()); + let found = self.tables.lock().unwrap().get(&input.table_name).cloned(); + Box::pin(async move { + found.ok_or_else(|| { + StorageError::TableNotFound(format!("Table not found: {}", input.table_name)) + }) + }) + } + + fn list_tables( + &self, + _account_id: &str, + _input: ListTablesInput, + ) -> BoxedFuture<'_, Result> { + unused() + } + + fn update_table( + &self, + _account_id: &str, + _input: UpdateTableInput, + ) -> BoxedFuture<'_, Result> { + unused() + } + + fn table_key_info( + &self, + _account_id: &str, + _table_name: &str, + ) -> BoxedFuture<'_, Result> { + unused() + } + + fn index_info( + &self, + _account_id: &str, + _table_name: &str, + _index_name: &str, + ) -> BoxedFuture<'_, Result> { + unused() + } + + fn index_info_by_table_id( + &self, + _table_id: &str, + _index_name: &str, + ) -> BoxedFuture<'_, Result> { + unused() + } +} + +impl LegacyBackupReader for Fake { + fn create( &self, _: &str, - _: &str, - _: &str, + table_name: &str, + backup_name: &str, + permit: JobPermit, ) -> BoxedFuture<'_, Result> { - unused() + self.creates + .lock() + .unwrap() + .push((table_name.to_owned(), backup_name.to_owned())); + self.permits.lock().unwrap().push(permit); + Box::pin(async { + Ok(BackupDetails { + backup_arn: arn(ACCOUNT), + backup_name: "nightly".to_owned(), + backup_status: "AVAILABLE".to_owned(), + backup_type: "USER".to_owned(), + backup_size_bytes: 0, + backup_creation_date_time: CREATED, + }) + }) } - fn describe_backup( + fn describe( &self, _account_id: &str, backup_arn: &str, @@ -95,7 +426,7 @@ impl BackupEngine for Fake { }) } - fn list_backups( + fn list( &self, _account_id: &str, table_name: Option<&str>, @@ -111,11 +442,12 @@ impl BackupEngine for Fake { /// Removes the backup and answers its description as the backends do: /// the row's status (not yet `DELETED`), `SourceTableDetails` without /// `ProvisionedThroughput`, no feature details. - fn delete_backup( + fn delete( &self, _account_id: &str, backup_arn: &str, ) -> BoxedFuture<'_, Result> { + self.deleted.lock().unwrap().push(backup_arn.to_owned()); let found = self.descriptions.lock().unwrap().remove(backup_arn); self.sources.lock().unwrap().remove(backup_arn); let backup_arn = backup_arn.to_owned(); @@ -124,7 +456,7 @@ impl BackupEngine for Fake { }) } - fn backup_restore_source( + fn restore_source( &self, _account_id: &str, backup_arn: &str, @@ -145,14 +477,18 @@ impl BackupEngine for Fake { }) } - fn restore_table_from_backup( + fn restore( &self, _account_id: &str, request: RestoreRequest, + permit: JobPermit, ) -> BoxedFuture<'_, Result> { let err = self.restore_error.lock().unwrap().take(); let desc = described(&request); self.restores.lock().unwrap().push(request); + if err.is_none() { + self.permits.lock().unwrap().push(permit); + } Box::pin(async move { match err { Some(e) => Err(e), @@ -161,6 +497,14 @@ impl BackupEngine for Fake { }) } + fn abandoned_creating_backups( + &self, + _stale_after: Duration, + ) -> BoxedFuture<'_, Result, StorageError>> { + let found = self.abandoned_backups.lock().unwrap().clone(); + Box::pin(async move { Ok(found) }) + } + fn describe_continuous_backups( &self, _: &str, @@ -322,9 +666,16 @@ fn restore_body(extra: Value) -> Value { } async fn restore(fake: &Fake, body: Value) -> Result { - restore_table_from_backup(fake, ACCOUNT, REGION, &LimitsConfig::default(), &body) - .await - .map(|(_, v)| v) + restore_table_from_backup( + fake, + &runtime(), + ACCOUNT, + REGION, + &LimitsConfig::default(), + &body, + ) + .await + .map(|(_, v)| v) } fn validation_message(err: DynamoDbError) -> String { @@ -613,7 +964,7 @@ async fn a_backup_stored_under_another_region_resolves_as_written() { // DeleteBackup: the full description, and the stored row is gone. let fake = with_stored(); - let deleted = delete_backup(&fake, ACCOUNT, REGION, &spellings) + let deleted = delete_backup(&fake, &runtime(), ACCOUNT, REGION, &spellings) .await .unwrap(); assert_eq!(deleted.backup_details.backup_status, "DELETED"); @@ -637,7 +988,7 @@ async fn a_backup_stored_under_another_region_resolves_as_written() { .unwrap(); assert_eq!(fake.restores()[0].backup_arn, ours); assert_eq!(fake.source_reads(), 1); - let deleted = delete_backup(&fake, ACCOUNT, REGION, &spellings) + let deleted = delete_backup(&fake, &runtime(), ACCOUNT, REGION, &spellings) .await .unwrap(); assert_eq!(deleted.backup_details.backup_status, "DELETED"); @@ -664,7 +1015,7 @@ async fn a_backup_stored_under_another_region_resolves_as_written() { DynamoDbError::BackupNotFoundException(_) )); assert!(matches!( - delete_backup(&fake, ACCOUNT, REGION, &own(&body)) + delete_backup(&fake, &runtime(), ACCOUNT, REGION, &own(&body)) .await .unwrap_err(), DynamoDbError::BackupNotFoundException(_) @@ -760,6 +1111,7 @@ async fn a_restore_request_dropped_while_the_backup_is_read_still_restores() { let notify = Arc::clone(&after_ran); let mut request = Box::pin(restore_table_from_backup_detached( Arc::clone(&fake), + Arc::new(runtime()), Arc::from(ACCOUNT), Arc::from(REGION), Arc::new(LimitsConfig::default()), @@ -2497,7 +2849,7 @@ async fn delete_backup_answers_the_full_description_as_deleted() { let on_demand = def.billing_mode == "PAY_PER_REQUEST"; let fake = fake_with_backup("AVAILABLE", source(Some(def))); let described = describe(&fake).await; - let deleted = delete_backup(&fake, ACCOUNT, REGION, &own(&arn(ACCOUNT))) + let deleted = delete_backup(&fake, &runtime(), ACCOUNT, REGION, &own(&arn(ACCOUNT))) .await .unwrap(); let deleted = serde_json::to_value(deleted).unwrap(); @@ -2524,7 +2876,7 @@ async fn delete_backup_answers_the_full_description_as_deleted() { assert_eq!(deleted, expected); // The backup is gone. assert!(matches!( - delete_backup(&fake, ACCOUNT, REGION, &own(&arn(ACCOUNT))) + delete_backup(&fake, &runtime(), ACCOUNT, REGION, &own(&arn(ACCOUNT))) .await .unwrap_err(), DynamoDbError::BackupNotFoundException(_) @@ -2545,7 +2897,7 @@ async fn delete_backup_answers_the_full_description_as_deleted() { let what = format!("{err:?}"); let fake = fake_with_backup("AVAILABLE", source(Some(definition("PROVISIONED")))); *fake.source_error.lock().unwrap() = Some(err); - let deleted = delete_backup(&fake, ACCOUNT, REGION, &own(&arn(ACCOUNT))) + let deleted = delete_backup(&fake, &runtime(), ACCOUNT, REGION, &own(&arn(ACCOUNT))) .await .unwrap_or_else(|e| panic!("{what}: {e:?}")); assert_eq!(deleted.backup_details.backup_status, "DELETED", "{what}"); @@ -2566,7 +2918,7 @@ async fn delete_backup_answers_the_full_description_as_deleted() { let fake = fake_with_backup("AVAILABLE", source(Some(definition("PROVISIONED")))); *fake.source_error.lock().unwrap() = Some(StorageError::Connection("refused".to_owned())); assert!(matches!( - delete_backup(&fake, ACCOUNT, REGION, &own(&arn(ACCOUNT))) + delete_backup(&fake, &runtime(), ACCOUNT, REGION, &own(&arn(ACCOUNT))) .await .unwrap_err(), DynamoDbError::InternalServerError(_) @@ -3300,3 +3652,1418 @@ async fn list_backups_filters_on_the_time_range_before_paging() { )); } } + +// --- the catalog index --------------------------------------------------------- + +/// An ARN of the `Music` table in account `account` with backup id `id`. +fn music_arn(account: &str, id: &str) -> String { + format!("arn:aws:dynamodb:{REGION}:{account}:table/Music/backup/{id}") +} + +fn fake_with_row(status: BackupStatus) -> Fake { + let fake = Fake::default(); + fake.rows + .lock() + .unwrap() + .insert(arn(ACCOUNT), row(&arn(ACCOUNT), status)); + fake +} + +fn runtime_with_store() -> BackupRuntime { + runtime().with_store(store()) +} + +/// DescribeBackup answers from the catalog row when one exists, derived from +/// the recorded source table, and never reads the legacy reader for it. +#[tokio::test] +async fn describe_backup_reads_the_catalog_row_first() { + let fake = fake_with_row(BackupStatus::Available); + // A legacy description under the same ARN would answer differently. + fake.descriptions.lock().unwrap().insert( + arn(ACCOUNT), + backup_description("AVAILABLE", "PAY_PER_REQUEST"), + ); + let desc = describe_backup(&fake, ACCOUNT, REGION, &own(&arn(ACCOUNT))) + .await + .unwrap(); + assert_eq!(desc.backup_details.backup_name, "music-nightly"); + assert_eq!(desc.backup_details.backup_status, "AVAILABLE"); + assert_eq!(desc.backup_details.backup_size_bytes, 61828); + assert_eq!( + desc.source_table_details.table_id, + "12345a12-abcd-123a-ab12-1234abc12345" + ); + assert_eq!( + desc.source_table_details.billing_mode.as_deref(), + Some("PROVISIONED") + ); + let features = desc.source_table_feature_details.expect("features"); + assert_eq!( + features.global_secondary_indexes.unwrap()[0].index_name, + "GenreIndex" + ); + assert_eq!( + fake.source_reads(), + 0, + "the legacy reader was not consulted" + ); +} + +/// The status mapping of the catalog index: CREATING as itself, DELETING as +/// DELETED, DELETED and FAILED as `BackupNotFoundException`, with no fall +/// back to the legacy reader for a tombstone. +#[tokio::test] +async fn describe_backup_maps_the_row_status() { + let fake = fake_with_row(BackupStatus::Creating); + let desc = describe_backup(&fake, ACCOUNT, REGION, &own(&arn(ACCOUNT))) + .await + .unwrap(); + assert_eq!(desc.backup_details.backup_status, "CREATING"); + + let fake = fake_with_row(BackupStatus::Deleting); + let desc = describe_backup(&fake, ACCOUNT, REGION, &own(&arn(ACCOUNT))) + .await + .unwrap(); + assert_eq!(desc.backup_details.backup_status, "DELETED"); + + for status in [BackupStatus::Deleted, BackupStatus::Failed] { + let fake = fake_with_row(status); + fake.descriptions + .lock() + .unwrap() + .insert(arn(ACCOUNT), backup_description("AVAILABLE", "PROVISIONED")); + let err = describe_backup(&fake, ACCOUNT, REGION, &own(&arn(ACCOUNT))) + .await + .unwrap_err(); + assert!( + matches!(err, DynamoDbError::BackupNotFoundException(ref m) if m.contains(&arn(ACCOUNT))), + "{status}: {err:?}" + ); + assert_eq!(fake.source_reads(), 0, "{status}"); + } +} + +/// An ARN of account B that carries the id of account A's backup is read by +/// B's account and the id, matches no row, and is BackupNotFound: never A's +/// row. +#[tokio::test] +async fn describe_backup_of_another_accounts_id_is_not_found() { + const OTHER: &str = "999999999999"; + let fake = fake_with_row(BackupStatus::Available); + let other = own_for(&music_arn(OTHER, "01791976431438-73d8d5bc"), OTHER); + let err = describe_backup(&fake, OTHER, REGION, &other) + .await + .unwrap_err(); + assert!( + matches!(err, DynamoDbError::BackupNotFoundException(_)), + "{err:?}" + ); + // The lookup carried B's account, so B's own rows (none) were read. + assert_eq!( + fake.row_reads.lock().unwrap().as_slice(), + [music_arn(OTHER, "01791976431438-73d8d5bc")] + ); +} + +/// `own` for another account. +fn own_for(arn: &str, account: &str) -> BackupArnSpellings { + backup_arn_field(&json!({ "BackupArn": arn }), account, REGION).unwrap() +} + +/// A row stored under another region resolves under the ARN as written, +/// after the canonical spelling found nothing, as a legacy backup does. +#[tokio::test] +async fn describe_backup_tries_the_row_under_the_arn_as_written() { + let stored = + format!("arn:aws:dynamodb:eu-west-1:{ACCOUNT}:table/Music/backup/01791976431438-73d8d5bc"); + let fake = Fake::default(); + fake.rows + .lock() + .unwrap() + .insert(stored.clone(), row(&stored, BackupStatus::Available)); + let desc = describe_backup(&fake, ACCOUNT, REGION, &own(&stored)) + .await + .unwrap(); + assert_eq!(desc.backup_details.backup_arn, stored); + assert_eq!( + fake.row_reads.lock().unwrap().as_slice(), + [arn(ACCOUNT), stored.clone()], + "canonical first, then as written" + ); +} + +/// ListBackups merges the catalog's rows with the legacy reader's summaries +/// into one (creation time, ARN) order, lists CREATING and AVAILABLE rows +/// only, and pages the merged list with one cursor. +#[tokio::test] +async fn list_backups_merges_the_catalog_rows_with_the_legacy_summaries() { + let fake = fake_with_summaries(vec![summary(1, 1_000), summary(3, 3_000)]); + let ids = |n: u32, millis: u64| extenddb_storage::util::format_backup_id(millis, n); + let catalog = |n: u32, millis: u64, status: BackupStatus| { + let a = music_arn(ACCOUNT, &ids(n, millis)); + row(&a, status) + }; + { + let mut rows = fake.rows.lock().unwrap(); + for r in [ + catalog(2, 2_000, BackupStatus::Available), + catalog(4, 4_000, BackupStatus::Creating), + catalog(5, 5_000, BackupStatus::Deleting), + catalog(6, 6_000, BackupStatus::Deleted), + catalog(7, 7_000, BackupStatus::Failed), + ] { + rows.insert(r.backup_arn.clone(), r); + } + } + let page = list(&fake, json!({ "Limit": 3 })).await.unwrap(); + assert_eq!( + arns(&page), + [ + summary(1, 1_000).backup_arn, + music_arn(ACCOUNT, &ids(2, 2_000)), + summary(3, 3_000).backup_arn, + ] + ); + let cursor = page["LastEvaluatedBackupArn"].as_str().unwrap().to_owned(); + let page = list(&fake, json!({ "ExclusiveStartBackupArn": cursor })) + .await + .unwrap(); + assert_eq!(arns(&page), [music_arn(ACCOUNT, &ids(4, 4_000))]); + assert!(page.get("LastEvaluatedBackupArn").is_none()); + let creating = &page["BackupSummaries"][0]; + assert_eq!(creating["BackupStatus"], "CREATING"); + assert_eq!(creating["TableId"], "12345a12-abcd-123a-ab12-1234abc12345"); +} + +/// `TableName` may be the table's ARN: it lists what the name lists, and the +/// cursor's table check runs against the resolved name. +#[tokio::test] +async fn list_backups_accepts_a_table_arn_as_table_name() { + let fake = fake_with_summaries(vec![summary(1, 1_000)]); + let by_name = list(&fake, json!({ "TableName": "Music" })).await.unwrap(); + let by_arn = list(&fake, json!({ "TableName": source_arn() })) + .await + .unwrap(); + assert_eq!(arns(&by_name), arns(&by_arn)); + assert_eq!( + fake.listed_for.lock().unwrap().as_slice(), + [Some("Music".to_owned()), Some("Music".to_owned())] + ); + let cursor = summary(1, 1_000).backup_arn; + list( + &fake, + json!({ "TableName": source_arn(), "ExclusiveStartBackupArn": cursor }), + ) + .await + .unwrap(); + let other = format!("arn:aws:dynamodb:{REGION}:{ACCOUNT}:table/Other"); + let msg = validation_message( + list( + &fake, + json!({ "TableName": other, "ExclusiveStartBackupArn": summary(1, 1_000).backup_arn }), + ) + .await + .unwrap_err(), + ); + assert!(msg.contains("table name predicate"), "{msg}"); + let foreign = format!("arn:aws:dynamodb:{REGION}:999999999999:table/Music"); + assert!(matches!( + list(&fake, json!({ "TableName": foreign })) + .await + .unwrap_err(), + DynamoDbError::AccessDeniedException(_) + )); +} + +/// The time range is handed to the catalog as bounds and applied to the +/// legacy summaries as before, so both sides are filtered before paging. +#[tokio::test] +async fn list_backups_applies_the_time_range_to_both_sources() { + let fake = fake_with_summaries(vec![summary(1, 1_000), summary(3, 3_000)]); + let a2 = music_arn(ACCOUNT, &extenddb_storage::util::format_backup_id(2_000, 2)); + let a4 = music_arn(ACCOUNT, &extenddb_storage::util::format_backup_id(4_000, 4)); + { + let mut rows = fake.rows.lock().unwrap(); + rows.insert(a2.clone(), row(&a2, BackupStatus::Available)); + rows.insert(a4.clone(), row(&a4, BackupStatus::Available)); + } + let page = list( + &fake, + json!({ "TimeRangeLowerBound": 2.0, "TimeRangeUpperBound": 4.0 }), + ) + .await + .unwrap(); + assert_eq!(arns(&page), [a2, summary(3, 3_000).backup_arn]); + assert_eq!( + fake.bounds_seen.lock().unwrap().as_slice(), + [TimeBounds { + lower: Some(2.0), + upper: Some(4.0), + }], + "the bounds reached the catalog as bounds" + ); +} + +/// DeleteBackup over a catalog row, in the request's order: a tombstone is +/// BackupNotFound, a CREATING row BackupInUse, a row of another store a +/// ValidationException that names no root, and nothing reaches the row's +/// lock for any of them. +#[tokio::test] +async fn delete_backup_over_a_row_refuses_before_the_lock() { + for status in [BackupStatus::Deleted, BackupStatus::Failed] { + let fake = fake_with_row(status); + let err = delete_backup( + &fake, + &runtime_with_store(), + ACCOUNT, + REGION, + &own(&arn(ACCOUNT)), + ) + .await + .unwrap_err(); + assert!( + matches!(err, DynamoDbError::BackupNotFoundException(_)), + "{status}: {err:?}" + ); + assert!(fake.delete_starts_seen.lock().unwrap().is_empty()); + } + let fake = fake_with_row(BackupStatus::Creating); + let err = delete_backup( + &fake, + &runtime_with_store(), + ACCOUNT, + REGION, + &own(&arn(ACCOUNT)), + ) + .await + .unwrap_err(); + assert!( + matches!(err, DynamoDbError::BackupInUseException(_)), + "{err:?}" + ); + assert!(fake.delete_starts_seen.lock().unwrap().is_empty()); + + // No store configured: every row is a row of another store. + let fake = fake_with_row(BackupStatus::Available); + let msg = validation_message( + delete_backup(&fake, &runtime(), ACCOUNT, REGION, &own(&arn(ACCOUNT))) + .await + .unwrap_err(), + ); + assert!(msg.contains("store other than"), "{msg}"); + assert!( + !msg.contains(&store().store_root), + "no root is named: {msg}" + ); + assert!(fake.delete_starts_seen.lock().unwrap().is_empty()); + // The legacy reader was never asked. + assert!(fake.deleted.lock().unwrap().is_empty()); + assert_eq!(fake.source_reads(), 0); + + // The in-use check precedes the other-store check, as it does for a + // restore: a row of another store with a CREATING target registered + // from it (by an instance configured with that store) is BackupInUse + // naming the target, not the other-store refusal. + let fake = fake_with_row(BackupStatus::Available); + fake.in_use + .lock() + .unwrap() + .insert(arn(ACCOUNT), "Earlier".to_owned()); + let err = delete_backup(&fake, &runtime(), ACCOUNT, REGION, &own(&arn(ACCOUNT))) + .await + .unwrap_err(); + assert!( + matches!(err, DynamoDbError::BackupInUseException(ref m) if m.contains("Earlier")), + "{err:?}" + ); + assert!(fake.delete_starts_seen.lock().unwrap().is_empty()); +} + +/// DeleteBackup over an AVAILABLE row of the configured store: the in-use +/// check and the DELETING write run under the row's lock +/// (`start_backup_delete`), whose answer decides: a registration that won is +/// BackupInUse naming its target; a row that was CREATING under the lock is +/// BackupInUse; one that moved to a tombstone is BackupNotFound; DELETING +/// (new or resumed) answers the full description as DELETED. +#[tokio::test] +async fn delete_backup_over_a_row_runs_the_in_use_check_under_the_lock() { + let fake = fake_with_row(BackupStatus::Available); + let desc = delete_backup( + &fake, + &runtime_with_store(), + ACCOUNT, + REGION, + &own(&arn(ACCOUNT)), + ) + .await + .unwrap(); + assert_eq!(desc.backup_details.backup_status, "DELETED"); + assert_eq!(desc.backup_details.backup_name, "music-nightly"); + assert!(desc.source_table_details.provisioned_throughput.is_some()); + assert!(desc.source_table_feature_details.is_some()); + assert_eq!( + fake.delete_starts_seen.lock().unwrap().as_slice(), + [BackupKey::from_arn( + &BackupArn::parse(&arn(ACCOUNT)).unwrap() + )] + ); + + let fake = fake_with_row(BackupStatus::Deleting); + let desc = delete_backup( + &fake, + &runtime_with_store(), + ACCOUNT, + REGION, + &own(&arn(ACCOUNT)), + ) + .await + .unwrap(); + assert_eq!( + desc.backup_details.backup_status, "DELETED", + "a retry resumes" + ); + + for (answer, want) in [ + ( + DeleteStart::InUse("MusicRestored".to_owned()), + "BackupInUseException", + ), + ( + DeleteStart::Status(BackupStatus::Creating), + "BackupInUseException", + ), + ( + DeleteStart::Status(BackupStatus::Deleted), + "BackupNotFoundException", + ), + ( + DeleteStart::Status(BackupStatus::Failed), + "BackupNotFoundException", + ), + ] { + let fake = fake_with_row(BackupStatus::Available); + fake.delete_starts + .lock() + .unwrap() + .insert(arn(ACCOUNT), answer.clone()); + let err = delete_backup( + &fake, + &runtime_with_store(), + ACCOUNT, + REGION, + &own(&arn(ACCOUNT)), + ) + .await + .unwrap_err(); + let got = err.full_error_type(); + assert!(got.ends_with(want), "{answer:?}: {got}"); + if let DeleteStart::InUse(table) = &answer { + let DynamoDbError::BackupInUseException(m) = err else { + unreachable!() + }; + assert!(m.contains(table), "{m}"); + } + } + + // `Status(AVAILABLE)` is an answer the contract excludes (an AVAILABLE + // row becomes `Deleting`): a backend that gives it has broken the + // contract, which is an internal error, not an absent backup. + let fake = fake_with_row(BackupStatus::Available); + fake.delete_starts + .lock() + .unwrap() + .insert(arn(ACCOUNT), DeleteStart::Status(BackupStatus::Available)); + let err = delete_backup( + &fake, + &runtime_with_store(), + ACCOUNT, + REGION, + &own(&arn(ACCOUNT)), + ) + .await + .unwrap_err(); + assert!( + matches!(err, DynamoDbError::InternalServerError(_)), + "{err:?}" + ); +} + +/// A row whose `source_table_details_json` this build cannot describe: the +/// listing leaves it out and lists the rest (not an internal error for the +/// whole account), and DeleteBackup answers the internal error before the +/// DELETING write, so the row stays AVAILABLE and a retry sees the same. +#[tokio::test] +async fn an_unreadable_source_table_is_left_out_of_the_listing_and_stops_a_delete_before_the_write() +{ + let readable = music_arn(ACCOUNT, &extenddb_storage::util::format_backup_id(1_000, 1)); + let unreadable = music_arn(ACCOUNT, &extenddb_storage::util::format_backup_id(2_000, 2)); + let fake = Fake::default(); + { + let mut rows = fake.rows.lock().unwrap(); + rows.insert(readable.clone(), row(&readable, BackupStatus::Available)); + let mut damaged = row(&unreadable, BackupStatus::Available); + damaged.source_table_details_json = json!({ "table_name": "Music" }); + rows.insert(unreadable.clone(), damaged); + } + let page = list(&fake, json!({})).await.unwrap(); + assert_eq!(arns(&page), std::slice::from_ref(&readable)); + + let err = delete_backup( + &fake, + &runtime_with_store(), + ACCOUNT, + REGION, + &own(&unreadable), + ) + .await + .unwrap_err(); + assert!( + matches!(err, DynamoDbError::InternalServerError(_)), + "{err:?}" + ); + assert!( + fake.delete_starts_seen.lock().unwrap().is_empty(), + "the description is built before the DELETING write" + ); + assert_eq!( + fake.rows.lock().unwrap()[&unreadable].status, + BackupStatus::Available + ); + // A row the summary can still read (its `table_id` and `table_arn` are + // there, the rest is not) lists; its description is the internal error. + fake.rows + .lock() + .unwrap() + .get_mut(&unreadable) + .unwrap() + .source_table_details_json = json!({ + "table_id": "tid-2", + "table_arn": source_arn(), + }); + let page = list(&fake, json!({})).await.unwrap(); + assert_eq!(arns(&page), [readable, unreadable.clone()]); + assert_eq!(page["BackupSummaries"][1]["TableId"], "tid-2"); + let err = describe_backup(&fake, ACCOUNT, REGION, &own(&unreadable)) + .await + .unwrap_err(); + assert!( + matches!(err, DynamoDbError::InternalServerError(_)), + "{err:?}" + ); +} + +/// A legacy backup of a table whose name the store-prefix rules refuse (a +/// name made only of dots, which main's CreateTable and CreateBackup accept; +/// a name over 255 bytes under a raised `limits.max_table_name_length`) has +/// no parsed ARN and no catalog-index read, and describes, restores and +/// deletes through the legacy reader as on main. +#[tokio::test] +async fn a_backup_of_a_name_the_store_prefix_refuses_resolves_through_the_legacy_reader() { + for table in ["...", &"x".repeat(256)] { + let id = "01791976431438-73d8d5bc"; + let legacy_arn = format!("arn:aws:dynamodb:{REGION}:{ACCOUNT}:table/{table}/backup/{id}"); + assert!( + BackupArn::parse(&legacy_arn).is_err(), + "the store-prefix rules refuse {table:?}" + ); + let spellings = own(&legacy_arn); + assert_eq!(spellings.canonical(), legacy_arn); + assert_eq!(spellings.arns().count(), 0, "no parsed spelling"); + + let mut src = source(Some(definition("PROVISIONED"))); + src.backup_arn = legacy_arn.clone(); + src.source_table_name = table.to_owned(); + let fake = Fake::with_source(src); + let mut description = backup_description("AVAILABLE", "PROVISIONED"); + description.backup_details.backup_arn = legacy_arn.clone(); + description.source_table_details.table_name = table.to_owned(); + fake.descriptions + .lock() + .unwrap() + .insert(legacy_arn.clone(), description); + + let desc = describe_backup(&fake, ACCOUNT, REGION, &spellings) + .await + .unwrap(); + assert_eq!(desc.backup_details.backup_arn, legacy_arn); + assert_eq!(desc.source_table_details.table_name, table); + assert!(desc.source_table_feature_details.is_some()); + + let limits = LimitsConfig { + max_table_name_length: 1024, + ..LimitsConfig::default() + }; + let (_, response) = restore_table_from_backup( + &fake, + &runtime(), + ACCOUNT, + REGION, + &limits, + &json!({ "BackupArn": legacy_arn, "TargetTableName": "Restored" }), + ) + .await + .unwrap(); + assert_eq!(response["TableDescription"]["TableName"], "Restored"); + assert_eq!(fake.restores().len(), 1); + assert_eq!(fake.restores()[0].backup_arn, legacy_arn); + + let desc = delete_backup(&fake, &runtime(), ACCOUNT, REGION, &spellings) + .await + .unwrap(); + assert_eq!(desc.backup_details.backup_status, "DELETED"); + assert_eq!( + fake.deleted.lock().unwrap().as_slice(), + std::slice::from_ref(&legacy_arn) + ); + assert!( + fake.row_reads.lock().unwrap().is_empty(), + "the catalog index was not read for {table:?}" + ); + } +} + +/// RestoreTableFromBackup over a catalog row, before the permit and reading +/// no store: CREATING is BackupInUse; DELETING, DELETED and FAILED are +/// BackupNotFound; a backup in use is BackupInUse naming the target, whether +/// or not the row is of the configured store (the in-use check runs before +/// the other-store check, as the delete's does); a row +/// of another store is a ValidationException, with no store configured and +/// with a different one configured; and an AVAILABLE row of the +/// configured store is refused by this release. The runtime has no free +/// slot, so a permit taken anywhere before the row's refusal, even one +/// dropped again, would turn the refusal into LimitExceeded; the legacy +/// reader is never reached. +#[tokio::test] +async fn restore_over_a_row_refuses_before_the_permit() { + async fn check(fake: Fake, runtime: &BackupRuntime) -> DynamoDbError { + assert_eq!(runtime.free_slots(), 0, "every slot is held"); + let err = restore_table_from_backup( + &fake, + runtime, + ACCOUNT, + REGION, + &LimitsConfig::default(), + &restore_body(json!({})), + ) + .await + .unwrap_err(); + assert!( + !matches!(err, DynamoDbError::LimitExceededException(_)), + "the row's refusal, not the permit's: {err:?}" + ); + assert_eq!(fake.source_reads(), 0, "the legacy reader was not reached"); + assert!(fake.restores().is_empty()); + err + } + let configured = BackupRuntime::new(1).with_store(store()); + let _held_configured = JobPermit::try_acquire(configured.slots()).unwrap(); + let none = BackupRuntime::new(1); + let _held_none = JobPermit::try_acquire(none.slots()).unwrap(); + let err = check(fake_with_row(BackupStatus::Creating), &configured).await; + assert!( + matches!(err, DynamoDbError::BackupInUseException(_)), + "{err:?}" + ); + for status in [ + BackupStatus::Deleting, + BackupStatus::Deleted, + BackupStatus::Failed, + ] { + let err = check(fake_with_row(status), &configured).await; + assert!( + matches!(err, DynamoDbError::BackupNotFoundException(_)), + "{status}: {err:?}" + ); + } + let fake = fake_with_row(BackupStatus::Available); + fake.in_use + .lock() + .unwrap() + .insert(arn(ACCOUNT), "Earlier".to_owned()); + let err = check(fake, &configured).await; + assert!( + matches!(err, DynamoDbError::BackupInUseException(ref m) if m.contains("Earlier")), + "{err:?}" + ); + // The in-use check runs before the other-store check, as the delete's + // does: a row of another store that is in use is BackupInUse, not the + // other-store ValidationException. + let fake = fake_with_row(BackupStatus::Available); + fake.in_use + .lock() + .unwrap() + .insert(arn(ACCOUNT), "Earlier".to_owned()); + let err = check(fake, &none).await; + assert!( + matches!(err, DynamoDbError::BackupInUseException(ref m) if m.contains("Earlier")), + "the in-use check runs before the other-store check: {err:?}" + ); + let msg = validation_message(check(fake_with_row(BackupStatus::Available), &configured).await); + assert!(msg.contains("in-catalog backups only"), "{msg}"); + let msg = validation_message(check(fake_with_row(BackupStatus::Available), &none).await); + assert!(msg.contains("store other than"), "{msg}"); + // A configured store that is not the row's: the row is of another store, + // not of "the" store because one is configured. + let other = BackupRuntime::new(1).with_store(StoreIdentity { + store_id: "store-2".to_owned(), + ..store() + }); + let _held_other = JobPermit::try_acquire(other.slots()).unwrap(); + let msg = validation_message(check(fake_with_row(BackupStatus::Available), &other).await); + assert!( + msg.contains("store other than"), + "a configured store other than the row's: {msg}" + ); +} + +/// The job permit is the last check of a restore: a request refused earlier +/// takes no slot, a request that reaches the backend hands its slot to the +/// backend for the job's life, and with no slot free the request is +/// LimitExceeded after every other check passed, with the backend untouched. +#[tokio::test] +async fn restore_takes_the_permit_last() { + async fn restore_with( + fake: &Fake, + runtime: &BackupRuntime, + ) -> Result<(String, Value), DynamoDbError> { + let body = restore_body(json!({})); + restore_table_from_backup( + fake, + runtime, + ACCOUNT, + REGION, + &LimitsConfig::default(), + &body, + ) + .await + } + let runtime = BackupRuntime::new(1); + let fake = Fake::with_source(source(Some(definition("PROVISIONED")))); + restore_with(&fake, &runtime).await.unwrap(); + assert_eq!(runtime.free_slots(), 0, "the backend holds the slot"); + assert_eq!(fake.restores().len(), 1); + + let err = restore_with(&fake, &runtime).await.unwrap_err(); + assert!( + matches!( + err, + DynamoDbError::LimitExceededException(ref m) + if m.contains("(at most 1 at once, the [backup] max_concurrent_jobs bound)") + ), + "the refusal names the bound: {err:?}" + ); + assert_eq!(fake.restores().len(), 1, "the backend was not asked"); + assert_eq!( + fake.source_reads(), + 2, + "the source was read before the permit" + ); + + // A refusal earlier in the order takes no slot: a missing backup. + let runtime = BackupRuntime::new(1); + let empty = Fake::default(); + let err = restore_with(&empty, &runtime).await.unwrap_err(); + assert!( + matches!(err, DynamoDbError::BackupNotFoundException(_)), + "{err:?}" + ); + assert_eq!(runtime.free_slots(), 1); + + // The backend's refusal returns the slot with the dropped permit. + let fake = Fake::with_source(source(Some(definition("PROVISIONED")))); + *fake.restore_error.lock().unwrap() = + Some(StorageError::TableAlreadyExists("MusicRestored".to_owned())); + let err = restore_with(&fake, &runtime).await.unwrap_err(); + assert!( + matches!(err, DynamoDbError::TableAlreadyExistsException(_)), + "{err:?}" + ); + assert_eq!(runtime.free_slots(), 1); + + // Dropping the backend's permit frees the slot. + fake.permits.lock().unwrap().clear(); + assert_eq!(runtime.free_slots(), 1); +} + +/// The target name is checked before the backup and before the permit: with +/// every permit held and the backup restorable, a restore into a name a table +/// has is TableAlreadyExists, a retry into a CREATING restore target is +/// TableInUse naming the target's source backup in the backend's own text, a +/// CREATING table with no RestoreSummary (an ordinary create) and an ACTIVE +/// former target (its summary no longer in progress) are TableAlreadyExists; +/// the backup is not read and the backend not asked. A free name passes. +#[tokio::test] +async fn restore_checks_the_target_name_before_the_backup_and_the_permit() { + fn table(status: TableStatus, source_backup_arn: Option<&str>) -> TableDescription { + TableDescription { + table_name: "MusicRestored".to_owned(), + table_status: status, + restore_summary: source_backup_arn.map(|arn| RestoreSummary { + source_backup_arn: Some(arn.to_owned()), + source_table_arn: Some(source_arn()), + restore_date_time: CREATED, + restore_in_progress: status == TableStatus::Creating, + }), + ..Default::default() + } + } + async fn check(target: TableDescription) -> DynamoDbError { + let full = BackupRuntime::new(1); + let _held = full.take_permit().unwrap(); + let fake = Fake::with_source(source(Some(definition("PROVISIONED")))); + fake.tables + .lock() + .unwrap() + .insert("MusicRestored".to_owned(), target); + let err = restore_table_from_backup( + &fake, + &full, + ACCOUNT, + REGION, + &LimitsConfig::default(), + &restore_body(json!({})), + ) + .await + .unwrap_err(); + assert!( + !matches!(err, DynamoDbError::LimitExceededException(_)), + "the name's refusal, not the permit's: {err:?}" + ); + assert_eq!( + fake.table_reads.lock().unwrap().as_slice(), + ["MusicRestored"], + "the name was read once, under the request's spelling" + ); + assert_eq!(fake.source_reads(), 0, "the backup was not read"); + assert!(fake.row_reads.lock().unwrap().is_empty()); + assert!(fake.restores().is_empty(), "the backend was not asked"); + err + } + + let err = check(table(TableStatus::Active, None)).await; + assert!( + matches!(err, DynamoDbError::TableAlreadyExistsException(ref m) if m.contains("MusicRestored")), + "{err:?}" + ); + let err = check(table(TableStatus::Creating, Some(&arn(ACCOUNT)))).await; + let text = format!( + "Table: MusicRestored is already being restored from backup: {}", + arn(ACCOUNT) + ); + assert!( + matches!(err, DynamoDbError::TableInUseException(ref m) if *m == text), + "{err:?}" + ); + let err = check(table(TableStatus::Creating, None)).await; + assert!( + matches!(err, DynamoDbError::TableAlreadyExistsException(_)), + "an ordinary CREATING table: {err:?}" + ); + let err = check(table(TableStatus::Active, Some(&arn(ACCOUNT)))).await; + assert!( + matches!(err, DynamoDbError::TableAlreadyExistsException(_)), + "an ACTIVE former target: {err:?}" + ); + + // A free name: the check passes and the restore reaches the backend. + let runtime = BackupRuntime::new(1); + let fake = Fake::with_source(source(Some(definition("PROVISIONED")))); + restore_table_from_backup( + &fake, + &runtime, + ACCOUNT, + REGION, + &LimitsConfig::default(), + &restore_body(json!({})), + ) + .await + .unwrap(); + assert_eq!( + fake.table_reads.lock().unwrap().as_slice(), + ["MusicRestored"] + ); + assert_eq!(fake.restores().len(), 1); +} + +/// CreateBackup's checks run in the service's order, every one before the +/// permit: a missing or not-ACTIVE table is TableNotFound whether or not a +/// slot is free; a table with vector indexes is refused; then the permit, +/// LimitExceeded when none is free; then the in-catalog create holds the slot. +#[tokio::test] +async fn create_backup_checks_the_table_then_takes_the_permit_last() { + let active = TableDescription { + table_name: "Music".to_owned(), + table_status: TableStatus::Active, + ..Default::default() + }; + let full = BackupRuntime::new(1); + let _held = full.take_permit().unwrap(); + let fake = Fake::default(); + + let err = create_backup( + &fake, + &full, + ACCOUNT, + REGION, + "Music", + "nightly", + Err(StorageError::TableNotFound("Music".to_owned())), + ) + .await + .unwrap_err(); + assert!( + matches!(err, DynamoDbError::TableNotFoundException(ref m) if m == "Table not found: Music"), + "{err:?}" + ); + let creating = TableDescription { + table_status: TableStatus::Creating, + ..active.clone() + }; + let err = create_backup( + &fake, + &full, + ACCOUNT, + REGION, + "Music", + "nightly", + Ok(creating), + ) + .await + .unwrap_err(); + assert!( + matches!(err, DynamoDbError::TableNotFoundException(_)), + "{err:?}" + ); + + let with_vectors: TableDescription = serde_json::from_value(json!({ + "TableName": "Music", + "KeySchema": [], + "AttributeDefinitions": [], + "TableStatus": "ACTIVE", + "CreationDateTime": 0.0, + "TableSizeBytes": 0, + "ItemCount": 0, + "TableArn": "", + "TableId": "", + "ProvisionedThroughput": { "ReadCapacityUnits": 0, "WriteCapacityUnits": 0, + "NumberOfDecreasesToday": 0 }, + "DeletionProtectionEnabled": false, + "VectorIndexes": [{ + "IndexName": "v", + "VectorAttribute": { "AttributeName": "emb" }, + "Dimensions": 3, + "DistanceFunction": "COSINE", + "IndexStatus": "ACTIVE", + "IndexSizeBytes": 0, + "ItemCount": 0, + "IndexArn": "" + }] + })) + .unwrap(); + let msg = validation_message( + create_backup( + &fake, + &full, + ACCOUNT, + REGION, + "Music", + "nightly", + Ok(with_vectors), + ) + .await + .unwrap_err(), + ); + assert!(msg.contains("vector"), "{msg}"); + + let err = create_backup( + &fake, + &full, + ACCOUNT, + REGION, + "Music", + "nightly", + Ok(active.clone()), + ) + .await + .unwrap_err(); + assert!( + matches!(err, DynamoDbError::LimitExceededException(_)), + "{err:?}" + ); + assert!( + fake.creates.lock().unwrap().is_empty(), + "the backend was not asked" + ); + + let free = BackupRuntime::new(1); + create_backup( + &fake, + &free, + ACCOUNT, + REGION, + "Music", + "nightly", + Ok(active), + ) + .await + .unwrap(); + assert_eq!( + fake.creates.lock().unwrap().as_slice(), + [("Music".to_owned(), "nightly".to_owned())] + ); + assert_eq!(free.free_slots(), 0, "the backend holds the slot"); + fake.permits.lock().unwrap().clear(); + assert_eq!(free.free_slots(), 1); +} + +/// A table name that fits CreateTable but not a backup ARN's store prefix +/// is refused before the permit, naming the limit: the runtime's one permit +/// is held, so a check placed after the permit would answer LimitExceeded. +#[tokio::test] +async fn create_backup_refuses_a_name_no_backup_arn_can_carry() { + let runtime = BackupRuntime::new(1); + let _held = runtime.take_permit().unwrap(); + let fake = Fake::default(); + // 256 bytes, and 1024 bytes: a name the handler admits under a raised + // `limits.max_table_name_length` that takes the ARN itself over its + // 1024-byte bound, so the limit must be named before the ARN is formed. + for len in [256, 1024] { + let long = "x".repeat(len); + let desc = TableDescription { + table_name: long.clone(), + table_status: TableStatus::Active, + ..Default::default() + }; + let msg = validation_message( + create_backup(&fake, &runtime, ACCOUNT, REGION, &long, "nightly", Ok(desc)) + .await + .unwrap_err(), + ); + assert!( + msg.contains(&format!("{len} bytes")) && msg.contains("255"), + "{len}: {msg}" + ); + assert!(!msg.contains("not a backup ARN"), "{len}: {msg}"); + } + let dots = "..."; + let desc = TableDescription { + table_name: dots.to_owned(), + table_status: TableStatus::Active, + ..Default::default() + }; + let msg = validation_message( + create_backup(&fake, &runtime, ACCOUNT, REGION, dots, "nightly", Ok(desc)) + .await + .unwrap_err(), + ); + assert!( + msg.contains("TableName") && msg.contains("only dots"), + "{msg}" + ); + assert_eq!(runtime.free_slots(), 0, "the test still holds the permit"); + assert!(fake.creates.lock().unwrap().is_empty()); +} + +/// Under a `server.region` outside `[a-z0-9-]` no backup ARN can be formed +/// for any table, so CreateBackup is refused naming `server.region`, not the +/// table name, before the permit (the runtime's one permit is held). The +/// other operations are not affected for a backup created under a valid +/// region: its stored ARN is reached as written, the canonical spelling in +/// the server's region having no parsed form. A backup an earlier release +/// created under such a region keeps an ARN `check_backup_arn` refuses as +/// not valid, before any lookup. +#[tokio::test] +async fn create_backup_under_a_region_outside_the_arn_alphabet_names_the_setting() { + let runtime = BackupRuntime::new(1); + let _held = runtime.take_permit().unwrap(); + let fake = Fake::default(); + let desc = TableDescription { + table_name: "Music".to_owned(), + table_status: TableStatus::Active, + ..Default::default() + }; + let msg = validation_message( + create_backup( + &fake, + &runtime, + ACCOUNT, + "LOCAL", + "Music", + "nightly", + Ok(desc), + ) + .await + .unwrap_err(), + ); + assert!( + msg.contains("server.region") && msg.contains("\"LOCAL\"") && msg.contains("[a-z0-9-]"), + "{msg}" + ); + assert!( + !msg.contains("TableName"), + "the table is not the cause: {msg}" + ); + assert_eq!(runtime.free_slots(), 0, "the test still holds the permit"); + assert!(fake.creates.lock().unwrap().is_empty()); + + // A backup created under a valid region, described under "LOCAL": the + // row is read under the ARN as written, and only that spelling. + let stored = arn(ACCOUNT); + fake.rows + .lock() + .unwrap() + .insert(stored.clone(), row(&stored, BackupStatus::Available)); + let spellings = backup_arn_field(&json!({ "BackupArn": stored }), ACCOUNT, "LOCAL").unwrap(); + let desc = describe_backup(&fake, ACCOUNT, "LOCAL", &spellings) + .await + .unwrap(); + assert_eq!(desc.backup_details.backup_arn, stored); + assert_eq!( + fake.row_reads.lock().unwrap().as_slice(), + std::slice::from_ref(&stored), + "as written only: the canonical spelling under \"LOCAL\" has no parsed form" + ); + + // A backup created under "LOCAL" by an earlier release: its ARN is + // refused before any lookup. + let local = + format!("arn:aws:dynamodb:LOCAL:{ACCOUNT}:table/Music/backup/01791976431438-73d8d5bc"); + let msg = validation_message( + backup_arn_field(&json!({ "BackupArn": local }), ACCOUNT, "LOCAL").unwrap_err(), + ); + assert_eq!(msg, BACKUP_ARN_NOT_VALID); + assert_eq!(fake.row_reads.lock().unwrap().len(), 1, "no further read"); +} + +// --- the reconciler ------------------------------------------------------------- + +fn dead_job(kind: extenddb_storage::backup_catalog::RestoreKind, name: &str) -> RestoreJobRow { + RestoreJobRow { + target: RestoreTargetKey { + account_id: ACCOUNT.to_owned(), + table_name: name.to_owned(), + table_id: format!("tid-{name}"), + }, + source_backup_arn: arn(ACCOUNT), + kind, + status: extenddb_storage::backup_catalog::JobStatus::Running, + owner_generation: 0, + heartbeat_at: 0.0, + } +} + +/// A legacy restore whose owner is dead is marked FAILED and its target +/// removed; a manifest restore is left for its resumer; an abandoned +/// in-catalog CREATING backup is removed through the reader's delete. +#[tokio::test] +async fn the_reconciler_fails_dead_legacy_restores_and_removes_abandoned_backups() { + use extenddb_storage::backup_catalog::RestoreKind; + let fake = Fake::default(); + *fake.target_still_creating.lock().unwrap() = true; + *fake.record_still_running.lock().unwrap() = true; + fake.dead_jobs.lock().unwrap().extend([ + dead_job(RestoreKind::Legacy, "LegacyTarget"), + dead_job(RestoreKind::Manifest, "ManifestTarget"), + ]); + fake.abandoned_backups + .lock() + .unwrap() + .push(AbandonedLegacyBackup { + account_id: ACCOUNT.to_owned(), + backup_arn: arn(ACCOUNT), + }); + fake.descriptions + .lock() + .unwrap() + .insert(arn(ACCOUNT), backup_description("CREATING", "PROVISIONED")); + + let report = reconciler::cycle( + &fake, + Duration::from_secs(300), + &mut reconciler::ReconcilerState::default(), + ) + .await + .unwrap(); + assert!(!report.maintenance_held); + assert_eq!(report.legacy_restores_removed, ["LegacyTarget"]); + assert!(report.failed_targets_removed.is_empty()); + assert_eq!(report.manifest_restores_seen, ["ManifestTarget"]); + assert_eq!(report.manifest_restores_logged, ["ManifestTarget"]); + assert_eq!(report.legacy_backups_removed, [arn(ACCOUNT)]); + + let failed = fake.failed_jobs.lock().unwrap().clone(); + assert_eq!(failed.len(), 1); + assert_eq!(failed[0].0.table_name, "LegacyTarget"); + assert_eq!(failed[0].1, reconciler::OWNER_DEAD); + assert_eq!( + fake.removed_targets.lock().unwrap().as_slice(), + [dead_job(RestoreKind::Legacy, "LegacyTarget").target] + ); + assert_eq!(fake.deleted.lock().unwrap().as_slice(), [arn(ACCOUNT)]); +} + +/// A record that is no longer RUNNING when the reconciler reaches it belongs +/// to whoever moved it: the target is not removed. +#[tokio::test] +async fn the_reconciler_leaves_a_target_whose_record_moved() { + use extenddb_storage::backup_catalog::RestoreKind; + let fake = Fake::default(); + *fake.record_still_running.lock().unwrap() = false; + *fake.target_still_creating.lock().unwrap() = true; + fake.dead_jobs + .lock() + .unwrap() + .push(dead_job(RestoreKind::Legacy, "LegacyTarget")); + let report = reconciler::cycle( + &fake, + Duration::from_secs(300), + &mut reconciler::ReconcilerState::default(), + ) + .await + .unwrap(); + assert!(report.legacy_restores_removed.is_empty()); + assert!(report.targets_left_to_owner.is_empty()); + assert_eq!(fake.failed_jobs.lock().unwrap().len(), 1); + assert!(fake.removed_targets.lock().unwrap().is_empty()); +} + +/// A removal that finds the target no longer a CREATING restore target +/// (`remove_restore_target` answered `false`: the target flipped ACTIVE, was +/// deleted, or was re-created under another id) leaves it to its current +/// owner: the target is not counted as removed, the cycle reports it as left, +/// and the record is not written again, whether this cycle wrote the FAILED +/// (a dead legacy restore) or found it (the FAILED-target duty). +#[tokio::test] +async fn the_reconciler_leaves_a_target_that_is_no_longer_creating_after_the_failed_write() { + use extenddb_storage::backup_catalog::{JobStatus, RestoreKind}; + let fake = Fake::default(); + *fake.record_still_running.lock().unwrap() = true; + *fake.target_still_creating.lock().unwrap() = false; + let dead = dead_job(RestoreKind::Legacy, "LegacyTarget"); + let mut failed = dead_job(RestoreKind::Legacy, "FailedTarget"); + failed.status = JobStatus::Failed; + fake.dead_jobs + .lock() + .unwrap() + .extend([dead.clone(), failed.clone()]); + let report = reconciler::cycle( + &fake, + Duration::from_secs(300), + &mut reconciler::ReconcilerState::default(), + ) + .await + .unwrap(); + assert!( + report.legacy_restores_removed.is_empty() && report.failed_targets_removed.is_empty(), + "a target the removal found moved is not counted as removed: {report:?}" + ); + assert_eq!( + report.targets_left_to_owner, + ["LegacyTarget", "FailedTarget"] + ); + assert_eq!( + fake.removed_targets.lock().unwrap().as_slice(), + [dead.target.clone(), failed.target] + ); + assert_eq!( + fake.failed_jobs.lock().unwrap().as_slice(), + [(dead.target, reconciler::OWNER_DEAD.to_owned())], + "the dead record is written FAILED once; the FAILED record is not written again" + ); +} + +/// The maintenance claim is checked first and reported; the duties that +/// resolve dead owners still run while it is held. +#[tokio::test] +async fn the_reconciler_reports_a_held_maintenance_claim_and_still_resolves_dead_owners() { + use extenddb_storage::backup_catalog::RestoreKind; + let fake = Fake::default(); + *fake.maintenance_held.lock().unwrap() = true; + *fake.target_still_creating.lock().unwrap() = true; + *fake.record_still_running.lock().unwrap() = true; + fake.dead_jobs + .lock() + .unwrap() + .push(dead_job(RestoreKind::Legacy, "LegacyTarget")); + let report = reconciler::cycle( + &fake, + Duration::from_secs(300), + &mut reconciler::ReconcilerState::default(), + ) + .await + .unwrap(); + assert!(report.maintenance_held); + assert_eq!(report.legacy_restores_removed, ["LegacyTarget"]); +} + +/// The worker runs a cycle at once (observed through the maintenance check, +/// the cycle's first read), records the worker metrics the other workers +/// record, then waits an interval, and stops at the shutdown token. +#[tokio::test] +async fn the_reconciler_worker_runs_at_startup_and_stops_on_shutdown() { + use extenddb_core::metrics::{MetricName, MetricsCollector}; + let fake = Arc::new(Fake::default()); + *fake.maintenance_held.lock().unwrap() = true; + let metrics = Arc::new(MetricsCollector::new()); + let shutdown = extenddb_storage::CancellationToken::new(); + let worker = tokio::spawn(reconciler::run( + Arc::clone(&fake), + reconciler::ReconcilerSettings { + interval: Duration::from_secs(3600), + stale_job_after: Duration::from_secs(300), + }, + Arc::clone(&metrics), + shutdown.clone(), + )); + let deadline = tokio::time::Instant::now() + Duration::from_secs(5); + while *fake.maintenance_checks.lock().unwrap() == 0 { + assert!( + tokio::time::Instant::now() < deadline, + "the first cycle did not run" + ); + tokio::time::sleep(Duration::from_millis(5)).await; + } + assert_eq!( + *fake.maintenance_checks.lock().unwrap(), + 1, + "one cycle before the first interval" + ); + shutdown.cancel(); + tokio::time::timeout(Duration::from_secs(5), worker) + .await + .expect("the worker stops at shutdown") + .unwrap(); + assert_eq!(*fake.maintenance_checks.lock().unwrap(), 1); + let recorded: Vec = metrics + .drain(Duration::ZERO) + .into_iter() + .filter(|b| b.operation == "BackupReconciler") + .map(|b| b.metric) + .collect(); + assert!( + recorded.contains(&MetricName::WorkerLastSuccess) + && recorded.contains(&MetricName::WorkerCycleLatency), + "{recorded:?}" + ); + assert!(!recorded.contains(&MetricName::WorkerErrorCount)); +} + +/// A cycle that fails is recorded as a worker error, as the other workers +/// record theirs, and the worker goes on to the next cycle. +#[tokio::test] +async fn a_failed_reconciler_cycle_records_a_worker_error() { + use extenddb_core::metrics::{MetricName, MetricsCollector}; + let fake = Arc::new(Fake::default()); + *fake.maintenance_error.lock().unwrap() = Some("catalog unreachable".to_owned()); + let metrics = Arc::new(MetricsCollector::new()); + let shutdown = extenddb_storage::CancellationToken::new(); + let worker = tokio::spawn(reconciler::run( + Arc::clone(&fake), + reconciler::ReconcilerSettings { + interval: Duration::from_millis(5), + stale_job_after: Duration::from_secs(300), + }, + Arc::clone(&metrics), + shutdown.clone(), + )); + let deadline = tokio::time::Instant::now() + Duration::from_secs(5); + while *fake.maintenance_checks.lock().unwrap() < 2 { + assert!( + tokio::time::Instant::now() < deadline, + "the worker did not go on after a failed cycle" + ); + tokio::time::sleep(Duration::from_millis(5)).await; + } + shutdown.cancel(); + tokio::time::timeout(Duration::from_secs(5), worker) + .await + .expect("the worker stops at shutdown") + .unwrap(); + let errors: f64 = metrics + .drain(Duration::ZERO) + .into_iter() + .filter(|b| b.operation == "BackupReconciler") + .filter(|b| b.metric == MetricName::WorkerErrorCount) + .map(|b| b.sum) + .sum(); + assert!(errors >= 2.0, "{errors} worker errors recorded"); +} + +/// The FAILED-target duty: a FAILED record whose target is still CREATING +/// (the FAILED write landed, the removal did not) is listed by the backend +/// once stale, and the reconciler removes the target with no second FAILED +/// write. +#[tokio::test] +async fn the_reconciler_removes_the_target_of_a_failed_record_without_writing_failed_again() { + use extenddb_storage::backup_catalog::{JobStatus, RestoreKind}; + let fake = Fake::default(); + *fake.target_still_creating.lock().unwrap() = true; + let mut failed = dead_job(RestoreKind::Legacy, "FailedTarget"); + failed.status = JobStatus::Failed; + let mut failed_manifest = dead_job(RestoreKind::Manifest, "FailedManifestTarget"); + failed_manifest.status = JobStatus::Failed; + fake.dead_jobs + .lock() + .unwrap() + .extend([failed.clone(), failed_manifest.clone()]); + let report = reconciler::cycle( + &fake, + Duration::from_secs(300), + &mut reconciler::ReconcilerState::default(), + ) + .await + .unwrap(); + assert!( + fake.failed_jobs.lock().unwrap().is_empty(), + "a FAILED record is not written FAILED again" + ); + assert_eq!( + report.failed_targets_removed, + ["FailedTarget", "FailedManifestTarget"] + ); + assert!(report.legacy_restores_removed.is_empty()); + assert!(report.manifest_restores_seen.is_empty()); + assert_eq!( + fake.removed_targets.lock().unwrap().as_slice(), + [failed.target, failed_manifest.target] + ); +} + +/// A dead manifest record, which this release cannot act on, is reported +/// once per record for as long as the process runs, not every cycle. +#[tokio::test] +async fn a_dead_manifest_record_is_logged_once_per_record() { + use extenddb_storage::backup_catalog::RestoreKind; + let fake = Fake::default(); + fake.dead_jobs + .lock() + .unwrap() + .push(dead_job(RestoreKind::Manifest, "ManifestTarget")); + let mut state = reconciler::ReconcilerState::default(); + let first = reconciler::cycle(&fake, Duration::from_secs(300), &mut state) + .await + .unwrap(); + assert_eq!(first.manifest_restores_seen, ["ManifestTarget"]); + assert_eq!(first.manifest_restores_logged, ["ManifestTarget"]); + let second = reconciler::cycle(&fake, Duration::from_secs(300), &mut state) + .await + .unwrap(); + assert_eq!(second.manifest_restores_seen, ["ManifestTarget"]); + assert!( + second.manifest_restores_logged.is_empty(), + "the same record is not reported twice" + ); + fake.dead_jobs + .lock() + .unwrap() + .push(dead_job(RestoreKind::Manifest, "Another")); + let third = reconciler::cycle(&fake, Duration::from_secs(300), &mut state) + .await + .unwrap(); + assert_eq!(third.manifest_restores_logged, ["Another"]); + assert!( + fake.failed_jobs.lock().unwrap().is_empty() + && fake.removed_targets.lock().unwrap().is_empty(), + "a manifest record is never acted on by this release" + ); +} diff --git a/crates/engine/src/create_table.rs b/crates/engine/src/create_table.rs index 7b827cd3a..ff51fac17 100755 --- a/crates/engine/src/create_table.rs +++ b/crates/engine/src/create_table.rs @@ -150,6 +150,12 @@ pub fn storage_err_to_dynamo(e: extenddb_storage::error::StorageError) -> Dynamo StorageError::IndexAlreadyExists(name) => DynamoDbError::ValidationException(format!( "One or more parameter values were invalid: Index already exists: {name}" )), + // Raised by the backup catalog's claim alone, whose caller handles + // it; here it is a programming error, reported without the row. + StorageError::AlreadyExists(msg) => { + tracing::error!(internal_error = %msg, "unexpected already-exists storage error"); + DynamoDbError::InternalServerError("Internal server error".to_owned()) + } // The resource exists and the request is well formed; its current state // forbids the change. The backend owns the wording because it owns the // state: the documented delete-table sentence and the measured diff --git a/crates/engine/src/lib.rs b/crates/engine/src/lib.rs index 74083fbfa..a5c6ed77a 100755 --- a/crates/engine/src/lib.rs +++ b/crates/engine/src/lib.rs @@ -8,6 +8,7 @@ //! `DynamoDB`-format error responses. mod backup; +pub use backup::{BackupRuntime, reconciler}; mod batch_get_item; mod batch_write_item; pub mod capacity_helpers; @@ -341,6 +342,9 @@ pub type AuthorizedKeyInfo = std::collections::HashMap, + /// The process's job slots and configured backup store, which the backup + /// operations read. + pub backup: Arc, pub limits: Arc, pub region: Arc, pub account_id: Arc, diff --git a/crates/server/src/handler.rs b/crates/server/src/handler.rs index ef3765bda..aeef4ccd1 100755 --- a/crates/server/src/handler.rs +++ b/crates/server/src/handler.rs @@ -147,6 +147,7 @@ pub(crate) async fn handle_request( // Dispatch let ctx = OperationContext { storage: state.storage.clone(), + backup: state.backup_runtime.clone(), limits: state.limits.clone(), region: state.region.clone(), account_id, diff --git a/crates/server/src/lib.rs b/crates/server/src/lib.rs index a8d423599..a14715cb6 100755 --- a/crates/server/src/lib.rs +++ b/crates/server/src/lib.rs @@ -53,6 +53,8 @@ use tower_http::set_header::SetResponseHeaderLayer; /// Application state shared across all handlers. pub struct AppState { pub storage: Arc, + /// The job slots and configured backup store the backup operations read. + pub backup_runtime: Arc, pub auth: Arc, pub limits: Arc, // Fix #9: Use Arc to avoid per-request cloning diff --git a/crates/server/src/serve.rs b/crates/server/src/serve.rs index ee52ba9f2..a9dce5254 100644 --- a/crates/server/src/serve.rs +++ b/crates/server/src/serve.rs @@ -320,6 +320,7 @@ async fn serve_inner(params: ServeParams, port: u16) -> anyhow::Result<()> { component_options.bootstrap_if_uninitialized = dev_mode; component_options.release = build.version; + component_options.max_concurrent_jobs = app_config.backup.max_concurrent_jobs; // Wire the node's own bind address as the instance ID so each ExtendDB node // has a distinct node ID (to support logical clocks for sequences, etc.). @@ -627,8 +628,12 @@ async fn serve_inner(params: ServeParams, port: u16) -> anyhow::Result<()> { initial_throttling, )); + let backup_runtime = Arc::new(extenddb_engine::BackupRuntime::new( + app_config.backup.max_concurrent_jobs, + )); let state = AppState { - storage, + storage: storage.clone(), + backup_runtime, auth, limits, region: Arc::from(app_config.server.region.as_str()), @@ -705,6 +710,18 @@ async fn serve_inner(params: ServeParams, port: u16) -> anyhow::Result<()> { worker_handles.extend(hooks.spawn_workers(&worker_ctx).await); } + // The backup reconciler: backend-agnostic, so it runs here beside the + // backends' workers rather than in each backend's `spawn_workers`. + worker_handles.push(tokio::spawn(extenddb_engine::reconciler::run( + storage.clone(), + extenddb_engine::reconciler::ReconcilerSettings { + interval: std::time::Duration::from_secs(app_config.backup.reconcile_interval_secs), + stale_job_after: std::time::Duration::from_secs(app_config.backup.stale_job_after_secs), + }, + metrics.clone(), + shutdown.clone(), + ))); + let tls_config = if tls_enabled { let cert_path = config::expand_tilde(&app_config.server.tls.cert_path); let key_path = config::expand_tilde(&app_config.server.tls.key_path); diff --git a/crates/storage-cassandra/Cargo.toml b/crates/storage-cassandra/Cargo.toml index 92e3fab95..ba3378b32 100644 --- a/crates/storage-cassandra/Cargo.toml +++ b/crates/storage-cassandra/Cargo.toml @@ -7,6 +7,11 @@ edition.workspace = true rust-version.workspace = true license.workspace = true +[features] +# Coordination hooks used only by this crate's integration test binary +# (`src/test_hooks.rs`). Production builds do not compile them. +test-hooks = [] + [dependencies] extenddb-core = { workspace = true } extenddb-storage = { workspace = true } diff --git a/crates/storage-cassandra/migrations/catalog/V006__backup_catalog_index.cql b/crates/storage-cassandra/migrations/catalog/V006__backup_catalog_index.cql new file mode 100644 index 000000000..d681ee4bf --- /dev/null +++ b/crates/storage-cassandra/migrations/catalog/V006__backup_catalog_index.cql @@ -0,0 +1,76 @@ +-- Copyright 2026 ExtendDB contributors +-- SPDX-License-Identifier: Apache-2.0 +-- The catalog index of backups and restore jobs (catalog version 0.0.5). +-- +-- backups_by_arn is reused for the index rows of backups whose bytes live in +-- a backup store. A row with store_kind set is such a row and belongs to the +-- catalog index. A row with store_kind unset is an in-catalog backup written +-- by this or an earlier release and belongs to the legacy reader. Neither +-- path sees the other's rows. owner_generation and heartbeat_at are the claim +-- of the job that owns the row (both kinds). backup_size_bytes and item_count, +-- already present, are the index row's size_bytes and item_count. +-- +-- backup_index_by_status gives the status-keyed read backups_by_arn cannot: +-- one partition per (store root, status), maintained beside every status +-- write of an index row. +-- +-- tables.restore_* gain the restore job record: status, kind, the store the +-- restore reads, the claim (restore_owner_generation and restore_heartbeat_at, +-- beside the restore_owner and restore_heartbeat columns of the previous +-- release, which stop being written), spool and progress fields, the failure +-- reason and the completion time. The registrations that exist are backfilled +-- by the migration runner after these statements: kind legacy, DONE where the +-- table is not CREATING, RUNNING with the epoch heartbeat and the unowned +-- generation where it is. +-- +-- backup_maintenance holds the catalog-level maintenance claim, one row. +-- +-- settings.catalog_id is written by the runner after these statements, once. +-- settings.catalog_version is written by the runner too, after the backfill +-- and after this file is recorded in schema_history, so a run that fails in +-- the backfill leaves the version at 0.0.4 and migrate --yes runs this file +-- again. +-- +-- NOTE: the migration runner splits files on the semicolon character, +-- including inside comments. Do not use one in a comment. +ALTER TABLE backups_by_arn ADD IF NOT EXISTS backup_id text; +ALTER TABLE backups_by_arn ADD IF NOT EXISTS store_kind text; +ALTER TABLE backups_by_arn ADD IF NOT EXISTS store_root text; +ALTER TABLE backups_by_arn ADD IF NOT EXISTS store_id text; +ALTER TABLE backups_by_arn ADD IF NOT EXISTS owner_generation bigint; +ALTER TABLE backups_by_arn ADD IF NOT EXISTS heartbeat_at timestamp; +ALTER TABLE backups_by_arn ADD IF NOT EXISTS completed_at timestamp; +ALTER TABLE backups_by_arn ADD IF NOT EXISTS failure_reason text; +ALTER TABLE backups_by_arn ADD IF NOT EXISTS source_table_details_json text; + +CREATE TABLE IF NOT EXISTS backup_index_by_status ( + store_root text, + backup_status text, + backup_arn text, + account_id text, + store_kind text, + store_id text, + PRIMARY KEY ((store_root, backup_status), backup_arn) +); + +ALTER TABLE tables ADD IF NOT EXISTS restore_status text; +ALTER TABLE tables ADD IF NOT EXISTS restore_kind text; +ALTER TABLE tables ADD IF NOT EXISTS restore_store_kind text; +ALTER TABLE tables ADD IF NOT EXISTS restore_store_root text; +ALTER TABLE tables ADD IF NOT EXISTS restore_store_id text; +ALTER TABLE tables ADD IF NOT EXISTS restore_owner_generation bigint; +ALTER TABLE tables ADD IF NOT EXISTS restore_heartbeat_at timestamp; +ALTER TABLE tables ADD IF NOT EXISTS restore_spool_host text; +ALTER TABLE tables ADD IF NOT EXISTS restore_file_index int; +ALTER TABLE tables ADD IF NOT EXISTS restore_items_applied bigint; +ALTER TABLE tables ADD IF NOT EXISTS restore_failure_reason text; +ALTER TABLE tables ADD IF NOT EXISTS restore_completed_at timestamp; + +CREATE TABLE IF NOT EXISTS backup_maintenance ( + claim text, + holder text, + owner_generation bigint, + claimed_at timestamp, + heartbeat_at timestamp, + PRIMARY KEY (claim) +); diff --git a/crates/storage-cassandra/src/backup_catalog.rs b/crates/storage-cassandra/src/backup_catalog.rs new file mode 100644 index 000000000..bfd2a2d2c --- /dev/null +++ b/crates/storage-cassandra/src/backup_catalog.rs @@ -0,0 +1,1282 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! The catalog index on Cassandra: the index rows in the reused +//! `backups_by_arn`, `backups_by_account` and `backups_by_table` tables, the +//! status-keyed `backup_index_by_status`, the restore job record on the +//! target's `tables` row, `backup_maintenance` and `settings.catalog_id`. +//! +//! An index row is a `backups_by_arn` row with `store_kind` set; a row without +//! it is an in-catalog backup of the legacy reader (`backup_engine.rs`), and +//! each path skips the other's rows. `backups_by_arn`, partitioned by +//! `(account_id, backup_arn)`, is authoritative and is written first by every +//! writer; the list rows and the `backup_index_by_status` row follow, and a +//! reader of those re-reads the authoritative row and skips an entry it no +//! longer matches, so a writer that dies between the writes leaves nothing a +//! reader believes. An index row's list rows exist for as long as the row +//! does, of whatever status (the legacy create writes its list rows only when +//! its row becomes AVAILABLE); `remove_backup_row` takes them with the row. +//! +//! The claim is the row: `claim_backup` inserts it `IF NOT EXISTS` with a +//! fresh `owner_generation` and `heartbeat_at = toTimestamp(now())`, every +//! later write by the job is a lightweight transaction `IF owner_generation = +//! `, and a heartbeat that does not apply means another owner has +//! the row. Times are the coordinator's clock; staleness is measured against +//! it (`coordinator_now_millis`), never an application host's. +//! +//! DeleteBackup's lock is the lightweight transaction on the `backups_by_arn` +//! row, `IF backup_status = 'AVAILABLE' AND active_restores = null`, on the +//! same partition a restore registers on, so the two order wholly one before +//! the other, as the in-catalog paths do. +//! +//! A status change moves the row's `backup_index_by_status` entry with a +//! plain batch after the lightweight transaction on the row. A process that +//! dies between the two leaves an entry whose status the row no longer +//! matches; every reader of the status table re-reads the row and skips such +//! an entry, and `remove_backup_row` deletes the entry under the status it +//! read, so the orphan costs a wasted read until then. The sweep that +//! removes orphan entries is `backup sync`'s, in a later release. +//! +//! A conditional write whose error is the driver's `Io` or `Timeout`, or the +//! server's `WriteTimeout` or `WriteFailure`, has an unknown outcome +//! (`WriteOutcome::Unknown`: the Paxos round may still commit); every other +//! error is `Err`. + +use std::time::Duration; + +use cdrs_tokio::consistency::Consistency; +use cdrs_tokio::error::Error as DriverError; +use cdrs_tokio::frame::message_error::ErrorType; +use cdrs_tokio::query::{BatchQueryBuilder, QueryValues}; +use cdrs_tokio::statement::StatementParamsBuilder; +use cdrs_tokio::types::IntoRustByName; +use cdrs_tokio::types::rows::Row; +use cdrs_tokio::types::value::Value; +use extenddb_core::backup_format::BackupArn; +use extenddb_storage::backup_catalog::{ + BackupCatalog, BackupKey, BackupRow, BackupStatus, DeleteStart, Heartbeat, JobClaim, JobRow, + JobStatus, RestoreJobRow, RestoreKind, RestoreTargetKey, StatusFields, StoreIdentity, + StoreKind, TimeBounds, WriteOutcome, billing_mode_name, new_generation, +}; +use extenddb_storage::error::StorageError; +use futures::future::BoxFuture; +use futures::stream::BoxStream; + +use crate::CassandraEngine; +use crate::engine::CassandraSession; + +/// Attempts `start_backup_delete` makes when a restore registers or +/// unregisters between its in-use check and the DELETING claim. +const DELETE_CLAIM_ATTEMPTS: usize = 3; + +/// The `owner_generation` column value of a generation: the column is a +/// signed 64-bit integer and `new_generation` never exceeds `i64::MAX`. +pub(crate) fn generation_column(generation: u64) -> i64 { + i64::try_from(generation).unwrap_or(i64::MAX) +} + +/// The generation a column holds; a negative value no release writes reads +/// as the unowned sentinel. +fn generation_from_column(column: Option) -> u64 { + column + .and_then(|g| u64::try_from(g).ok()) + .unwrap_or_default() +} + +fn not_found(backup_arn: &str) -> StorageError { + extenddb_storage::backup_catalog::backup_not_found(backup_arn) +} + +fn millis_to_seconds(ms: i64) -> f64 { + crate::cassandra_util::millis_to_seconds_f64(ms) +} + +#[allow(clippy::cast_possible_truncation)] +fn seconds_to_millis(seconds: f64) -> i64 { + (seconds * 1_000.0).round() as i64 +} + +/// The coordinator's clock, in epoch milliseconds: the clock every heartbeat +/// is stamped with, so a stale window is measured on it. +pub(crate) async fn coordinator_now_millis(engine: &CassandraEngine) -> Result { + let rows = crate::cassandra_util::query_rows::( + &engine.session, + "SELECT toTimestamp(now()) AS now FROM system.local", + cdrs_tokio::query_values!(), + "coordinator clock", + ) + .await?; + let row = rows + .first() + .ok_or_else(|| StorageError::Internal("coordinator clock: no row".to_owned()))?; + crate::cassandra_util::get_column(row, "now", "coordinator clock") +} + +/// Refresh `heartbeat_at` of a `backups_by_arn` row on the coordinator's +/// clock, conditioned on the claim's generation; `false` when another owner +/// holds the row. Shared by the index claim and the in-catalog create. +#[doc(hidden)] +pub async fn heartbeat_backup_row( + engine: &CassandraEngine, + account_id: &str, + backup_arn: &str, + generation: u64, +) -> Result { + let query = format!( + "UPDATE {}.backups_by_arn SET heartbeat_at = toTimestamp(now()) \ + WHERE account_id = ? AND backup_arn = ? IF owner_generation = ?", + engine.catalog_keyspace() + ); + let result = crate::cassandra_util::query_lwt( + &engine.session, + &query, + cdrs_tokio::query_values!(account_id, backup_arn, generation_column(generation)), + ) + .await?; + crate::cassandra_util::lwt_applied(&result) +} + +/// Whether a write's outcome cannot be told from its error: the statement +/// may have reached a replica before the answer was lost. A write timeout +/// or failure on a lightweight transaction is the Paxos round not completing +/// within the coordinator's view; the proposal may still commit. +fn outcome_unknown(error: &DriverError) -> bool { + match error { + DriverError::Io(_) | DriverError::Timeout(_) => true, + DriverError::Server { body, .. } => matches!( + body.ty, + ErrorType::WriteTimeout(_) | ErrorType::WriteFailure(_) + ), + _ => false, + } +} + +/// Run a conditional write and report what it did. +async fn conditional_write( + session: &CassandraSession, + query: &str, + values: QueryValues, +) -> Result { + let params = StatementParamsBuilder::new() + .with_consistency(Consistency::LocalQuorum) + .with_serial_consistency(Consistency::LocalSerial) + .with_values(values) + .build(); + match session.query_with_params(query, params).await { + Ok(result) => Ok(if crate::cassandra_util::lwt_applied(&result)? { + WriteOutcome::Matched + } else { + WriteOutcome::NotMatched + }), + Err(error) if outcome_unknown(&error) => { + tracing::warn!("conditional catalog write with an unknown outcome: {error}"); + Ok(WriteOutcome::Unknown) + } + Err(error) => Err(StorageError::Internal(format!("LWT query failed: {error}"))), + } +} + +/// The index row a `backups_by_arn` row carries, or `None` for an in-catalog +/// backup (no `store_kind`). +fn index_row_from_row(row: &Row) -> Result, StorageError> { + let store_kind: Option = row.get_by_name("store_kind").ok().flatten(); + let Some(store_kind) = store_kind else { + return Ok(None); + }; + let column = |name: &str| -> Result { + crate::cassandra_util::get_column(row, name, "backup index row") + }; + let optional = |name: &str| -> Option { row.get_by_name(name).ok().flatten() }; + let backup_arn = column("backup_arn")?; + let store_kind = StoreKind::parse(&store_kind).ok_or_else(|| { + StorageError::Internal(format!( + "backup {backup_arn} carries an unknown store kind {store_kind:?}" + )) + })?; + let status: String = column("backup_status")?; + let backup_id = match optional("backup_id") { + Some(id) => id, + None => BackupArn::parse(&backup_arn) + .map(|arn| arn.backup_id().to_owned()) + .map_err(|e| { + StorageError::Internal(format!("backup {backup_arn} carries no backup id: {e}")) + })?, + }; + let source_table_details_json = match optional("source_table_details_json") { + Some(text) => serde_json::from_str(&text).map_err(|e| { + StorageError::Internal(format!( + "backup {backup_arn} carries a source table this build cannot read: {e}" + )) + })?, + None => serde_json::Value::Null, + }; + let created_at: i64 = crate::cassandra_util::get_column(row, "created_at", "backup index row")?; + let completed_at: Option = row.get_by_name("completed_at").ok().flatten(); + let heartbeat_at: Option = row.get_by_name("heartbeat_at").ok().flatten(); + let owner_generation: Option = row.get_by_name("owner_generation").ok().flatten(); + Ok(Some(BackupRow { + backup_id, + account_id: column("account_id")?, + table_name: column("table_name")?, + backup_arn, + backup_name: column("backup_name")?, + status: BackupStatus::parse(&status)?, + store: StoreIdentity { + store_kind, + store_root: column("store_root")?, + store_id: column("store_id")?, + }, + size_bytes: row + .get_by_name("backup_size_bytes") + .ok() + .flatten() + .unwrap_or(0), + item_count: row.get_by_name("item_count").ok().flatten().unwrap_or(0), + created_at: millis_to_seconds(created_at), + completed_at: completed_at.map(millis_to_seconds), + failure_reason: optional("failure_reason"), + source_table_details_json, + owner_generation: generation_from_column(owner_generation), + heartbeat_at: heartbeat_at.map_or(0.0, millis_to_seconds), + })) +} + +/// The rows `read` fetches, as a stream: the read runs once, on the first +/// poll, and a failed read is one `Err` item. +fn rows_stream<'a, F>(read: F) -> BoxStream<'a, Result> +where + F: std::future::Future, StorageError>> + Send + 'a, +{ + use futures::StreamExt; + Box::pin( + futures::stream::once(async move { + match read.await { + Ok(rows) => rows.into_iter().map(Ok).collect::>(), + Err(e) => vec![Err(e)], + } + }) + .flat_map(futures::stream::iter), + ) +} + +/// `(created_at, backup_arn)` ascending, the order every listing returns. +fn sort_listing(rows: &mut [BackupRow]) { + rows.sort_by(|a, b| { + a.created_at + .partial_cmp(&b.created_at) + .unwrap_or(std::cmp::Ordering::Equal) + .then_with(|| a.backup_arn.cmp(&b.backup_arn)) + }); +} + +/// A backup job's handle on its `backups_by_arn` row. +struct CassandraBackupClaim { + engine: CassandraEngine, + key: BackupKey, + store: StoreIdentity, + generation: u64, +} + +impl JobClaim for CassandraBackupClaim { + fn generation(&self) -> u64 { + self.generation + } + + fn heartbeat(&self) -> BoxFuture<'_, Result> { + Box::pin(async move { + let confirmed = heartbeat_backup_row( + &self.engine, + &self.key.account_id, + &self.key.backup_arn, + self.generation, + ) + .await?; + Ok(if confirmed { + Heartbeat::Confirmed + } else { + Heartbeat::Lost + }) + }) + } + + fn set_status<'a>( + &'a self, + from: JobStatus, + to: JobStatus, + fields: &'a StatusFields, + ) -> BoxFuture<'a, Result> { + Box::pin(async move { + let mut assignments = vec!["backup_status = ?".to_owned()]; + let mut values = vec![Value::from(to.as_str())]; + if let Some(size) = fields.size_bytes { + assignments.push("backup_size_bytes = ?".to_owned()); + values.push(Value::from(size)); + } + if let Some(count) = fields.item_count { + assignments.push("item_count = ?".to_owned()); + values.push(Value::from(count)); + } + if let Some(reason) = fields.failure_reason.as_deref() { + assignments.push("failure_reason = ?".to_owned()); + values.push(Value::from(reason)); + } + if fields.completed { + assignments.push("completed_at = toTimestamp(now())".to_owned()); + } + values.extend([ + Value::from(self.key.account_id.as_str()), + Value::from(self.key.backup_arn.as_str()), + Value::from(generation_column(self.generation)), + Value::from(from.as_str()), + ]); + let query = format!( + "UPDATE {}.backups_by_arn SET {} WHERE account_id = ? AND backup_arn = ? \ + IF owner_generation = ? AND backup_status = ?", + self.engine.catalog_keyspace(), + assignments.join(", ") + ); + let outcome = conditional_write( + &self.engine.session, + &query, + QueryValues::SimpleValues(values), + ) + .await?; + if outcome == WriteOutcome::Matched { + self.engine + .move_status_index_row(&self.key, &self.store, from.as_str(), to.as_str()) + .await; + } + Ok(outcome) + }) + } + + fn read(&self) -> BoxFuture<'_, Result> { + Box::pin(async move { + let query = format!( + "SELECT backup_status, owner_generation, heartbeat_at, failure_reason \ + FROM {}.backups_by_arn WHERE account_id = ? AND backup_arn = ?", + self.engine.catalog_keyspace() + ); + let rows = crate::cassandra_util::query_rows_quorum::( + &self.engine.session, + &query, + cdrs_tokio::query_values!( + self.key.account_id.as_str(), + self.key.backup_arn.as_str() + ), + "job row", + ) + .await?; + let row = rows + .first() + .ok_or_else(|| not_found(&self.key.backup_arn))?; + let status: String = + crate::cassandra_util::get_column(row, "backup_status", "job row")?; + let generation: Option = row.get_by_name("owner_generation").ok().flatten(); + let heartbeat_at: Option = row.get_by_name("heartbeat_at").ok().flatten(); + Ok(JobRow { + status: JobStatus::parse(&status)?, + owner_generation: generation_from_column(generation), + heartbeat_at: heartbeat_at.map_or(0.0, millis_to_seconds), + failure_reason: row.get_by_name("failure_reason").ok().flatten(), + }) + }) + } +} + +impl CassandraEngine { + /// The index row `backup_arn` names in `account_id`, if there is one; an + /// in-catalog row is `None`. + async fn read_index_row( + &self, + account_id: &str, + backup_arn: &str, + ) -> Result, StorageError> { + let query = format!( + "SELECT * FROM {}.backups_by_arn WHERE account_id = ? AND backup_arn = ?", + self.catalog_keyspace() + ); + let rows = crate::cassandra_util::query_rows_quorum::( + &self.session, + &query, + cdrs_tokio::query_values!(account_id, backup_arn), + "backup index row", + ) + .await?; + rows.first() + .map(index_row_from_row) + .transpose() + .map(|row| row.flatten().filter(|r| r.account_id == account_id)) + } + + /// Move the `backup_index_by_status` entry of a row from one status + /// partition to another, after the authoritative row changed. A failure + /// is logged: a reader of the status index re-reads the authoritative + /// row and skips an entry whose status it no longer matches. + async fn move_status_index_row( + &self, + key: &BackupKey, + store: &StoreIdentity, + from: &str, + to: &str, + ) { + let catalog = self.catalog_keyspace(); + let batch = BatchQueryBuilder::new() + .with_consistency(Consistency::LocalQuorum) + .add_query( + format!( + "DELETE FROM {catalog}.backup_index_by_status \ + WHERE store_root = ? AND backup_status = ? AND backup_arn = ?" + ), + cdrs_tokio::query_values!(store.store_root.as_str(), from, key.backup_arn.as_str()), + ) + .add_query( + format!( + "INSERT INTO {catalog}.backup_index_by_status \ + (store_root, backup_status, backup_arn, account_id, store_kind, store_id) \ + VALUES (?, ?, ?, ?, ?, ?)" + ), + cdrs_tokio::query_values!( + store.store_root.as_str(), + to, + key.backup_arn.as_str(), + key.account_id.as_str(), + store.store_kind.as_str(), + store.store_id.as_str() + ), + ); + let sent = match batch.build() { + Ok(batch) => self.session.batch(batch).await.map(|_| ()), + Err(error) => Err(error), + }; + if let Err(error) = sent { + tracing::warn!( + "backup {}: its status index entry could not be moved from {from} to {to} \ + ({error}); readers verify against the row", + key.backup_arn + ); + } + } + + /// The ARNs the status index lists for `store` under `status`. + async fn status_index_arns( + &self, + store: &StoreIdentity, + status: BackupStatus, + ) -> Result, StorageError> { + let query = format!( + "SELECT backup_arn, account_id, store_kind, store_id FROM {}.backup_index_by_status \ + WHERE store_root = ? AND backup_status = ?", + self.catalog_keyspace() + ); + let rows = crate::cassandra_util::query_rows_quorum::( + &self.session, + &query, + cdrs_tokio::query_values!(store.store_root.as_str(), status.as_str()), + "status index", + ) + .await?; + let mut arns = Vec::with_capacity(rows.len()); + for row in rows { + let store_kind: Option = row.get_by_name("store_kind").ok().flatten(); + let store_id: Option = row.get_by_name("store_id").ok().flatten(); + if store_kind.as_deref() != Some(store.store_kind.as_str()) + || store_id.as_deref() != Some(store.store_id.as_str()) + { + continue; + } + let backup_arn: String = + crate::cassandra_util::get_column(&row, "backup_arn", "status index")?; + let account_id: String = + crate::cassandra_util::get_column(&row, "account_id", "status index")?; + arns.push((account_id, backup_arn)); + } + Ok(arns) + } + + /// The `tables` row of a restore target, read at quorum. + async fn restore_target_row( + &self, + account_id: &str, + table_name: &str, + ) -> Result, StorageError> { + let query = format!( + "SELECT table_id, table_status, restore_source_backup_arn, restore_status \ + FROM {}.tables WHERE account_id = ? AND table_name = ?", + self.catalog_keyspace() + ); + let rows = crate::cassandra_util::query_rows_quorum::( + &self.session, + &query, + cdrs_tokio::query_values!(account_id, table_name), + "restore target", + ) + .await?; + rows.first() + .map(|row| { + Ok(RestoreTargetRow { + table_id: crate::cassandra_util::get_column(row, "table_id", "restore target")?, + table_status: row.get_by_name("table_status").ok().flatten(), + source_backup_arn: row.get_by_name("restore_source_backup_arn").ok().flatten(), + job_status: row.get_by_name("restore_status").ok().flatten(), + }) + }) + .transpose() + } +} + +/// The columns of a `tables` row `remove_restore_target` decides on. +struct RestoreTargetRow { + table_id: String, + table_status: Option, + source_backup_arn: Option, + job_status: Option, +} + +impl RestoreTargetRow { + /// Whether this row is still the CREATING restore target the removal + /// claims first. + fn is_creating_target(&self) -> bool { + self.table_status.as_deref() == Some("CREATING") + } + + /// Whether this row is a target whose removal was claimed (CREATING to + /// DELETING) and did not finish: DELETING with its registration and a + /// FAILED record, the state `remove_restore_target` resumes from the + /// drop. A DELETING row without a registration is a client's DeleteTable + /// in flight, and one whose record is not FAILED (a restored table a + /// client is deleting) belongs to that delete. + fn is_claimed_for_removal(&self) -> bool { + self.table_status.as_deref() == Some("DELETING") + && self.source_backup_arn.is_some() + && self.job_status.as_deref() == Some("FAILED") + } +} + +impl BackupCatalog for CassandraEngine { + fn catalog_id(&self) -> BoxFuture<'_, Result> { + Box::pin(async move { + let query = format!( + "SELECT value FROM {}.settings WHERE key = 'catalog_id'", + self.catalog_keyspace() + ); + let rows = crate::cassandra_util::query_rows_quorum::( + &self.session, + &query, + cdrs_tokio::query_values!(), + "catalog id", + ) + .await?; + let row = rows.first().ok_or_else(|| { + StorageError::Internal( + "the catalog has no catalog_id; run extenddb migrate --yes".to_owned(), + ) + })?; + crate::cassandra_util::get_column(row, "value", "catalog id") + }) + } + + fn claim_backup<'a>( + &'a self, + row: &'a BackupRow, + ) -> BoxFuture<'a, Result, StorageError>> { + Box::pin(async move { + Self::validate_account_id(&row.account_id)?; + self.require_migrated_catalog().await?; + let source = row.source_table()?; + let key_schema = serde_json::to_string(&source.key_schema) + .map_err(|e| StorageError::Internal(format!("Serialize key schema: {e}")))?; + let attribute_definitions = serde_json::to_string(&source.attribute_definitions) + .map_err(|e| StorageError::Internal(format!("Serialize attributes: {e}")))?; + let generation = new_generation(); + let catalog = self.catalog_keyspace(); + // The row is the claim: it exists only owned. The legacy columns + // every `backups_by_arn` row carries are filled from the source + // table, so the row describes itself to either reader of it. + let insert = format!( + "INSERT INTO {catalog}.backups_by_arn (account_id, backup_arn, backup_id, \ + backup_name, table_id, table_name, table_arn, backup_status, backup_type, \ + backup_size_bytes, item_count, key_schema, attribute_definitions, billing_mode, \ + table_created_at, created_at, store_kind, store_root, store_id, \ + source_table_details_json, owner_generation, heartbeat_at) \ + VALUES (?, ?, ?, ?, ?, ?, ?, 'CREATING', 'USER', ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, \ + ?, ?, toTimestamp(now())) IF NOT EXISTS" + ); + let inserted = crate::cassandra_util::query_lwt( + &self.session, + &insert, + cdrs_tokio::query_values!( + row.account_id.as_str(), + row.backup_arn.as_str(), + row.backup_id.as_str(), + row.backup_name.as_str(), + source.table_id.as_str(), + row.table_name.as_str(), + source.table_arn.as_str(), + row.size_bytes, + row.item_count, + key_schema.as_str(), + attribute_definitions.as_str(), + billing_mode_name(source.billing_mode), + seconds_to_millis(source.table_creation_date_time), + seconds_to_millis(row.created_at), + row.store.store_kind.as_str(), + row.store.store_root.as_str(), + row.store.store_id.as_str(), + row.source_table_details_json.to_string().as_str(), + generation_column(generation) + ), + ) + .await?; + if !crate::cassandra_util::lwt_applied(&inserted)? { + return Err(StorageError::AlreadyExists(format!( + "backup {} already has a row", + row.backup_arn + ))); + } + // The index rows, after the authoritative row. + let created_at = seconds_to_millis(row.created_at); + let batch = BatchQueryBuilder::new() + .with_consistency(Consistency::LocalQuorum) + .add_query( + format!( + "INSERT INTO {catalog}.backups_by_account (account_id, created_at, \ + backup_arn, backup_name, table_id, table_name, table_arn, \ + backup_size_bytes) VALUES (?, ?, ?, ?, ?, ?, ?, ?)" + ), + cdrs_tokio::query_values!( + row.account_id.as_str(), + created_at, + row.backup_arn.as_str(), + row.backup_name.as_str(), + source.table_id.as_str(), + row.table_name.as_str(), + source.table_arn.as_str(), + row.size_bytes + ), + ) + .add_query( + format!( + "INSERT INTO {catalog}.backups_by_table (account_id, table_name, \ + created_at, backup_arn, backup_name, table_id, table_arn, \ + backup_size_bytes) VALUES (?, ?, ?, ?, ?, ?, ?, ?)" + ), + cdrs_tokio::query_values!( + row.account_id.as_str(), + row.table_name.as_str(), + created_at, + row.backup_arn.as_str(), + row.backup_name.as_str(), + source.table_id.as_str(), + source.table_arn.as_str(), + row.size_bytes + ), + ) + .add_query( + format!( + "INSERT INTO {catalog}.backup_index_by_status \ + (store_root, backup_status, backup_arn, account_id, store_kind, store_id) \ + VALUES (?, 'CREATING', ?, ?, ?, ?)" + ), + cdrs_tokio::query_values!( + row.store.store_root.as_str(), + row.backup_arn.as_str(), + row.account_id.as_str(), + row.store.store_kind.as_str(), + row.store.store_id.as_str() + ), + ); + self.session + .batch( + batch + .build() + .map_err(|e| StorageError::Internal(e.to_string()))?, + ) + .await + .map_err(|e| StorageError::Internal(format!("Backup index rows: {e}")))?; + Ok(Box::new(CassandraBackupClaim { + engine: self.clone(), + key: row.key(), + store: row.store.clone(), + generation, + }) as Box) + }) + } + + fn list_backup_rows<'a>( + &'a self, + status: Option, + store: &'a StoreIdentity, + ) -> BoxStream<'a, Result> { + rows_stream(async move { + let statuses = match status { + Some(status) => vec![status], + None => vec![ + BackupStatus::Creating, + BackupStatus::Available, + BackupStatus::Deleting, + BackupStatus::Deleted, + BackupStatus::Failed, + ], + }; + let mut rows = Vec::new(); + for status in statuses { + for (account_id, backup_arn) in self.status_index_arns(store, status).await? { + // The entry is a hint: the authoritative row decides. + if let Some(row) = self.read_index_row(&account_id, &backup_arn).await? + && row.status == status + && row.store == *store + { + rows.push(row); + } + } + } + sort_listing(&mut rows); + Ok(rows) + }) + } + + fn list_account_backup_rows<'a>( + &'a self, + account_id: &'a str, + table_name: Option<&'a str>, + bounds: &'a TimeBounds, + ) -> BoxStream<'a, Result> { + rows_stream(async move { + Self::validate_account_id(account_id)?; + let catalog = self.catalog_keyspace(); + let mut query = match table_name { + Some(_) => format!( + "SELECT backup_arn FROM {catalog}.backups_by_table \ + WHERE account_id = ? AND table_name = ?" + ), + None => { + format!( + "SELECT backup_arn FROM {catalog}.backups_by_account WHERE account_id = ?" + ) + } + }; + let mut values = vec![Value::from(account_id)]; + if let Some(table_name) = table_name { + values.push(Value::from(table_name)); + } + // The clustering range narrows the read; `contains` is the exact + // rule (lower inclusive, upper exclusive) on the row's seconds. + #[allow(clippy::cast_possible_truncation)] + if let Some(lower) = bounds.lower { + query.push_str(" AND created_at >= ?"); + values.push(Value::from((lower * 1_000.0).floor() as i64)); + } + #[allow(clippy::cast_possible_truncation)] + if let Some(upper) = bounds.upper { + query.push_str(" AND created_at < ?"); + values.push(Value::from((upper * 1_000.0).ceil() as i64)); + } + let params = StatementParamsBuilder::new() + .with_consistency(Consistency::LocalQuorum) + .with_values(QueryValues::SimpleValues(values)) + .build(); + let listed = self + .session + .query_with_params(&query, params) + .await + .map_err(|e| StorageError::Internal(format!("List backup index: {e}")))? + .response_body() + .map_err(|e| StorageError::Internal(format!("Parse backup index list: {e}")))? + .into_rows() + .unwrap_or_default(); + let mut rows = Vec::with_capacity(listed.len()); + for entry in listed { + let backup_arn: String = + crate::cassandra_util::get_column(&entry, "backup_arn", "backup index list")?; + // An in-catalog backup's entry reads as `None` here and is the + // legacy reader's to list. + if let Some(row) = self.read_index_row(account_id, &backup_arn).await? + && bounds.contains(row.created_at) + { + rows.push(row); + } + } + sort_listing(&mut rows); + Ok(rows) + }) + } + + fn read_backup_row<'a>( + &'a self, + arn: &'a BackupArn, + ) -> BoxFuture<'a, Result, StorageError>> { + Box::pin(async move { + let row = self + .read_index_row(arn.account_id(), &arn.to_arn_string()) + .await?; + Ok(row.filter(|row| { + row.backup_id == arn.backup_id() + && row.table_name == arn.table_name() + && row.account_id == arn.account_id() + })) + }) + } + + fn start_backup_delete<'a>( + &'a self, + key: &'a BackupKey, + ) -> BoxFuture<'a, Result> { + Box::pin(async move { + let mark_deleting = format!( + "UPDATE {}.backups_by_arn SET backup_status = 'DELETING' \ + WHERE account_id = ? AND backup_arn = ? \ + IF backup_status = 'AVAILABLE' AND active_restores = null", + self.catalog_keyspace() + ); + let mut attempt = 0; + loop { + let stored = self + .load_stored_backup(&key.account_id, &key.backup_arn) + .await? + .filter(|stored| !stored.is_in_catalog()) + .ok_or_else(|| not_found(&key.backup_arn))?; + let status = BackupStatus::parse(&stored.backup_status)?; + if status != BackupStatus::Available { + return Ok(DeleteStart::Status(status)); + } + // The in-use rule reads the target's status; a registration + // whose target is no longer being restored is cleared here so + // it cannot hold the row's lock. + if let Some(target) = self.creating_restore_target(&stored).await? { + return Ok(DeleteStart::InUse(target)); + } + crate::test_hooks::before_delete_claim(&key.backup_arn).await; + let result = crate::cassandra_util::query_lwt( + &self.session, + &mark_deleting, + cdrs_tokio::query_values!(key.account_id.as_str(), key.backup_arn.as_str()), + ) + .await?; + if crate::cassandra_util::lwt_applied(&result)? { + let store = StoreIdentity { + store_kind: StoreKind::parse(stored.store_kind.as_deref().unwrap_or("")) + .unwrap_or(StoreKind::Filesystem), + store_root: stored.store_root.clone().unwrap_or_default(), + store_id: stored.store_id.clone().unwrap_or_default(), + }; + self.move_status_index_row(key, &store, "AVAILABLE", "DELETING") + .await; + return Ok(DeleteStart::Deleting); + } + attempt += 1; + if attempt >= DELETE_CLAIM_ATTEMPTS { + // A registration landed between the check and the claim + // on every attempt: the backup is in use by it. + let stored = self + .load_stored_backup(&key.account_id, &key.backup_arn) + .await? + .ok_or_else(|| not_found(&key.backup_arn))?; + let status = BackupStatus::parse(&stored.backup_status)?; + if status != BackupStatus::Available { + return Ok(DeleteStart::Status(status)); + } + let target = stored + .active_restores + .values() + .next() + .cloned() + .unwrap_or_default(); + return Ok(DeleteStart::InUse(target)); + } + } + }) + } + + fn set_backup_deleted<'a>( + &'a self, + key: &'a BackupKey, + ) -> BoxFuture<'a, Result> { + Box::pin(async move { + let Some(row) = self + .read_index_row(&key.account_id, &key.backup_arn) + .await? + else { + return Ok(false); + }; + let query = format!( + "UPDATE {}.backups_by_arn SET backup_status = 'DELETED', \ + completed_at = toTimestamp(now()) \ + WHERE account_id = ? AND backup_arn = ? IF backup_status = 'DELETING'", + self.catalog_keyspace() + ); + let result = crate::cassandra_util::query_lwt( + &self.session, + &query, + cdrs_tokio::query_values!(key.account_id.as_str(), key.backup_arn.as_str()), + ) + .await?; + let applied = crate::cassandra_util::lwt_applied(&result)?; + if applied { + self.move_status_index_row(key, &row.store, "DELETING", "DELETED") + .await; + } + Ok(applied) + }) + } + + fn remove_backup_row<'a>( + &'a self, + key: &'a BackupKey, + ) -> BoxFuture<'a, Result> { + Box::pin(async move { + let Some(row) = self + .read_index_row(&key.account_id, &key.backup_arn) + .await? + else { + return Ok(false); + }; + if !matches!(row.status, BackupStatus::Deleted | BackupStatus::Failed) { + return Ok(false); + } + let catalog = self.catalog_keyspace(); + // The authoritative row first, conditioned on the tombstone + // status read; then its index entries. + let delete = format!( + "DELETE FROM {catalog}.backups_by_arn WHERE account_id = ? AND backup_arn = ? \ + IF backup_status = ?" + ); + let result = crate::cassandra_util::query_lwt( + &self.session, + &delete, + cdrs_tokio::query_values!( + key.account_id.as_str(), + key.backup_arn.as_str(), + row.status.as_str() + ), + ) + .await?; + if !crate::cassandra_util::lwt_applied(&result)? { + return Ok(false); + } + let created_at = seconds_to_millis(row.created_at); + let batch = BatchQueryBuilder::new() + .with_consistency(Consistency::LocalQuorum) + .add_query( + format!( + "DELETE FROM {catalog}.backup_index_by_status \ + WHERE store_root = ? AND backup_status = ? AND backup_arn = ?" + ), + cdrs_tokio::query_values!( + row.store.store_root.as_str(), + row.status.as_str(), + key.backup_arn.as_str() + ), + ) + .add_query( + format!( + "DELETE FROM {catalog}.backups_by_account \ + WHERE account_id = ? AND created_at = ? AND backup_arn = ?" + ), + cdrs_tokio::query_values!( + key.account_id.as_str(), + created_at, + key.backup_arn.as_str() + ), + ) + .add_query( + format!( + "DELETE FROM {catalog}.backups_by_table \ + WHERE account_id = ? AND table_name = ? AND created_at = ? AND backup_arn = ?" + ), + cdrs_tokio::query_values!( + key.account_id.as_str(), + row.table_name.as_str(), + created_at, + key.backup_arn.as_str() + ), + ); + self.session + .batch( + batch + .build() + .map_err(|e| StorageError::Internal(e.to_string()))?, + ) + .await + .map_err(|e| StorageError::Internal(format!("Remove backup index rows: {e}")))?; + Ok(true) + }) + } + + fn backup_in_use<'a>( + &'a self, + key: &'a BackupKey, + ) -> BoxFuture<'a, Result, StorageError>> { + Box::pin(async move { + let Some(stored) = self + .load_stored_backup(&key.account_id, &key.backup_arn) + .await? + else { + return Ok(None); + }; + self.creating_restore_target(&stored).await + }) + } + + fn remove_restore_target<'a>( + &'a self, + key: &'a RestoreTargetKey, + ) -> BoxFuture<'a, Result> { + Box::pin(async move { + // The key must name a restore target with this table id; a table + // re-created under the name has another id and is not touched. + // The removal is two writes: a claim from CREATING to DELETING, + // one lightweight transaction on the row the ACTIVE flip + // conditions on, so the two exclude each other; then the drop. + // A row the claim already moved to DELETING, whose drop did not + // finish, is resumed from the drop (`is_claimed_for_removal`). + let Some(row) = self + .restore_target_row(&key.account_id, &key.table_name) + .await? + else { + return Ok(false); + }; + let Some(backup_arn) = row.source_backup_arn.clone() else { + return Ok(false); + }; + if row.table_id != key.table_id { + return Ok(false); + } + if row.is_creating_target() { + let claim = format!( + "UPDATE {}.tables SET table_status = 'DELETING' \ + WHERE account_id = ? AND table_name = ? \ + IF table_id = ? AND table_status = 'CREATING'", + self.catalog_keyspace() + ); + let result = crate::cassandra_util::query_lwt( + &self.session, + &claim, + cdrs_tokio::query_values!( + key.account_id.as_str(), + key.table_name.as_str(), + key.table_id.as_str() + ), + ) + .await?; + if !crate::cassandra_util::lwt_applied(&result)? { + return Ok(false); + } + } else if !row.is_claimed_for_removal() { + return Ok(false); + } + self.drop_restore_target(&key.account_id, &key.table_name, &key.table_id, &backup_arn) + .await?; + Ok(true) + }) + } + + fn list_dead_restore_jobs( + &self, + stale_after: Duration, + ) -> BoxFuture<'_, Result, StorageError>> { + Box::pin(async move { + let now = coordinator_now_millis(self).await?; + #[allow(clippy::cast_possible_truncation, clippy::cast_possible_wrap)] + let cutoff = now - stale_after.as_millis() as i64; + // Two filtered scans over the tables (CQL takes one equality on a + // non-key column per scan): RUNNING records, dead when their + // heartbeat is older than the cutoff; and FAILED records whose + // table is still a restore target the removal has not finished + // (CREATING: the FAILED write landed and the removal did not; + // DELETING with its registration: the removal's claim landed and + // its drop did not), dead when their `restore_completed_at` (or, + // where none was stamped, their heartbeat) is that old. Both + // kinds are few. + let columns = "account_id, table_name, table_id, table_status, \ + restore_source_backup_arn, restore_kind, restore_status, \ + restore_owner_generation, restore_heartbeat_at, restore_completed_at"; + let mut dead = Vec::new(); + for status in ["RUNNING", "FAILED"] { + let query = format!( + "SELECT {columns} FROM {}.tables WHERE restore_status = ? ALLOW FILTERING", + self.catalog_keyspace() + ); + let rows = crate::cassandra_util::query_rows_quorum::( + &self.session, + &query, + cdrs_tokio::query_values!(status), + "dead restore jobs", + ) + .await?; + for row in rows { + let heartbeat_at: Option = + row.get_by_name("restore_heartbeat_at").ok().flatten(); + let completed_at: Option = + row.get_by_name("restore_completed_at").ok().flatten(); + let column = |name: &str| -> Result { + crate::cassandra_util::get_column(&row, name, "dead restore jobs") + }; + // A RUNNING record always carries a heartbeat (the insert + // or the backfill wrote it); one without is as dead as the + // epoch. + let age_of = if status == "FAILED" { + let target = RestoreTargetRow { + table_id: column("table_id")?, + table_status: row.get_by_name("table_status").ok().flatten(), + source_backup_arn: row + .get_by_name("restore_source_backup_arn") + .ok() + .flatten(), + job_status: Some(status.to_owned()), + }; + if !(target.is_creating_target() || target.is_claimed_for_removal()) { + continue; + } + completed_at.or(heartbeat_at) + } else { + heartbeat_at + }; + if age_of.is_some_and(|at| at >= cutoff) { + continue; + } + let kind = column("restore_kind")?; + let generation: Option = + row.get_by_name("restore_owner_generation").ok().flatten(); + dead.push(RestoreJobRow { + target: RestoreTargetKey { + account_id: column("account_id")?, + table_name: column("table_name")?, + table_id: column("table_id")?, + }, + source_backup_arn: column("restore_source_backup_arn")?, + kind: RestoreKind::parse(&kind)?, + status: JobStatus::parse(status)?, + owner_generation: generation_from_column(generation), + heartbeat_at: heartbeat_at.map_or(0.0, millis_to_seconds), + }); + } + } + Ok(dead) + }) + } + + fn fail_restore_job<'a>( + &'a self, + key: &'a RestoreTargetKey, + reason: &'a str, + ) -> BoxFuture<'a, Result> { + Box::pin(async move { + let query = format!( + "UPDATE {}.tables SET restore_status = 'FAILED', restore_failure_reason = ?, \ + restore_completed_at = toTimestamp(now()) \ + WHERE account_id = ? AND table_name = ? \ + IF table_id = ? AND restore_status = 'RUNNING'", + self.catalog_keyspace() + ); + let result = crate::cassandra_util::query_lwt( + &self.session, + &query, + cdrs_tokio::query_values!( + reason, + key.account_id.as_str(), + key.table_name.as_str(), + key.table_id.as_str() + ), + ) + .await?; + crate::cassandra_util::lwt_applied(&result) + }) + } + + fn check_maintenance_claim(&self) -> BoxFuture<'_, Result> { + Box::pin(async move { + let query = format!( + "SELECT claim FROM {}.backup_maintenance LIMIT 1", + self.catalog_keyspace() + ); + let rows = crate::cassandra_util::query_rows_quorum::( + &self.session, + &query, + cdrs_tokio::query_values!(), + "maintenance claim", + ) + .await?; + Ok(!rows.is_empty()) + }) + } +} + +#[cfg(test)] +mod tests { + use std::net::{IpAddr, Ipv4Addr, SocketAddr}; + + use cdrs_tokio::frame::message_error::{ErrorBody, WriteTimeoutError, WriteType}; + + use super::*; + + fn server_error(ty: ErrorType) -> DriverError { + DriverError::Server { + body: ErrorBody { + message: "synthetic".to_owned(), + ty, + }, + addr: SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 9042), + } + } + + /// Only a lost answer leaves a write unknown: an I/O or driver timeout, + /// or the server's write timeout on the Paxos round. A refused statement + /// (syntax, invalid) is an error with a known outcome. + #[test] + fn only_a_lost_answer_leaves_a_conditional_write_unknown() { + let cas_timeout = server_error(ErrorType::WriteTimeout(WriteTimeoutError { + cl: Consistency::LocalSerial, + received: 0, + block_for: 1, + write_type: WriteType::Cas, + contentions: Some(1), + })); + assert!(outcome_unknown(&cas_timeout)); + assert!(outcome_unknown(&DriverError::Io(std::io::Error::new( + std::io::ErrorKind::ConnectionReset, + "reset", + )))); + assert!(outcome_unknown(&DriverError::Timeout("waited".to_owned()))); + assert!(!outcome_unknown(&server_error(ErrorType::Invalid))); + assert!(!outcome_unknown(&server_error(ErrorType::Syntax))); + assert!(!outcome_unknown(&DriverError::General("x".to_owned()))); + } + + #[test] + fn generations_round_trip_through_the_column_and_a_bad_value_is_unowned() { + assert_eq!(generation_from_column(Some(generation_column(7))), 7); + assert_eq!(generation_from_column(None), 0); + assert_eq!(generation_from_column(Some(-1)), 0); + assert_eq!(generation_column(u64::MAX), i64::MAX); + } + + #[test] + fn listings_sort_by_created_at_then_arn() { + let row = |created_at: f64, arn: &str| BackupRow { + backup_id: String::new(), + account_id: String::new(), + table_name: String::new(), + backup_arn: arn.to_owned(), + backup_name: String::new(), + status: BackupStatus::Available, + store: StoreIdentity { + store_kind: StoreKind::Filesystem, + store_root: String::new(), + store_id: String::new(), + }, + size_bytes: 0, + item_count: 0, + created_at, + completed_at: None, + failure_reason: None, + source_table_details_json: serde_json::Value::Null, + owner_generation: 0, + heartbeat_at: 0.0, + }; + let mut rows = vec![row(2.0, "b"), row(1.0, "z"), row(2.0, "a")]; + sort_listing(&mut rows); + let arns: Vec<&str> = rows.iter().map(|r| r.backup_arn.as_str()).collect(); + assert_eq!(arns, ["z", "a", "b"]); + } +} diff --git a/crates/storage-cassandra/src/backup_engine.rs b/crates/storage-cassandra/src/backup_engine.rs index 7c070934e..6064211b1 100755 --- a/crates/storage-cassandra/src/backup_engine.rs +++ b/crates/storage-cassandra/src/backup_engine.rs @@ -41,12 +41,29 @@ //! DeleteBackup removes stale registrations before claiming the backup, so one //! never blocks it. //! -//! From the insert of the target row until the copy ends, the restoring -//! process refreshes `tables.restore_heartbeat` every -//! [`RESTORE_HEARTBEAT_INTERVAL`] (first refresh immediately); -//! [`CassandraEngine::sweep_abandoned_restores`] removes a target whose -//! heartbeat is older than [`RESTORE_STALE_AFTER`]. The servers run it at -//! startup and every [`RESTORE_SWEEP_INTERVAL`] ([`poll_abandoned_restores`]). +//! The target's `tables` row is also its restore job record (`restore_status` +//! RUNNING, `restore_kind` legacy, `restore_owner_generation`, +//! `restore_heartbeat_at`), written by the one `INSERT ... IF NOT EXISTS`. +//! From that insert until the copy ends, the restoring process refreshes +//! `restore_heartbeat_at` every +//! [`extenddb_storage::backup_catalog::JOB_HEARTBEAT_INTERVAL`] (first refresh +//! immediately; the one period every backend's jobs heartbeat at, which +//! `[backup] heartbeat_interval_secs` reaches in a later release) with a +//! lightweight transaction conditioned on the generation, +//! the claim's fencing token; a refresh that does not apply means another +//! owner took the target, and the copy stops without touching it +//! ([`JobHeartbeat`] states when a copy stops). The +//! engine's reconciler fails and removes a target whose heartbeat is older +//! than its stale window, through the `BackupCatalog` methods of +//! `backup_catalog.rs`. The ACTIVE flip marks the record DONE in the same +//! lightweight transaction; a failed copy marks it FAILED, conditioned on the +//! generation and on RUNNING, before the target is removed. +//! +//! The in-catalog create holds the same kind of claim on its CREATING +//! `backups_by_arn` row (`owner_generation`, `heartbeat_at`) and heartbeats it +//! while it copies. [`LegacyBackupReader::abandoned_creating_backups`] reports +//! a CREATING row whose heartbeat is stale, fenced to DELETING, for the +//! reconciler to finish through [`LegacyBackupReader::delete`]. //! //! `restore_table_from_backup` spawns one task that registers and then //! copies; the request future waits only for the registration outcome and @@ -68,11 +85,14 @@ use extenddb_core::types::{ PointInTimeRecoveryDescription, ProvisionedThroughput, SourceTableDetails, TableDescription, TableKeyInfo, }; +use extenddb_storage::backup_catalog::{AbandonedLegacyBackup, new_generation}; use extenddb_storage::backup_definition::{ BACKUP_DEFINITION_VERSION, BackupTableDefinition, throughput_from_catalog, }; use extenddb_storage::error::StorageError; -use extenddb_storage::{BackupEngine, BackupRestoreSource, MetadataEngine, RestoreRequest}; +use extenddb_storage::{ + BackupRestoreSource, JobPermit, LegacyBackupReader, MetadataEngine, RestoreRequest, +}; use futures::StreamExt; use futures::future::BoxFuture; @@ -90,15 +110,13 @@ const BACKUP_SCAN_PAGE_SIZE: i64 = 1_000; /// Items of one bucket a restore copy writes concurrently. const RESTORE_WRITE_CONCURRENCY: usize = 8; -/// How often a running restore copy refreshes its target's heartbeat. -pub const RESTORE_HEARTBEAT_INTERVAL: Duration = Duration::from_secs(15); - -/// A restore target whose heartbeat is older than this is abandoned: the -/// process copying into it is gone. Four missed heartbeats. -pub const RESTORE_STALE_AFTER: Duration = Duration::from_secs(60); - -/// How often the abandoned-restore sweep runs. -pub const RESTORE_SWEEP_INTERVAL: Duration = Duration::from_secs(15); +/// Heartbeats that error in a row after which a job stops: three beats could +/// not confirm the claim, so the row may already be another owner's. One +/// error under a loaded node does not stop a job. A beat that hangs rather +/// than erroring is not counted (the loop waits on it); the bound then is +/// the per-bucket poll of [`JobHeartbeat::is_lost`], set by the beat that +/// returns once the row carries another generation. +const HEARTBEAT_ERRORS_TO_STOP: u32 = 3; /// Attempts a restore makes to register on its backup when a stale /// registration (cleared between attempts) kept the previous one from @@ -117,12 +135,23 @@ fn backup_not_found(backup_arn: &str) -> StorageError { StorageError::Validation(format!("Backup not found: {backup_arn}")) } -fn backup_in_use(table_name: &str, backup_arn: &str) -> StorageError { - StorageError::BackupInUse(format!( - "Backup is being used to restore table {table_name}: {backup_arn}" +/// The error of a copy whose row it can no longer call its own: a heartbeat +/// found another owner on it (the reconciler found the heartbeat stale), or +/// [`HEARTBEAT_ERRORS_TO_STOP`] heartbeats in a row could not confirm the +/// claim. Nothing of the row or its payload is touched: whoever claims the +/// row (the reconciler, once the heartbeat is stale) removes it. +fn claim_lost(backup_arn: &str) -> StorageError { + StorageError::Internal(format!( + "backup {backup_arn}: the claim on its row was lost or could not be confirmed while \ + the items copied; the row's owner removes it" )) } +/// The shared text, [`extenddb_storage::backup_catalog::backup_in_use`]. +fn backup_in_use(table_name: &str, backup_arn: &str) -> StorageError { + extenddb_storage::backup_catalog::backup_in_use(table_name, backup_arn) +} + /// The error of a restore whose [`REGISTRATION_ATTEMPTS`] all found the /// backup's registration taken by a restore that was just starting (or just /// stopping, its stale entry cleared between attempts). The backup is @@ -140,7 +169,10 @@ fn registration_exhausted( } } -/// See [`CassandraEngine::require_migrated_catalog`]. +/// See [`CassandraEngine::require_migrated_catalog`]. `pending` is what +/// `pending_catalog_migrations` returned, so one refusal names only the +/// files the catalog lacks: on a 0.0.4 catalog, V006 alone (V005 is +/// recorded there). fn unmigrated_catalog_refusal(pending: &[&str]) -> StorageError { StorageError::Unsupported(format!( "Backups are unavailable until the ExtendDB catalog is migrated: it does not record \ @@ -150,14 +182,14 @@ fn unmigrated_catalog_refusal(pending: &[&str]) -> StorageError { } #[derive(Debug)] -struct StoredBackup { +pub(crate) struct StoredBackup { backup_arn: String, account_id: String, backup_name: String, table_id: String, table_name: String, table_arn: String, - backup_status: String, + pub(crate) backup_status: String, backup_type: String, backup_size_bytes: i64, item_count: i64, @@ -170,7 +202,13 @@ struct StoredBackup { /// `BackupTableDefinition` JSON; `None` for a row written before V005. definition: Option, /// Restore target `table_id` -> target table name, while registered. - active_restores: HashMap, + pub(crate) active_restores: HashMap, + /// Set on a catalog index row (a backup whose bytes live in a store); + /// unset on an in-catalog backup. The two readers never see each other's + /// rows. + pub(crate) store_kind: Option, + pub(crate) store_root: Option, + pub(crate) store_id: Option, } impl StoredBackup { @@ -229,9 +267,18 @@ impl StoredBackup { .map_err(|e| StorageError::Internal(format!("Parse created_at: {e}")))?, definition: row.get_by_name("definition").ok().flatten(), active_restores, + store_kind: row.get_by_name("store_kind").ok().flatten(), + store_root: row.get_by_name("store_root").ok().flatten(), + store_id: row.get_by_name("store_id").ok().flatten(), }) } + /// Whether this row belongs to the in-catalog (legacy) format rather than + /// the catalog index. + pub(crate) fn is_in_catalog(&self) -> bool { + self.store_kind.is_none() + } + fn details(&self) -> BackupDetails { BackupDetails { backup_arn: self.backup_arn.clone(), @@ -319,12 +366,13 @@ struct RestoreJob { backup_size_bytes: i64, key_schema: Vec, attribute_definitions: Vec, - owner: uuid::Uuid, + /// The claim's fencing token on the target row. + generation: u64, } enum CopyError { - /// The target is no longer this copy's (the sweep claimed it). Whoever - /// claimed it removes it. + /// The row is no longer this copy's (another owner claimed it). Whoever + /// claimed it owns what happens to it. OwnershipLost, Failed(StorageError), } @@ -341,20 +389,57 @@ impl Drop for AbortOnDrop { } } -/// The heartbeat of one restore target, running from the moment the target -/// row is inserted until this value is dropped. -pub(crate) struct RestoreHeartbeat { +/// The heartbeat of one job's row (a restore target or an in-catalog +/// backup being created), running from the moment the row is inserted until +/// this value is dropped. +/// +/// When a copy stops, in one place: a heartbeat that does not apply +/// (`Ok(false)`: another owner holds the row) stops the heartbeat and sets +/// [`Self::is_lost`]; so does the [`HEARTBEAT_ERRORS_TO_STOP`]th heartbeat +/// that errors in a row (three beats could not confirm the claim, which may +/// be another owner's by now); a heartbeat error before that is logged and +/// the next period tries again, so one error under a loaded node does not +/// stop a job. A beat that hangs is not an error and is not counted: the +/// loop waits on it, and the copy's own writes, on the same session, hang +/// with it; the beat that returns is conditioned on the row's generation, so +/// a row the reconciler fenced meanwhile sets [`Self::is_lost`] and the +/// copy's per-bucket poll stops it after at most one more bucket, its row +/// writes (activation, abandonment) conditioned on the generation too. The +/// copy polls [`Self::is_lost`] between units of work and, once it is set, +/// touches neither the row nor its data again. +pub(crate) struct JobHeartbeat { _task: AbortOnDrop, lost: Arc, } -impl RestoreHeartbeat { - /// Run `beat` now and then every `interval` until this value is dropped. - /// `beat` returns whether the target is still this process's; the first - /// `Ok(false)` sets [`Self::is_lost`] and stops the heartbeat. An error - /// is logged and the next tick tries again (the sweep, not one missed - /// beat, decides that a target is abandoned). - fn spawn(interval: Duration, target_name: String, mut beat: F) -> Self +impl JobHeartbeat { + /// The heartbeat of a job of `account_id`: [`Self::spawn_every`] at the + /// period the jobs heartbeat at (`JOB_HEARTBEAT_INTERVAL`, or the period + /// a test hook set for the account), the beats failing where a test hook + /// asks them to. + fn spawn(account_id: &str, label: String, mut beat: F) -> Self + where + F: FnMut() -> Fut + Send + 'static, + Fut: std::future::Future> + Send, + { + let interval = crate::test_hooks::heartbeat_interval(account_id); + let account_id = account_id.to_owned(); + Self::spawn_every(interval, label, move || { + let injected = crate::test_hooks::heartbeat_error(&account_id); + let real = beat(); + async move { + match injected { + Some(error) => Err(error), + None => real.await, + } + } + }) + } + + /// Run `beat` now and then every `interval` until this value is dropped + /// or the heartbeat stops as the type's doc says. `beat` returns whether + /// the row is still this job's. + fn spawn_every(interval: Duration, label: String, mut beat: F) -> Self where F: FnMut() -> Fut + Send + 'static, Fut: std::future::Future> + Send, @@ -364,16 +449,26 @@ impl RestoreHeartbeat { let task = tokio::spawn(async move { // The first tick completes immediately. let mut ticker = tokio::time::interval(interval); + let mut errors_in_a_row = 0_u32; loop { ticker.tick().await; match beat().await { - Ok(true) => {} + Ok(true) => errors_in_a_row = 0, Ok(false) => { flag.store(true, Ordering::Release); break; } Err(error) => { - tracing::warn!("Restore of table {target_name}: heartbeat failed: {error}") + errors_in_a_row += 1; + if errors_in_a_row >= HEARTBEAT_ERRORS_TO_STOP { + tracing::error!( + "{label}: heartbeat failed {errors_in_a_row} times in a row \ + ({error}); the claim is unconfirmed and the copy stops" + ); + flag.store(true, Ordering::Release); + break; + } + tracing::warn!("{label}: heartbeat failed: {error}"); } } } @@ -384,7 +479,9 @@ impl RestoreHeartbeat { } } - /// Whether a heartbeat found the target no longer owned by this process. + /// Whether the heartbeat stopped: a beat found the row no longer owned by + /// this job, or [`HEARTBEAT_ERRORS_TO_STOP`] beats in a row could not + /// confirm it. fn is_lost(&self) -> bool { self.lost.load(Ordering::Acquire) } @@ -395,17 +492,17 @@ struct HeartbeatTarget { account_id: String, target_name: String, table_id: String, - owner: uuid::Uuid, + generation: u64, } impl CassandraEngine { /// Refuse a backup operation on a catalog that has not run every catalog - /// migration this binary carries. Catalog migration V005 adds the columns - /// CreateBackup, DeleteBackup and RestoreTableFromBackup write; without - /// it they failed with an internal error (`Undefined column name ...`), - /// which SDKs retry. `Unsupported` reaches the client as - /// `ValidationException` with this message. Table, item, ListBackups and - /// DescribeBackup operations do not call this: they work on such a + /// migration this binary carries. Catalog migrations V005 and V006 add + /// the columns CreateBackup, DeleteBackup and RestoreTableFromBackup + /// write; without them they failed with an internal error (`Undefined + /// column name ...`), which SDKs retry. `Unsupported` reaches the client + /// as `ValidationException` with this message. Table, item, ListBackups + /// and DescribeBackup operations do not call this: they work on such a /// catalog. pub(crate) async fn require_migrated_catalog(&self) -> Result<(), StorageError> { use std::sync::atomic::Ordering; @@ -422,32 +519,41 @@ impl CassandraEngine { Err(unmigrated_catalog_refusal(&pending)) } - async fn load_backup( + /// The `backups_by_arn` row of either kind, read at quorum, if there is + /// one. + pub(crate) async fn load_stored_backup( &self, account_id: &str, backup_arn: &str, - ) -> Result { + ) -> Result, StorageError> { let query = format!( "SELECT * FROM {}.backups_by_arn WHERE account_id = ? AND backup_arn = ?", self.catalog_keyspace() ); - let result = self - .session - .query_with_values(&query, cdrs_tokio::query_values!(account_id, backup_arn)) - .await - .map_err(|e| StorageError::Internal(format!("Query backup: {e}")))?; - let rows = result - .response_body() - .map_err(|e| StorageError::Internal(format!("Parse backup response: {e}")))? - .into_rows() - .unwrap_or_default(); - let backup = rows - .first() + let rows = crate::cassandra_util::query_rows_quorum::( + &self.session, + &query, + cdrs_tokio::query_values!(account_id, backup_arn), + "backup", + ) + .await?; + rows.first() .map(StoredBackup::from_row) - .transpose()? - .filter(|backup| backup.account_id == account_id) - .ok_or_else(|| backup_not_found(backup_arn))?; - Ok(backup) + .transpose() + .map(|row| row.filter(|backup| backup.account_id == account_id)) + } + + /// The in-catalog backup `backup_arn` names; a catalog index row (one + /// with `store_kind` set) is not one, and is "not found" here. + async fn load_backup( + &self, + account_id: &str, + backup_arn: &str, + ) -> Result { + self.load_stored_backup(account_id, backup_arn) + .await? + .filter(StoredBackup::is_in_catalog) + .ok_or_else(|| backup_not_found(backup_arn)) } async fn load_available_backup( @@ -462,6 +568,11 @@ impl CassandraEngine { Ok(backup) } + /// Delete a backup's `backup_items` buckets: the `bucket_count(item_count)` + /// the row accounts for, then every further bucket until an empty one. + /// The create writes buckets in order and publishes `item_count` last, so + /// a create that died mid-copy leaves a row whose count is 0 and whose + /// buckets are contiguous from 0; the probe finds them all. async fn cleanup_backup_payload( &self, account_id: &str, @@ -469,18 +580,36 @@ impl CassandraEngine { item_count: i64, ) -> Result<(), StorageError> { let account_keyspace = self.account_keyspace(account_id); - let query = format!( + let delete = format!( "DELETE FROM {account_keyspace}.backup_items WHERE backup_arn = ? AND bucket = ?" ); - for bucket in 0..bucket_count(item_count) { + let probe = format!( + "SELECT item_index FROM {account_keyspace}.backup_items \ + WHERE backup_arn = ? AND bucket = ? LIMIT 1" + ); + let known = bucket_count(item_count); + let mut bucket = 0_i64; + loop { #[allow(clippy::cast_possible_truncation)] let bucket_i32 = bucket as i32; + if bucket >= known { + let rows = crate::cassandra_util::query_rows_quorum::( + &self.session, + &probe, + cdrs_tokio::query_values!(backup_arn, bucket_i32), + "probe backup payload", + ) + .await?; + if rows.is_empty() { + return Ok(()); + } + } self.session - .query_with_values(&query, cdrs_tokio::query_values!(backup_arn, bucket_i32)) + .query_with_values(&delete, cdrs_tokio::query_values!(backup_arn, bucket_i32)) .await .map_err(|e| StorageError::Internal(format!("Delete backup payload: {e}")))?; + bucket += 1; } - Ok(()) } async fn remove_backup_index_rows(&self, backup: &StoredBackup) -> Result<(), StorageError> { @@ -759,10 +888,15 @@ impl CassandraEngine { }) } - /// BackupInUse if a registered restore of `backup` is still copying; - /// otherwise remove every stale registration so it cannot block the - /// DELETING claim. - async fn ensure_backup_not_restoring(&self, backup: &StoredBackup) -> Result<(), StorageError> { + /// The name of a CREATING restore target registered on `backup` and still + /// being restored (the in-use rule: it reads the target's status, not the + /// registration alone); `None` when there is none. Every registration + /// whose target is no longer being restored is cleared on the way, so it + /// cannot hold the row's lock against a delete. + pub(crate) async fn creating_restore_target( + &self, + backup: &StoredBackup, + ) -> Result, StorageError> { for (table_id, target_name) in &backup.active_restores { if self .restore_target_in_progress( @@ -773,7 +907,7 @@ impl CassandraEngine { ) .await? { - return Err(backup_in_use(target_name, &backup.backup_arn)); + return Ok(Some(target_name.clone())); } tracing::info!( "Backup {}: clearing stale restore registration of table {target_name} \ @@ -783,7 +917,17 @@ impl CassandraEngine { self.unregister_restore(&backup.account_id, &backup.backup_arn, table_id) .await?; } - Ok(()) + Ok(None) + } + + /// BackupInUse naming the target if a registered restore of `backup` is + /// still copying; otherwise `Ok(())` with every stale registration + /// cleared. + async fn ensure_backup_not_restoring(&self, backup: &StoredBackup) -> Result<(), StorageError> { + match self.creating_restore_target(backup).await? { + Some(target_name) => Err(backup_in_use(&target_name, &backup.backup_arn)), + None => Ok(()), + } } // ── Restore copy (detached) ───────────────────────────────────────────── @@ -792,26 +936,26 @@ impl CassandraEngine { /// just been inserted. Called by the restore path of `create_table` right /// after the insert, before the target's data and index DDL, so the time /// that DDL takes (each statement waits for schema agreement) does not - /// count against [`RESTORE_STALE_AFTER`]. + /// count against the reconciler's stale window. pub(crate) fn start_restore_heartbeat( &self, account_id: &str, target_name: &str, table_id: &str, - owner: uuid::Uuid, - ) -> RestoreHeartbeat { + generation: u64, + ) -> JobHeartbeat { let state = Arc::new(( self.clone(), HeartbeatTarget { account_id: account_id.to_owned(), target_name: target_name.to_owned(), table_id: table_id.to_owned(), - owner, + generation, }, )); - RestoreHeartbeat::spawn( - RESTORE_HEARTBEAT_INTERVAL, - target_name.to_owned(), + JobHeartbeat::spawn( + account_id, + format!("Restore of table {target_name}"), move || { let state = Arc::clone(&state); async move { state.0.heartbeat_restore_target(&state.1).await } @@ -819,27 +963,34 @@ impl CassandraEngine { ) } + /// Refresh the target's `restore_heartbeat_at` on the coordinator's + /// clock, conditioned on the claim's generation; `false` when another + /// owner holds the target. async fn heartbeat_restore_target(&self, job: &HeartbeatTarget) -> Result { let query = format!( - "UPDATE {}.tables SET restore_heartbeat = ? WHERE account_id = ? AND table_name = ? \ - IF table_id = ? AND table_status = 'CREATING' AND restore_owner = ?", + "UPDATE {}.tables SET restore_heartbeat_at = toTimestamp(now()) \ + WHERE account_id = ? AND table_name = ? \ + IF table_id = ? AND table_status = 'CREATING' AND restore_owner_generation = ?", self.catalog_keyspace() ); let result = crate::cassandra_util::query_lwt( &self.session, &query, cdrs_tokio::query_values!( - crate::cassandra_util::now_millis(), job.account_id.as_str(), job.target_name.as_str(), job.table_id.as_str(), - job.owner + crate::backup_catalog::generation_column(job.generation) ), ) .await?; crate::cassandra_util::lwt_applied(&result) } + /// Flip the target ACTIVE and mark its job record DONE in one lightweight + /// transaction, conditioned on the table still being a CREATING target + /// whose record is RUNNING under this claim's generation. `false` when + /// another owner (the reconciler) claimed the target first. async fn activate_restore_target( &self, job: &RestoreJob, @@ -848,8 +999,11 @@ impl CassandraEngine { ) -> Result { let query = format!( "UPDATE {}.tables SET table_status = 'ACTIVE', status_transition_at = NULL, \ - item_count = ?, table_size_bytes = ? WHERE account_id = ? AND table_name = ? \ - IF table_id = ? AND table_status = 'CREATING' AND restore_owner = ?", + item_count = ?, table_size_bytes = ?, restore_status = 'DONE', \ + restore_completed_at = toTimestamp(now()) \ + WHERE account_id = ? AND table_name = ? \ + IF table_id = ? AND table_status = 'CREATING' AND restore_owner_generation = ? \ + AND restore_status = 'RUNNING'", self.catalog_keyspace() ); let result = crate::cassandra_util::query_lwt( @@ -861,7 +1015,7 @@ impl CassandraEngine { job.account_id.as_str(), job.target_name.as_str(), job.table_id.as_str(), - job.owner + crate::backup_catalog::generation_column(job.generation) ), ) .await?; @@ -870,9 +1024,9 @@ impl CassandraEngine { /// Remove a restore target this caller has claimed (or that never had a /// copy): unregister it from its backup first, so a crash between the two - /// steps leaves a target the sweep removes rather than a registration + /// steps leaves a target the reconciler removes rather than a registration /// that names nothing; then drop the incarnation with this `table_id`. - async fn remove_restore_target( + pub(crate) async fn drop_restore_target( &self, account_id: &str, target_name: &str, @@ -896,22 +1050,30 @@ impl CassandraEngine { } } - /// The copy failed while this process still owned the target: claim it - /// (CREATING -> DELETING, owner-fenced) and remove it. - async fn abandon_restore_target(&self, job: &RestoreJob) { + /// The copy failed while this process still owned the target: mark the + /// job record FAILED with the reason and claim the target for removal + /// (CREATING to DELETING) in one lightweight transaction conditioned on + /// the generation and on RUNNING, then remove it. A record another + /// generation owns means the claim holder owns the target, and nothing + /// further is touched. + async fn abandon_restore_target(&self, job: &RestoreJob, reason: &str) { let query = format!( - "UPDATE {}.tables SET table_status = 'DELETING' WHERE account_id = ? AND table_name = ? \ - IF table_id = ? AND table_status = 'CREATING' AND restore_owner = ?", + "UPDATE {}.tables SET table_status = 'DELETING', restore_status = 'FAILED', \ + restore_failure_reason = ?, restore_completed_at = toTimestamp(now()) \ + WHERE account_id = ? AND table_name = ? \ + IF table_id = ? AND table_status = 'CREATING' AND restore_owner_generation = ? \ + AND restore_status = 'RUNNING'", self.catalog_keyspace() ); let claimed = crate::cassandra_util::query_lwt( &self.session, &query, cdrs_tokio::query_values!( + reason, job.account_id.as_str(), job.target_name.as_str(), job.table_id.as_str(), - job.owner + crate::backup_catalog::generation_column(job.generation) ), ) .await @@ -929,8 +1091,8 @@ impl CassandraEngine { } Err(error) => { tracing::error!( - "Restore of table {} from {}: could not claim the failed target for \ - removal ({error}); the abandoned-restore sweep removes it", + "Restore of table {} from {}: could not mark the failed restore FAILED \ + ({error}); the reconciler removes the target once its heartbeat is stale", job.target_name, job.backup_arn ); @@ -938,7 +1100,7 @@ impl CassandraEngine { } } match self - .remove_restore_target( + .drop_restore_target( &job.account_id, &job.target_name, &job.table_id, @@ -953,7 +1115,7 @@ impl CassandraEngine { ), Err(error) => tracing::error!( "Restore of table {} from {}: removing the failed target failed ({error}); \ - the abandoned-restore sweep retries it", + the reconciler retries it", job.target_name, job.backup_arn ), @@ -963,7 +1125,7 @@ impl CassandraEngine { async fn copy_backup_into_target( &self, job: &RestoreJob, - heartbeat: &RestoreHeartbeat, + heartbeat: &JobHeartbeat, ) -> Result<(i64, i64), CopyError> { let key_info = TableKeyInfo { table_name: job.target_name.clone(), @@ -1044,7 +1206,7 @@ impl CassandraEngine { /// was started when the target row was inserted; it is dropped (which /// aborts it) when the copy ends, and on every other exit from this /// function, a panic or cancellation included. - async fn run_restore_copy(self, job: RestoreJob, heartbeat: RestoreHeartbeat) { + async fn run_restore_copy(self, job: RestoreJob, heartbeat: JobHeartbeat) { let outcome = self.copy_backup_into_target(&job, &heartbeat).await; drop(heartbeat); match outcome { @@ -1079,12 +1241,12 @@ impl CassandraEngine { job.target_name, job.backup_arn ); - self.abandon_restore_target(&job).await; + self.abandon_restore_target(&job, &error.to_string()).await; } }, Err(CopyError::OwnershipLost) => tracing::warn!( - "Restore of table {} from {}: stopped; the target was claimed by the \ - abandoned-restore sweep", + "Restore of table {} from {}: stopped; the claim on the target was lost or \ + could not be confirmed (see JobHeartbeat), and its owner removes it", job.target_name, job.backup_arn ), @@ -1094,157 +1256,51 @@ impl CassandraEngine { job.target_name, job.backup_arn ); - self.abandon_restore_target(&job).await; + self.abandon_restore_target(&job, &error.to_string()).await; } } } - // ── Abandoned-restore sweep ───────────────────────────────────────────── - - /// Remove restore targets whose copy is gone: a CREATING or DELETING - /// target carrying a restore registration whose heartbeat is older than - /// `stale_after` (or absent). Returns the names removed. - /// - /// Each removal first claims the target with a lightweight transaction - /// conditioned on the heartbeat it observed, setting DELETING, a fresh - /// heartbeat, and this engine as owner, so a copy that heartbeats in the - /// meantime wins and keeps its target, a copy that comes back later finds - /// itself no longer owner and stops, and two sweepers do not both remove - /// it. The servers' periodic worker calls this with - /// [`RESTORE_STALE_AFTER`]. - /// - /// # Errors - /// - /// [`StorageError`] if the catalog cannot be read; a failure to remove one - /// target is logged and left for the next pass. - pub async fn sweep_abandoned_restores( - &self, - stale_after: Duration, - ) -> Result, StorageError> { - let catalog = self.catalog_keyspace(); - #[allow(clippy::cast_possible_truncation, clippy::cast_possible_wrap)] - let cutoff = crate::cassandra_util::now_millis() - stale_after.as_millis() as i64; - let mut removed = Vec::new(); - for status in ["CREATING", "DELETING"] { - // Cluster-wide filtered scan, as the control-plane poller does for - // CREATING tables; restore targets are few and short-lived. - let query = format!( - "SELECT account_id, table_name, table_id, table_status, \ - restore_source_backup_arn, restore_heartbeat FROM {catalog}.tables \ - WHERE table_status = ? ALLOW FILTERING" - ); - let rows = crate::cassandra_util::query_rows::( - &self.session, - &query, - cdrs_tokio::query_values!(status), - "restore sweep", - ) - .await?; - for row in rows { - let backup_arn: Option = - row.get_by_name("restore_source_backup_arn").ok().flatten(); - let Some(backup_arn) = backup_arn else { - continue; - }; - let heartbeat: Option = row.get_by_name("restore_heartbeat").ok().flatten(); - if heartbeat.is_some_and(|at| at >= cutoff) { - continue; - } - let account_id: String = crate::cassandra_util::get_column::<_, StorageError>( - &row, - "account_id", - "restore sweep", - )?; - let table_name: String = crate::cassandra_util::get_column::<_, StorageError>( - &row, - "table_name", - "restore sweep", - )?; - let table_id: String = crate::cassandra_util::get_column::<_, StorageError>( - &row, - "table_id", - "restore sweep", - )?; - match self - .claim_abandoned_restore(&account_id, &table_name, &table_id, status, heartbeat) - .await - { - Ok(true) => {} - Ok(false) => continue, - Err(error) => { - tracing::warn!("Restore sweep: claiming table {table_name}: {error}"); - continue; - } - } - match self - .remove_restore_target(&account_id, &table_name, &table_id, &backup_arn) - .await - { - Ok(()) => { - tracing::info!( - "Restore sweep: removed abandoned restore target {table_name} \ - (account {account_id}, restoring from {backup_arn}); its copy \ - stopped heartbeating" - ); - removed.push(table_name); - } - Err(error) => tracing::warn!( - "Restore sweep: removing abandoned restore target {table_name}: {error}; \ - retrying on a later pass" - ), - } - } - } - Ok(removed) - } - - async fn claim_abandoned_restore( + /// Move a CREATING in-catalog row whose heartbeat was observed stale to + /// DELETING under a fresh `owner_generation`, conditioned on the status + /// and the heartbeat observed (a `null` heartbeat for a row from before + /// V006). The fresh generation fences the dead create's generation out: + /// a late heartbeat or publish of that create no longer applies. `heartbeat_at` + /// is left as it was, so the row stays stale for the pass that finishes + /// it. `false` when the row moved in between: its create beat, published, + /// or removed it. + async fn fence_abandoned_create( &self, account_id: &str, - table_name: &str, - table_id: &str, - observed_status: &str, + backup_arn: &str, observed_heartbeat: Option, ) -> Result { let catalog = self.catalog_keyspace(); - let now = crate::cassandra_util::now_millis(); + let generation = crate::backup_catalog::generation_column(new_generation()); let result = if let Some(heartbeat) = observed_heartbeat { let query = format!( - "UPDATE {catalog}.tables SET table_status = 'DELETING', restore_owner = ?, \ - restore_heartbeat = ? WHERE account_id = ? AND table_name = ? \ - IF table_id = ? AND table_status = ? AND restore_heartbeat = ?" + "UPDATE {catalog}.backups_by_arn SET backup_status = 'DELETING', \ + owner_generation = ? \ + WHERE account_id = ? AND backup_arn = ? \ + IF backup_status = 'CREATING' AND heartbeat_at = ?" ); crate::cassandra_util::query_lwt( &self.session, &query, - cdrs_tokio::query_values!( - self.restore_owner, - now, - account_id, - table_name, - table_id, - observed_status, - heartbeat - ), + cdrs_tokio::query_values!(generation, account_id, backup_arn, heartbeat), ) .await? } else { let query = format!( - "UPDATE {catalog}.tables SET table_status = 'DELETING', restore_owner = ?, \ - restore_heartbeat = ? WHERE account_id = ? AND table_name = ? \ - IF table_id = ? AND table_status = ? AND restore_heartbeat = null" + "UPDATE {catalog}.backups_by_arn SET backup_status = 'DELETING', \ + owner_generation = ? \ + WHERE account_id = ? AND backup_arn = ? \ + IF backup_status = 'CREATING' AND heartbeat_at = null" ); crate::cassandra_util::query_lwt( &self.session, &query, - cdrs_tokio::query_values!( - self.restore_owner, - now, - account_id, - table_name, - table_id, - observed_status - ), + cdrs_tokio::query_values!(generation, account_id, backup_arn), ) .await? }; @@ -1269,15 +1325,14 @@ impl CassandraEngine { /// copy starts from a backup DeleteBackup has claimed. What is not /// atomic: between the two steps a concurrent DescribeTable can see a /// CREATING target of a restore that then fails, and a process exit - /// between them leaves a target that the abandoned-restore sweep removes - /// once its heartbeat (refreshed from the insert until the process - /// exited) is older than - /// [`RESTORE_STALE_AFTER`]. + /// between them leaves a target that the reconciler removes once its + /// heartbeat (refreshed from the insert until the process exited) is + /// older than the stale window. async fn register_restore_target( &self, account_id: &str, request: RestoreRequest, - ) -> Result<(TableDescription, RestoreJob, RestoreHeartbeat), StorageError> { + ) -> Result<(TableDescription, RestoreJob, JobHeartbeat), StorageError> { let RestoreRequest { backup_arn, source_table_arn, @@ -1311,12 +1366,15 @@ impl CassandraEngine { let attribute_definitions = input.attribute_definitions.clone(); #[allow(clippy::cast_possible_truncation)] let restore_date_time_ms = (restore_date_time * 1_000.0).round() as i64; + // The claim is the row itself: a fresh generation, written by the + // IF NOT EXISTS insert, fences every later write of this copy. + let generation = new_generation(); let columns = RestoreTargetColumns { table_id: uuid::Uuid::new_v4().to_string(), source_backup_arn: backup_arn.clone(), source_table_arn, restore_date_time_ms, - owner: self.restore_owner, + owner_generation: generation, }; let table_id = columns.table_id.clone(); @@ -1335,12 +1393,12 @@ impl CassandraEngine { Err(error) => { // Part of the target may exist under our table_id. if let Err(cleanup) = self - .remove_restore_target(account_id, &target_name, &table_id, &backup_arn) + .drop_restore_target(account_id, &target_name, &table_id, &backup_arn) .await { tracing::warn!( "Restore of table {target_name}: removing the partly created \ - target failed ({cleanup}); the abandoned-restore sweep removes it" + target failed ({cleanup}); the reconciler removes it" ); } return Err(error); @@ -1370,12 +1428,12 @@ impl CassandraEngine { if !matches!(registered, Ok(true)) { drop(heartbeat); if let Err(cleanup) = self - .remove_restore_target(account_id, &target_name, &table_id, &backup_arn) + .drop_restore_target(account_id, &target_name, &table_id, &backup_arn) .await { tracing::warn!( "Restore of table {target_name}: removing the unregistered target \ - failed ({cleanup}); the abandoned-restore sweep removes it" + failed ({cleanup}); the reconciler removes it" ); } return Err(registration_exhausted(registered, &backup_arn)); @@ -1391,40 +1449,29 @@ impl CassandraEngine { backup_size_bytes: backup.backup_size_bytes, key_schema, attribute_definitions, - owner: self.restore_owner, + generation, }; Ok((description, job, heartbeat)) } } -/// The abandoned-restore sweep, for the server's worker set: one sweep at -/// startup, then one every [`RESTORE_SWEEP_INTERVAL`], until `shutdown` is -/// cancelled. A sweep already running when `shutdown` fires finishes first. -pub async fn poll_abandoned_restores( - engine: Arc, - shutdown: extenddb_storage::CancellationToken, -) { - loop { - if let Err(error) = engine.sweep_abandoned_restores(RESTORE_STALE_AFTER).await { - tracing::warn!("Abandoned-restore sweep failed: {error}"); - } - if !extenddb_storage::sleep_or_shutdown(&shutdown, RESTORE_SWEEP_INTERVAL).await { - return; - } - } -} - -impl BackupEngine for CassandraEngine { - fn create_backup( +impl LegacyBackupReader for CassandraEngine { + /// The in-catalog create: a CREATING `backups_by_arn` row owned by a + /// fresh generation and heartbeated while the items copy, the list rows, + /// then the AVAILABLE flip conditioned on the generation. The permit is + /// held until this future completes, which is when the copy ends. + fn create( &self, account_id: &str, table_name: &str, backup_name: &str, + permit: JobPermit, ) -> BoxFuture<'_, Result> { let account_id = account_id.to_owned(); let table_name = table_name.to_owned(); let backup_name = backup_name.to_owned(); Box::pin(async move { + let _permit = permit; Self::validate_account_id(&account_id)?; self.require_migrated_catalog().await?; @@ -1466,33 +1513,63 @@ impl BackupEngine for CassandraEngine { .transpose() .map_err(|e| StorageError::Internal(format!("Serialize stream: {e}")))?; + // The row is the claim: inserted CREATING under a fresh + // generation with a heartbeat on the coordinator's clock, and + // refreshed every JOB_HEARTBEAT_INTERVAL while the items copy. + let generation = new_generation(); let catalog = self.catalog_keyspace(); let insert_metadata = format!( - "INSERT INTO {catalog}.backups_by_arn (backup_arn, account_id, backup_name, table_id, table_name, table_arn, backup_status, backup_type, backup_size_bytes, item_count, key_schema, attribute_definitions, billing_mode, provisioned_throughput, stream_specification, table_created_at, created_at, definition) VALUES (?, ?, ?, ?, ?, ?, 'CREATING', 'USER', ?, 0, ?, ?, ?, ?, ?, ?, ?, ?) IF NOT EXISTS" + "INSERT INTO {catalog}.backups_by_arn (backup_arn, account_id, backup_name, table_id, table_name, table_arn, backup_status, backup_type, backup_size_bytes, item_count, key_schema, attribute_definitions, billing_mode, provisioned_throughput, stream_specification, table_created_at, created_at, definition, owner_generation, heartbeat_at) VALUES (?, ?, ?, ?, ?, ?, 'CREATING', 'USER', ?, 0, ?, ?, ?, ?, ?, ?, ?, ?, ?, toTimestamp(now())) IF NOT EXISTS" + ); + let inserted = crate::cassandra_util::query_lwt( + &self.session, + &insert_metadata, + cdrs_tokio::query_values!( + backup_arn.as_str(), + account_id.as_str(), + backup_name.as_str(), + table.table_id.as_str(), + table_name.as_str(), + table.table_arn.as_str(), + table.table_size_bytes, + key_schema.as_str(), + attribute_definitions.as_str(), + billing_mode, + provisioned_throughput.as_str(), + stream_specification.as_deref(), + table_created_at, + created_at, + definition_json.as_str(), + crate::backup_catalog::generation_column(generation) + ), + ) + .await + .map_err(|e| StorageError::Internal(format!("Create backup metadata: {e}")))?; + if !crate::cassandra_util::lwt_applied(&inserted)? { + return Err(StorageError::AlreadyExists(format!( + "backup {backup_arn} already has a row" + ))); + } + let heartbeat = { + let engine = self.clone(); + let (account, arn) = (account_id.clone(), backup_arn.clone()); + JobHeartbeat::spawn(&account_id, format!("Backup {backup_arn}"), move || { + let engine = engine.clone(); + let (account, arn) = (account.clone(), arn.clone()); + async move { + crate::backup_catalog::heartbeat_backup_row( + &engine, &account, &arn, generation, + ) + .await + } + }) + }; + // The row's generation-fenced removal, for a copy that fails + // while this process still owns the row. + let delete_metadata = format!( + "DELETE FROM {catalog}.backups_by_arn WHERE account_id = ? AND backup_arn = ? \ + IF owner_generation = ? AND backup_status = 'CREATING'" ); - self.session - .query_with_values( - &insert_metadata, - cdrs_tokio::query_values!( - backup_arn.as_str(), - account_id.as_str(), - backup_name.as_str(), - table.table_id.as_str(), - table_name.as_str(), - table.table_arn.as_str(), - table.table_size_bytes, - key_schema.as_str(), - attribute_definitions.as_str(), - billing_mode, - provisioned_throughput.as_str(), - stream_specification.as_deref(), - table_created_at, - created_at, - definition_json.as_str() - ), - ) - .await - .map_err(|e| StorageError::Internal(format!("Create backup metadata: {e}")))?; let account_keyspace = self.account_keyspace(&account_id); let insert_item = format!( @@ -1502,7 +1579,7 @@ impl BackupEngine for CassandraEngine { let mut payload_size = 0_i64; let mut start_key = None; - let snapshot_result: Result<(), StorageError> = async { + let snapshot_result: Result<(), CopyError> = async { loop { let (items, next_key) = self .scan_impl( @@ -1513,12 +1590,22 @@ impl BackupEngine for CassandraEngine { None, None, ) - .await?; + .await + .map_err(CopyError::Failed)?; for item in items { let item_data = serde_json::to_string(&item).map_err(|e| { - StorageError::Internal(format!("Serialize backup item: {e}")) + CopyError::Failed(StorageError::Internal(format!( + "Serialize backup item: {e}" + ))) })?; let bucket = item_count / BACKUP_ITEMS_PER_BUCKET; + // A bucket, not a page, is the unit of work between + // two polls: a copy that lost its row writes at most + // one bucket after the loss, into a payload whose + // owner is deleting bucket by bucket. + if item_count % BACKUP_ITEMS_PER_BUCKET == 0 && heartbeat.is_lost() { + return Err(CopyError::OwnershipLost); + } #[allow(clippy::cast_possible_truncation)] let bucket_i32 = bucket as i32; self.session @@ -1533,7 +1620,9 @@ impl BackupEngine for CassandraEngine { ) .await .map_err(|e| { - StorageError::Internal(format!("Write backup item: {e}")) + CopyError::Failed(StorageError::Internal(format!( + "Write backup item: {e}" + ))) })?; item_count += 1; #[allow(clippy::cast_possible_wrap)] @@ -1549,21 +1638,32 @@ impl BackupEngine for CassandraEngine { } .await; - if let Err(error) = snapshot_result { - let _ = self - .cleanup_backup_payload(&account_id, &backup_arn, item_count) + match snapshot_result { + Ok(()) => {} + Err(CopyError::OwnershipLost) => { + // The row is no longer this copy's to touch (see + // `claim_lost`); whoever owns it removes it and the + // payload. + drop(heartbeat); + return Err(claim_lost(&backup_arn)); + } + Err(CopyError::Failed(error)) => { + drop(heartbeat); + let _ = self + .cleanup_backup_payload(&account_id, &backup_arn, item_count) + .await; + let _ = crate::cassandra_util::query_lwt( + &self.session, + &delete_metadata, + cdrs_tokio::query_values!( + account_id.as_str(), + backup_arn.as_str(), + crate::backup_catalog::generation_column(generation) + ), + ) .await; - let delete_metadata = format!( - "DELETE FROM {catalog}.backups_by_arn \ - WHERE account_id = ? AND backup_arn = ? IF EXISTS" - ); - let _ = crate::cassandra_util::query_lwt( - &self.session, - &delete_metadata, - cdrs_tokio::query_values!(account_id.as_str(), backup_arn.as_str()), - ) - .await; - return Err(error); + return Err(error); + } } let backup_size_bytes = table.table_size_bytes.max(payload_size); @@ -1573,13 +1673,14 @@ impl BackupEngine for CassandraEngine { let insert_by_table = format!( "INSERT INTO {catalog}.backups_by_table (account_id, table_name, created_at, backup_arn, backup_name, table_id, table_arn, backup_size_bytes) VALUES (?, ?, ?, ?, ?, ?, ?, ?)" ); - let publish = format!( - "UPDATE {catalog}.backups_by_arn SET backup_status = 'AVAILABLE', backup_size_bytes = ?, item_count = ? WHERE account_id = ? AND backup_arn = ?" - ); - // Publication is a fixed three-statement operation. A logged batch - // prevents readers from observing list rows without the matching - // AVAILABLE authoritative row, or vice versa. - let publish_result: Result<(), StorageError> = async { + // The AVAILABLE flip is conditioned on the generation, so it is a + // lightweight transaction and cannot share a batch with the list + // rows (another partition). The list rows go first: a process + // that dies between the two leaves list rows naming a CREATING + // row, which `list` hides and the reconciler removes with the row + // once its heartbeat is stale, rather than an AVAILABLE backup no + // listing ever shows. + let list_rows: Result<(), StorageError> = async { let batch = BatchQueryBuilder::new() .with_consistency(Consistency::LocalQuorum) .add_query( @@ -1607,15 +1708,6 @@ impl BackupEngine for CassandraEngine { table.table_arn.as_str(), backup_size_bytes ), - ) - .add_query( - publish, - cdrs_tokio::query_values!( - backup_size_bytes, - item_count, - account_id.as_str(), - backup_arn.as_str() - ), ); self.session .batch( @@ -1628,25 +1720,60 @@ impl BackupEngine for CassandraEngine { Ok(()) } .await; - - if let Err(error) = publish_result { - if let Ok(backup) = self.load_backup(&account_id, &backup_arn).await { - let _ = self.remove_backup_index_rows(&backup).await; - } - let _ = self - .cleanup_backup_payload(&account_id, &backup_arn, item_count) - .await; - let delete_metadata = format!( - "DELETE FROM {catalog}.backups_by_arn \ - WHERE account_id = ? AND backup_arn = ? IF EXISTS" + let publish_result: Result<(), CopyError> = async { + list_rows.map_err(CopyError::Failed)?; + let publish = format!( + "UPDATE {catalog}.backups_by_arn SET backup_status = 'AVAILABLE', \ + backup_size_bytes = ?, item_count = ? WHERE account_id = ? AND backup_arn = ? \ + IF owner_generation = ? AND backup_status = 'CREATING'" ); - let _ = crate::cassandra_util::query_lwt( + let published = crate::cassandra_util::query_lwt( &self.session, - &delete_metadata, - cdrs_tokio::query_values!(account_id.as_str(), backup_arn.as_str()), + &publish, + cdrs_tokio::query_values!( + backup_size_bytes, + item_count, + account_id.as_str(), + backup_arn.as_str(), + crate::backup_catalog::generation_column(generation) + ), ) - .await; - return Err(error); + .await + .and_then(|result| crate::cassandra_util::lwt_applied(&result)) + .map_err(|e| { + CopyError::Failed(StorageError::Internal(format!("Publish backup: {e}"))) + })?; + if published { + Ok(()) + } else { + Err(CopyError::OwnershipLost) + } + } + .await; + drop(heartbeat); + + match publish_result { + Ok(()) => {} + Err(CopyError::OwnershipLost) => return Err(claim_lost(&backup_arn)), + Err(CopyError::Failed(error)) => { + if let Ok(backup) = self.load_backup(&account_id, &backup_arn).await { + let _ = self.remove_backup_index_rows(&backup).await; + } + let _ = self + .cleanup_backup_payload(&account_id, &backup_arn, item_count) + .await; + let _ = crate::cassandra_util::query_lwt( + &self.session, + &delete_metadata, + cdrs_tokio::query_values!( + account_id.as_str(), + backup_arn.as_str(), + crate::backup_catalog::generation_column(generation) + ), + ) + .await; + return Err(error); + } } Ok(BackupDetails { @@ -1660,7 +1787,7 @@ impl BackupEngine for CassandraEngine { }) } - fn describe_backup( + fn describe( &self, account_id: &str, backup_arn: &str, @@ -1674,7 +1801,7 @@ impl BackupEngine for CassandraEngine { }) } - fn list_backups( + fn list( &self, account_id: &str, table_name: Option<&str>, @@ -1727,7 +1854,7 @@ impl BackupEngine for CassandraEngine { }) } - fn delete_backup( + fn delete( &self, account_id: &str, backup_arn: &str, @@ -1747,9 +1874,13 @@ impl BackupEngine for CassandraEngine { let original_description = backup.description()?; // The claim is conditioned on the registration map being empty, on // the same partition the restore registers on (see the module - // docs), so a registered restore keeps the backup AVAILABLE. + // docs), so a registered restore keeps the backup AVAILABLE. The + // heartbeat is stamped with it, so a deletion this process does + // not finish is one `abandoned_creating_backups` reports once the + // stamp is stale, and a running one is left alone. let mark_deleting = format!( - "UPDATE {}.backups_by_arn SET backup_status = 'DELETING' \ + "UPDATE {}.backups_by_arn SET backup_status = 'DELETING', \ + heartbeat_at = toTimestamp(now()) \ WHERE account_id = ? AND backup_arn = ? \ IF backup_status = 'AVAILABLE' AND active_restores = null", self.catalog_keyspace() @@ -1757,6 +1888,7 @@ impl BackupEngine for CassandraEngine { let mut attempt = 0; while backup.backup_status == "AVAILABLE" { self.ensure_backup_not_restoring(&backup).await?; + crate::test_hooks::before_delete_claim(&backup_arn).await; let result = crate::cassandra_util::query_lwt( &self.session, &mark_deleting, @@ -1816,7 +1948,7 @@ impl BackupEngine for CassandraEngine { }) } - fn backup_restore_source( + fn restore_source( &self, account_id: &str, backup_arn: &str, @@ -1867,25 +1999,29 @@ impl BackupEngine for CassandraEngine { }) } - fn restore_table_from_backup( + fn restore( &self, account_id: &str, request: RestoreRequest, + permit: JobPermit, ) -> BoxFuture<'_, Result> { let account_id = account_id.to_owned(); Box::pin(async move { Self::validate_account_id(&account_id)?; // Registration and copy both run in one detached task that owns a - // clone of the engine and the request. The request future only - // waits for the registration outcome, so dropping it (a client - // timeout) at any await point after the first poll does not stop - // the restore; before the first poll nothing has been done. - // Contrast: if registration ran here and only the copy were + // clone of the engine, the request and the job permit (released + // when the task ends, which is when the copy ends). The request + // future only waits for the registration outcome, so dropping it + // (a client timeout) at any await point after the first poll does + // not stop the restore; before the first poll nothing has been + // done. Contrast: if registration ran here and only the copy were // spawned, a drop between creating the target and spawning would - // leave a CREATING target with no copy until the sweep removed it. + // leave a CREATING target with no copy until the reconciler + // removed it. let (registered_tx, registered_rx) = tokio::sync::oneshot::channel(); let engine = self.clone(); tokio::spawn(async move { + let _permit = permit; // Inside the task, like the registration, so the first poll // still spawns before any await. if let Err(error) = engine.require_migrated_catalog().await { @@ -1910,6 +2046,74 @@ impl BackupEngine for CassandraEngine { }) } + /// In-catalog rows whose owner is dead, for the reconciler's `delete`: + /// a DELETING row whose `heartbeat_at` is older than `stale_after` on the + /// coordinator's clock, a deletion its process did not finish; and a + /// CREATING row whose stamp is that old (or unset, a row from before the + /// heartbeat existed), fenced to DELETING here with a lightweight + /// transaction conditioned on the status and heartbeat observed and + /// setting a fresh `owner_generation`, so a create that beats in between + /// keeps its row and one that comes back later finds its claim lost at + /// its next heartbeat. DELETING is scanned first, so a row this pass + /// fences is reported once. `delete` finishes a DELETING row and refuses + /// a CREATING one, so a client's DeleteBackup of a backup still being + /// created stays "not found" while the reconciler's goes through. + fn abandoned_creating_backups( + &self, + stale_after: Duration, + ) -> BoxFuture<'_, Result, StorageError>> { + Box::pin(async move { + let catalog = self.catalog_keyspace(); + let now = crate::backup_catalog::coordinator_now_millis(self).await?; + #[allow(clippy::cast_possible_truncation, clippy::cast_possible_wrap)] + let cutoff = now - stale_after.as_millis() as i64; + let mut abandoned = Vec::new(); + for status in ["DELETING", "CREATING"] { + // A filtered scan of the backup rows: in-catalog backups + // mid-create or mid-delete are few and short-lived. + let query = format!( + "SELECT account_id, backup_arn, backup_status, store_kind, owner_generation, \ + heartbeat_at FROM {catalog}.backups_by_arn \ + WHERE backup_status = ? ALLOW FILTERING" + ); + let rows = crate::cassandra_util::query_rows_quorum::( + &self.session, + &query, + cdrs_tokio::query_values!(status), + "abandoned backups", + ) + .await?; + for row in rows { + let store_kind: Option = row.get_by_name("store_kind").ok().flatten(); + if store_kind.is_some() { + continue; + } + let heartbeat: Option = row.get_by_name("heartbeat_at").ok().flatten(); + if heartbeat.is_some_and(|at| at >= cutoff) { + continue; + } + let account_id: String = + crate::cassandra_util::get_column(&row, "account_id", "abandoned backups")?; + let backup_arn: String = + crate::cassandra_util::get_column(&row, "backup_arn", "abandoned backups")?; + if status == "CREATING" { + let fenced = self + .fence_abandoned_create(&account_id, &backup_arn, heartbeat) + .await?; + if !fenced { + continue; + } + } + abandoned.push(AbandonedLegacyBackup { + account_id, + backup_arn, + }); + } + } + Ok(abandoned) + }) + } + fn describe_continuous_backups( &self, account_id: &str, @@ -2017,11 +2221,6 @@ mod tests { assert_eq!(bucket_count(BACKUP_ITEMS_PER_BUCKET + 1), 2); } - #[test] - fn a_restore_is_declared_abandoned_only_after_missed_heartbeats() { - assert!(RESTORE_STALE_AFTER >= RESTORE_HEARTBEAT_INTERVAL * 4); - } - /// Registration attempts that all lost to a restore just starting end in /// `BackupInUse` naming that, not in a missing backup; an error from the /// attempts themselves is reported as it is. @@ -2045,14 +2244,39 @@ mod tests { use std::sync::atomic::AtomicUsize; /// A heartbeat that counts its beats, every 10 ms, answering `owned`. - fn counting_heartbeat(beats: &Arc, owned: bool) -> RestoreHeartbeat { + fn counting_heartbeat(beats: &Arc, owned: bool) -> JobHeartbeat { let beats = Arc::clone(beats); - RestoreHeartbeat::spawn(Duration::from_millis(10), "T".to_owned(), move || { + JobHeartbeat::spawn_every(Duration::from_millis(10), "T".to_owned(), move || { beats.fetch_add(1, Ordering::SeqCst); async move { Ok(owned) } }) } + /// A heartbeat every 10 ms whose first `errors` beats fail and whose + /// later beats confirm the claim. + fn erring_heartbeat(beats: &Arc, errors: usize) -> JobHeartbeat { + let beats = Arc::clone(beats); + JobHeartbeat::spawn_every(Duration::from_millis(10), "T".to_owned(), move || { + let beat = beats.fetch_add(1, Ordering::SeqCst); + async move { + if beat < errors { + Err(StorageError::Internal("node unreachable".to_owned())) + } else { + Ok(true) + } + } + }) + } + + /// Wait until the heartbeat reports the claim lost; panic after 5 s. + async fn wait_for_loss(heartbeat: &JobHeartbeat) { + let deadline = std::time::Instant::now() + Duration::from_secs(5); + while !heartbeat.is_lost() { + assert!(std::time::Instant::now() < deadline, "loss never reported"); + tokio::time::sleep(Duration::from_millis(5)).await; + } + } + /// Wait until `beats` reaches `at_least`; panic after 5 s. async fn wait_for_beats(beats: &AtomicUsize, at_least: usize) { let deadline = std::time::Instant::now() + Duration::from_secs(5); @@ -2121,12 +2345,34 @@ mod tests { let beats = Arc::new(AtomicUsize::new(0)); let heartbeat = counting_heartbeat(&beats, false); wait_for_beats(&beats, 1).await; - let deadline = std::time::Instant::now() + Duration::from_secs(5); - while !heartbeat.is_lost() { - assert!(std::time::Instant::now() < deadline, "loss never reported"); - tokio::time::sleep(Duration::from_millis(5)).await; - } + wait_for_loss(&heartbeat).await; assert_stopped(&beats).await; assert_eq!(beats.load(Ordering::SeqCst), 1, "one beat, then stopped"); } + + /// `HEARTBEAT_ERRORS_TO_STOP` errors in a row stop the heartbeat and + /// report the claim lost; fewer, followed by a confirmed beat, do not, + /// and the heartbeat keeps running. + #[tokio::test] + async fn three_heartbeat_errors_in_a_row_stop_the_job_and_two_do_not() { + let errors = usize::try_from(HEARTBEAT_ERRORS_TO_STOP).unwrap(); + + let beats = Arc::new(AtomicUsize::new(0)); + let heartbeat = erring_heartbeat(&beats, errors); + wait_for_loss(&heartbeat).await; + assert_stopped(&beats).await; + assert_eq!( + beats.load(Ordering::SeqCst), + errors, + "the third error in a row stopped the heartbeat" + ); + + let beats = Arc::new(AtomicUsize::new(0)); + let heartbeat = erring_heartbeat(&beats, errors - 1); + wait_for_beats(&beats, errors + 5).await; + assert!( + !heartbeat.is_lost(), + "two errors then a confirmed beat do not stop the job" + ); + } } diff --git a/crates/storage-cassandra/src/create_table.rs b/crates/storage-cassandra/src/create_table.rs index 9bf36d3ad..904b4ff83 100644 --- a/crates/storage-cassandra/src/create_table.rs +++ b/crates/storage-cassandra/src/create_table.rs @@ -13,7 +13,7 @@ use extenddb_storage::error::StorageError; use extenddb_storage::util::{index_arn, stream_arn, table_arn}; use crate::CassandraEngine; -use crate::backup_engine::RestoreHeartbeat; +use crate::backup_engine::JobHeartbeat; impl CassandraEngine { /// Initialize stream shards for a table atomically. @@ -101,8 +101,11 @@ impl CassandraEngine { } /// Create a restore target: CREATING, with no generic transition deadline, - /// carrying the restore registration columns. Only the restore copy (which - /// activates it) or the abandoned-restore sweep (which removes it) moves it + /// carrying the restore registration and its job record (RUNNING, kind + /// legacy, the claim's generation, a fresh heartbeat) in the one + /// `INSERT ... IF NOT EXISTS`, so no crash leaves a CREATING target + /// without a job record. Only the restore copy (which activates it) or + /// the reconciler (which removes it once its heartbeat is stale) moves it /// out of CREATING. Returns the target's heartbeat, started right after /// the row insert; dropping it stops it. pub(crate) async fn create_table_for_restore_impl( @@ -110,7 +113,7 @@ impl CassandraEngine { account_id: &str, input: CreateTableInput, restore: &RestoreTargetColumns, - ) -> Result<(TableDescription, RestoreHeartbeat), StorageError> { + ) -> Result<(TableDescription, JobHeartbeat), StorageError> { let (description, heartbeat) = self .create_table_impl_inner(account_id, input, Some(restore)) .await?; @@ -125,7 +128,7 @@ impl CassandraEngine { account_id: &str, input: CreateTableInput, restore: Option<&RestoreTargetColumns>, - ) -> Result<(TableDescription, Option), StorageError> { + ) -> Result<(TableDescription, Option), StorageError> { let schedule_activation = restore.is_none(); Self::validate_account_id(account_id)?; @@ -220,10 +223,10 @@ impl CassandraEngine { } // Insert table metadata with LWT (IF NOT EXISTS). Only a restore - // target names the restore columns (added by catalog migration V005), - // so an ordinary CreateTable also works on a catalog that has not run - // V005; CQL rejects a statement naming an undefined column even when - // no value is bound to it. + // target names the restore columns (added by catalog migrations V005 + // and V006), so an ordinary CreateTable also works on a catalog that + // has not run them; CQL rejects a statement naming an undefined column + // even when no value is bound to it. let mut columns = String::from( "account_id, table_name, table_id, table_arn, key_schema, \ attribute_definitions, billing_mode, provisioned_throughput, stream_specification, \ @@ -263,20 +266,26 @@ impl CassandraEngine { Value::from(deletion_protection), Value::from(status_transition_at), ]; + let mut placeholders = vec!["?"; values.len()]; if let Some(r) = restore { + // The job record: RUNNING under the claim's generation, kind + // legacy, the heartbeat stamped on the coordinator's clock. columns.push_str( ", restore_source_backup_arn, restore_source_table_arn, restore_date_time, \ - restore_owner, restore_heartbeat", + restore_status, restore_kind, restore_owner_generation, restore_heartbeat_at", ); values.extend([ Value::from(r.source_backup_arn.as_str()), Value::from(r.source_table_arn.as_str()), Value::from(r.restore_date_time_ms), - Value::from(r.owner), - Value::from(creation_timestamp), + Value::from("RUNNING"), + Value::from("legacy"), + Value::from(crate::backup_catalog::generation_column(r.owner_generation)), ]); + placeholders.extend(["?"; 6]); + placeholders.push("toTimestamp(now())"); } - let placeholders = vec!["?"; values.len()].join(", "); + let placeholders = placeholders.join(", "); let insert_table_cql = format!( "INSERT INTO {catalog_keyspace}.tables ({columns}) VALUES ({placeholders}) IF NOT EXISTS" ); @@ -331,7 +340,12 @@ impl CassandraEngine { // A restore target's heartbeat runs from here, before the DDL below. // An error return from this function drops it, which stops it. let heartbeat = restore.map(|r| { - self.start_restore_heartbeat(account_id, &input.table_name, &r.table_id, r.owner) + self.start_restore_heartbeat( + account_id, + &input.table_name, + &r.table_id, + r.owner_generation, + ) }); // Insert GSI metadata @@ -658,6 +672,7 @@ pub(crate) struct RestoreTargetColumns { pub(crate) source_table_arn: String, /// `RestoreSummary.RestoreDateTime`, epoch milliseconds. pub(crate) restore_date_time_ms: i64, - /// The engine instance running the copy; see `CassandraEngine::restore_owner`. - pub(crate) owner: uuid::Uuid, + /// The claim's fencing token (`tables.restore_owner_generation`): every + /// write the copy makes to the target row is conditioned on it. + pub(crate) owner_generation: u64, } diff --git a/crates/storage-cassandra/src/delete_table.rs b/crates/storage-cassandra/src/delete_table.rs index b1a5b426e..d280a2557 100644 --- a/crates/storage-cassandra/src/delete_table.rs +++ b/crates/storage-cassandra/src/delete_table.rs @@ -15,8 +15,7 @@ pub(crate) enum DeleteTableCaller<'a> { /// DeleteTable: refuses a deletion-protected table and a restore target /// still being restored. Client, - /// The restore copy or the abandoned-restore sweep removing a restore - /// target. Deletes only the incarnation with this `table_id`, so a target + /// The restore copy or the reconciler removing a restore target. Deletes only the incarnation with this `table_id`, so a target /// that was already removed and whose name was reused is left alone. RestoreCleanup { table_id: &'a str }, } @@ -92,7 +91,7 @@ impl CassandraEngine { } // A restore target is owned by its copy until the copy - // activates it (or the sweep removes it). Deleting it mid-copy + // activates it (or the reconciler removes it). Deleting it mid-copy // would race the copy's writes into dropped tables. let status: Option = table_row.get_by_name("table_status").ok().flatten(); let restoring_from: Option = table_row diff --git a/crates/storage-cassandra/src/engine.rs b/crates/storage-cassandra/src/engine.rs index d897602e7..f4dab2df6 100644 --- a/crates/storage-cassandra/src/engine.rs +++ b/crates/storage-cassandra/src/engine.rs @@ -97,11 +97,6 @@ pub struct CassandraEngine { pub(crate) ttl_repair_scan_cursors: Arc>>, - /// Identifies this engine instance as the owner of the restore copies it - /// runs (`tables.restore_owner`). Random per process start, so a restore - /// left behind by a crashed process is never mistaken for a live one. - pub(crate) restore_owner: uuid::Uuid, - /// Set once `schema_history` was seen to record every catalog migration /// this binary carries; until then each CreateBackup, DeleteBackup and /// RestoreTableFromBackup re-reads it (see `require_migrated_catalog`), so @@ -150,7 +145,6 @@ impl CassandraEngine { ttl_repair_scan_cursors: Arc::new(std::sync::Mutex::new( std::collections::HashMap::new(), )), - restore_owner: uuid::Uuid::new_v4(), catalog_migrated: Arc::new(std::sync::atomic::AtomicBool::new(false)), }) } @@ -235,6 +229,59 @@ impl CassandraEngine { format!("{}_catalog", self.keyspace_prefix) } + /// `serve`'s catalog guard: the catalog's `catalog_version` row must be + /// [`crate::CATALOG_VERSION`], else the server does not start and the + /// refusal names `extenddb migrate --yes`. The rule is + /// [`extenddb_storage::util::require_served_catalog_version`], shared + /// with the MongoDB factory; the PostgreSQL and SQLite factories read + /// their ledgers for the same refusal. A catalog whose `settings` table + /// does not exist is not a catalog (`Uninitialized`, with `extenddb + /// init` advice). `require_migrated_catalog` stays beside this for an + /// engine built without the factory (the tests) and refuses a backup + /// operation on a catalog behind the binary with the file it lacks. + /// + /// # Errors + /// + /// A failed read as `Store`; a version other than this binary's as the + /// refusal `serve` prints. + pub async fn require_served_catalog( + &self, + ) -> Result<(), extenddb_storage::migrations::MigrationError> { + use extenddb_storage::migrations::MigrationError; + let keyspace = self.catalog_keyspace(); + let settings_exists = crate::migrations::table_exists(&self.session, &keyspace, "settings") + .await + .map_err(|e| MigrationError::Store(format!("{e:?}")))?; + let stored = if settings_exists { + let rows = crate::cassandra_util::query_rows::( + &self.session, + &format!("SELECT value FROM {keyspace}.settings WHERE key = 'catalog_version'"), + query_values!(), + "catalog version", + ) + .await + .map_err(|e| MigrationError::Store(e.to_string()))?; + match rows.first() { + Some(row) => Some( + crate::cassandra_util::get_column::( + row, + "value", + "catalog version", + ) + .map_err(|e| MigrationError::Store(e.to_string()))?, + ), + None => None, + } + } else { + None + }; + extenddb_storage::util::require_served_catalog_version( + stored.as_deref(), + crate::CATALOG_VERSION, + "Cassandra", + ) + } + /// Get the account keyspace name for a given account ID. #[must_use] pub fn account_keyspace(&self, account_id: &str) -> String { diff --git a/crates/storage-cassandra/src/lib.rs b/crates/storage-cassandra/src/lib.rs index e21bf181e..33c4c8bdb 100644 --- a/crates/storage-cassandra/src/lib.rs +++ b/crates/storage-cassandra/src/lib.rs @@ -7,7 +7,7 @@ //! as the underlying database. It provides a DynamoDB-compatible API backed //! by Cassandra's distributed architecture. -pub const CATALOG_VERSION: &str = "0.0.4"; +pub const CATALOG_VERSION: &str = "0.0.5"; /// Default control-plane delay (seconds) when the `control_plane_delay_seconds` /// setting is absent or unparseable: CreateTable, the control-plane worker @@ -16,6 +16,7 @@ pub(crate) const DEFAULT_CONTROL_PLANE_DELAY_SECONDS: f64 = 0.25; mod admin_store; mod authorization_store; +mod backup_catalog; mod backup_engine; pub mod bootstrapper; pub mod cassandra_util; @@ -37,12 +38,14 @@ mod stream_engine; pub mod stream_util; pub mod table_engine; mod table_helpers; +#[doc(hidden)] +pub mod test_hooks; pub mod ttl_worker; mod update_table; mod worker_store; pub mod workers; -pub use backup_engine::poll_abandoned_restores; +pub use backup_catalog::heartbeat_backup_row; pub use bootstrapper::CassandraBootstrapper; pub use catalog_store::CassandraCatalogStore; pub use config::CassandraStorageConfig; @@ -162,16 +165,6 @@ impl ServerRuntimeHooks for CassandraRuntimeHooks { .await }); - // Removes restore targets whose copy stopped heartbeating (the - // process running it died): once now, then periodically until - // shutdown. - let restore_sweep_engine = self.engine.clone(); - let restore_sweep_shutdown = ctx.shutdown.clone(); - let restore_sweep = tokio::spawn(async move { - backup_engine::poll_abandoned_restores(restore_sweep_engine, restore_sweep_shutdown) - .await; - }); - vec![ control_plane, transaction_recovery, @@ -180,7 +173,6 @@ impl ServerRuntimeHooks for CassandraRuntimeHooks { ttl_cleanup, ttl_repair, ttl_audit, - restore_sweep, ] } @@ -330,6 +322,13 @@ fn server_components_factory( details: e.to_string(), })?; + // The catalog's version is this binary's, or `serve` does not start: + // the refusal names `extenddb migrate --yes`, as on the ledger backends. + engine + .require_served_catalog() + .await + .map_err(BackendError::Catalog)?; + let control_plane_notify = engine.control_plane_notify.clone(); let engine = Arc::new(engine); diff --git a/crates/storage-cassandra/src/migrations.rs b/crates/storage-cassandra/src/migrations.rs index 8eabe63b4..528d14058 100644 --- a/crates/storage-cassandra/src/migrations.rs +++ b/crates/storage-cassandra/src/migrations.rs @@ -33,6 +33,9 @@ //! 5. Name tables without a keyspace. The runner writes the keyspace it //! migrates into each `CREATE TABLE`, `CREATE INDEX`, `ALTER TABLE` and //! `INSERT INTO` statement, and rejects any other statement form. +//! 6. Do not write `settings.catalog_version` in the file: the runner writes +//! [`crate::CATALOG_VERSION`] once every file is recorded +//! (`write_catalog_version`). Bump that constant with the file. use cdrs_tokio::cluster::TcpConnectionManager; use cdrs_tokio::cluster::session::Session; @@ -87,8 +90,20 @@ pub(crate) const CATALOG_MIGRATIONS: &[(&str, &str)] = &[ "V005__backup_definitions_and_restore_registration.cql", include_str!("../migrations/catalog/V005__backup_definitions_and_restore_registration.cql"), ), + ( + BACKUP_CATALOG_INDEX_MIGRATION, + include_str!("../migrations/catalog/V006__backup_catalog_index.cql"), + ), ]; +/// The catalog migration that adds the backup catalog index. Its statements +/// are followed by a step in Rust ([`backfill_backup_catalog_index`]): CQL +/// has no `UPDATE ... WHERE` on a non-key predicate, so the restore +/// registrations that exist are backfilled row by row, and the catalog id is +/// a random value the runner generates. The version is written after the +/// step and the `schema_history` row ([`write_catalog_version`]). +pub(crate) const BACKUP_CATALOG_INDEX_MIGRATION: &str = "V006__backup_catalog_index.cql"; + /// Embedded data migration files, applied in order. pub(crate) const DATA_MIGRATIONS: &[(&str, &str)] = &[ ( @@ -120,12 +135,133 @@ pub async fn run_catalog_migrations( } println!(" Applying {filename}..."); execute_migration(session, keyspace, filename, sql).await?; + if *filename == BACKUP_CATALOG_INDEX_MIGRATION { + backfill_backup_catalog_index(session, keyspace).await?; + } record_migration(session, keyspace, filename).await?; } + write_catalog_version(session, keyspace).await?; println!(" Migrations applied."); Ok(()) } +/// Write `settings.catalog_version` = [`crate::CATALOG_VERSION`], the version +/// every file of [`CATALOG_MIGRATIONS`] brings the catalog to. The runner +/// writes it last, after the last file's statements, its step and its +/// `schema_history` row: a run that fails before this leaves the version +/// where it was, so `extenddb migrate` reports the catalog pending and +/// `migrate --yes` runs the unrecorded file again (every statement and step +/// is idempotent), and `serve` refuses the catalog until then. V001 to V005 +/// (released before this rule) also write their version as their last +/// statement; from V006 on a file does not name `catalog_version`. +async fn write_catalog_version(session: &Arc, keyspace: &str) -> OpResult<()> { + let insert = + format!("INSERT INTO {keyspace}.settings (key, value) VALUES ('catalog_version', ?)"); + session + .query_with_values(insert, cdrs_tokio::query_values!(crate::CATALOG_VERSION)) + .await + .map_err(|e| OpError::Internal(format!("Write catalog version: {e}")))?; + Ok(()) +} + +/// The step after the statements of [`BACKUP_CATALOG_INDEX_MIGRATION`]: +/// every `tables` row carrying a restore registration without a job status +/// gets `restore_kind = legacy` and `restore_owner_generation = 0` (the +/// unowned sentinel), with `restore_status = DONE` and `restore_completed_at` +/// the table's creation time where the table is not CREATING, and +/// `restore_status = RUNNING` with `restore_heartbeat_at` at the epoch where +/// it is (a restore the upgrade's stop interrupted, which the reconciler's +/// first pass fails and removes); then `settings.catalog_id` is written if +/// absent. Each write is a lightweight transaction conditioned on the field +/// being unset, so a second run changes nothing. +async fn backfill_backup_catalog_index( + session: &Arc, + keyspace: &str, +) -> OpResult<()> { + use cdrs_tokio::types::IntoRustByName; + let rows = crate::cassandra_util::query_rows_quorum::( + session, + &format!( + "SELECT account_id, table_name, table_id, table_status, created_at, \ + restore_source_backup_arn, restore_status FROM {keyspace}.tables" + ), + cdrs_tokio::query_values!(), + "backfill restore registrations", + ) + .await?; + let (mut done, mut running) = (0_u32, 0_u32); + for row in rows { + let source: Option = row.get_by_name("restore_source_backup_arn").ok().flatten(); + let status: Option = row.get_by_name("restore_status").ok().flatten(); + if source.is_none() || status.is_some() { + continue; + } + let account_id: String = + crate::cassandra_util::get_column(&row, "account_id", "backfill registrations")?; + let table_name: String = + crate::cassandra_util::get_column(&row, "table_name", "backfill registrations")?; + let table_id: String = + crate::cassandra_util::get_column(&row, "table_id", "backfill registrations")?; + let table_status: Option = row.get_by_name("table_status").ok().flatten(); + let created_at: i64 = row.get_by_name("created_at").ok().flatten().unwrap_or(0); + let interrupted = table_status.as_deref() == Some("CREATING"); + let update = if interrupted { + format!( + "UPDATE {keyspace}.tables SET restore_status = 'RUNNING', restore_kind = 'legacy', \ + restore_owner_generation = 0, restore_heartbeat_at = 0 \ + WHERE account_id = ? AND table_name = ? \ + IF table_id = ? AND restore_status = null" + ) + } else { + format!( + "UPDATE {keyspace}.tables SET restore_status = 'DONE', restore_kind = 'legacy', \ + restore_owner_generation = 0, restore_heartbeat_at = 0, restore_completed_at = ? \ + WHERE account_id = ? AND table_name = ? \ + IF table_id = ? AND restore_status = null" + ) + }; + let values = if interrupted { + cdrs_tokio::query_values!(account_id.as_str(), table_name.as_str(), table_id.as_str()) + } else { + cdrs_tokio::query_values!( + created_at, + account_id.as_str(), + table_name.as_str(), + table_id.as_str() + ) + }; + let applied = crate::cassandra_util::query_lwt(session, &update, values) + .await + .and_then(|result| crate::cassandra_util::lwt_applied(&result)) + .map_err(|e| OpError::Internal(format!("backfill restore registration: {e}")))?; + if applied { + if interrupted { + running += 1; + } else { + done += 1; + } + } + } + println!(" Backfilled {done} finished and {running} interrupted restore registration(s)."); + + let catalog_id = uuid::Uuid::new_v4().to_string(); + let insert_id = format!( + "INSERT INTO {keyspace}.settings (key, value) VALUES ('catalog_id', ?) IF NOT EXISTS" + ); + let written = crate::cassandra_util::query_lwt( + session, + &insert_id, + cdrs_tokio::query_values!(catalog_id.as_str()), + ) + .await + .and_then(|result| crate::cassandra_util::lwt_applied(&result)) + .map_err(|e| OpError::Internal(format!("write catalog_id: {e}")))?; + if written { + println!(" Catalog id {catalog_id}."); + } + Ok(()) +} + /// Run data database migrations. pub async fn run_data_migrations(session: &Arc, keyspace: &str) -> OpResult<()> { // Hold the process-wide migration lock for the whole USE-then-DDL sequence diff --git a/crates/storage-cassandra/src/test_hooks.rs b/crates/storage-cassandra/src/test_hooks.rs new file mode 100644 index 000000000..80821442c --- /dev/null +++ b/crates/storage-cassandra/src/test_hooks.rs @@ -0,0 +1,170 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! Coordination hooks for this crate's integration tests (the +//! `direct_integration` binary), compiled only with the `test-hooks` feature. +//! Without the feature every hook is a no-op the compiler removes, so a +//! production build carries none of this. Each hook is keyed by the account +//! (or the backup ARN) a test owns, so the other tests running in the same +//! process are not affected. +//! +//! The hooks: +//! +//! * the heartbeat period of the jobs of one account, in place of +//! [`extenddb_storage::backup_catalog::JOB_HEARTBEAT_INTERVAL`], so a test +//! can watch a short create or restore beat; +//! * a count of heartbeats of one account's jobs that fail with an error +//! before reaching the catalog, to exercise the consecutive-error rule of +//! `JobHeartbeat`, and a read of how many are still to fire; +//! * a pause between the in-use read and the DELETING claim of a backup +//! delete (the legacy `delete` and `start_backup_delete`), so a test can +//! register a restore in that gap and see the claim's own +//! `active_restores = null` condition refuse the delete. + +use std::time::Duration; + +use extenddb_storage::error::StorageError; + +#[cfg(feature = "test-hooks")] +mod armed { + use std::collections::HashMap; + use std::sync::{Arc, LazyLock, Mutex}; + use std::time::Duration; + + use tokio::sync::Notify; + + /// A pause a test arms at one point of one operation: the operation + /// signals `reached` when it gets there and waits for `released`. + pub struct Pause { + reached: Notify, + released: Notify, + } + + impl Pause { + pub(super) fn new() -> Self { + Self { + reached: Notify::new(), + released: Notify::new(), + } + } + + /// Wait until the paused operation has reached the pause point. + pub async fn reached(&self) { + self.reached.notified().await; + } + + /// Let the paused operation continue. + pub fn release(&self) { + self.released.notify_one(); + } + + pub(super) async fn hold(&self) { + self.reached.notify_one(); + self.released.notified().await; + } + } + + #[derive(Default)] + pub(super) struct Hooks { + pub(super) heartbeat_interval: HashMap, + pub(super) heartbeat_errors: HashMap, + pub(super) before_delete_claim: HashMap>, + } + + pub(super) static HOOKS: LazyLock> = LazyLock::new(Mutex::default); + + pub(super) fn with_hooks(f: impl FnOnce(&mut Hooks) -> T) -> T { + let mut hooks = HOOKS + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + f(&mut hooks) + } +} + +#[cfg(feature = "test-hooks")] +pub use armed::Pause; + +/// Make the jobs of `account_id` heartbeat every `interval` instead of +/// `JOB_HEARTBEAT_INTERVAL`. +#[cfg(feature = "test-hooks")] +pub fn set_heartbeat_interval(account_id: &str, interval: Duration) { + armed::with_hooks(|hooks| { + hooks + .heartbeat_interval + .insert(account_id.to_owned(), interval); + }); +} + +/// Make the next `count` heartbeats of the jobs of `account_id` fail with an +/// error before reaching the catalog. +#[cfg(feature = "test-hooks")] +pub fn fail_heartbeats(account_id: &str, count: usize) { + armed::with_hooks(|hooks| { + hooks.heartbeat_errors.insert(account_id.to_owned(), count); + }); +} + +/// How many of the heartbeat errors [`fail_heartbeats`] armed for +/// `account_id` no beat has taken yet, so a test can see that the beats it +/// armed have fired. +#[cfg(feature = "test-hooks")] +pub fn pending_heartbeat_errors(account_id: &str) -> usize { + armed::with_hooks(|hooks| hooks.heartbeat_errors.get(account_id).copied().unwrap_or(0)) +} + +/// Arm a pause in the next delete of `backup_arn`, between its in-use read +/// and its DELETING claim. The pause is taken once and then disarmed. +#[cfg(feature = "test-hooks")] +pub fn pause_before_delete_claim(backup_arn: &str) -> std::sync::Arc { + let pause = std::sync::Arc::new(Pause::new()); + armed::with_hooks(|hooks| { + hooks + .before_delete_claim + .insert(backup_arn.to_owned(), std::sync::Arc::clone(&pause)); + }); + pause +} + +/// The period the jobs of `account_id` heartbeat at. +pub(crate) fn heartbeat_interval(account_id: &str) -> Duration { + #[cfg(feature = "test-hooks")] + if let Some(interval) = + armed::with_hooks(|hooks| hooks.heartbeat_interval.get(account_id).copied()) + { + return interval; + } + let _ = account_id; + extenddb_storage::backup_catalog::JOB_HEARTBEAT_INTERVAL +} + +/// The error the next heartbeat of a job of `account_id` fails with, if a +/// test asked for one. +pub(crate) fn heartbeat_error(account_id: &str) -> Option { + #[cfg(feature = "test-hooks")] + { + let injected = + armed::with_hooks(|hooks| match hooks.heartbeat_errors.get_mut(account_id) { + Some(left) if *left > 0 => { + *left -= 1; + true + } + _ => false, + }); + if injected { + return Some(StorageError::Internal( + "test hook: heartbeat failed before reaching the catalog".to_owned(), + )); + } + } + let _ = account_id; + None +} + +/// Hold here if a test armed a pause for the delete of `backup_arn`. +pub(crate) async fn before_delete_claim(backup_arn: &str) { + #[cfg(feature = "test-hooks")] + if let Some(pause) = armed::with_hooks(|hooks| hooks.before_delete_claim.remove(backup_arn)) { + pause.hold().await; + } + let _ = backup_arn; +} diff --git a/crates/storage-cassandra/tests/direct/backup_engine.rs b/crates/storage-cassandra/tests/direct/backup_engine.rs index 3f0ef92de..b7f2933ef 100644 --- a/crates/storage-cassandra/tests/direct/backup_engine.rs +++ b/crates/storage-cassandra/tests/direct/backup_engine.rs @@ -11,6 +11,7 @@ use std::collections::BTreeMap; use std::time::{Duration, Instant}; +use extenddb_core::backup_format::BackupArn; use extenddb_core::expression::{Expr, ExpressionMaps, KeyCondition, PathElement}; use extenddb_core::types::{ AttributeDefinition, AttributeValue, BillingMode, CreateTableInput, DeleteTableInput, @@ -18,12 +19,17 @@ use extenddb_core::types::{ ProvisionedThroughput, ScalarAttributeType, TableDescription, TableKeyInfo, TableStatus, TimeToLiveStatus, }; +use extenddb_storage::backup_catalog::{ + BackupRow, BackupStatus, DeleteStart, Heartbeat, JobStatus, RestoreTargetKey, StatusFields, + StoreIdentity, StoreKind, TimeBounds, UNOWNED_GENERATION, WriteOutcome, +}; use extenddb_storage::error::StorageError; use extenddb_storage::{ - BackupEngine, DataEngine, RestoreRequest, RestoreTableOverrides, TableEngine, + BackupCatalog, DataEngine, JobPermit, LegacyBackupReader, RestoreRequest, + RestoreTableOverrides, TableEngine, }; use extenddb_storage_cassandra::CassandraEngine; -use futures::StreamExt; +use futures::{StreamExt, TryStreamExt}; use crate::helpers::{TestAccount, TestTable, setup_engine, unique_test_account}; @@ -140,11 +146,11 @@ async fn restore( target: &str, overrides: &RestoreTableOverrides, ) -> Result { - let source = engine.backup_restore_source(account_id, backup_arn).await?; + let source = engine.restore_source(account_id, backup_arn).await?; source.ensure_restorable()?; let input = source.restore_input(target, overrides); engine - .restore_table_from_backup( + .restore( account_id, RestoreRequest { backup_arn: backup_arn.to_owned(), @@ -152,6 +158,7 @@ async fn restore( restore_date_time: source.backup_creation_date_time, input, }, + JobPermit::detached(), ) .await } @@ -263,7 +270,12 @@ async fn backed_up_plain_table( ) .await; let backup = engine - .create_backup(&account.account_id, name, "copy-timing") + .create( + &account.account_id, + name, + "copy-timing", + JobPermit::detached(), + ) .await .expect("create backup"); (account, key_info, backup.backup_arn) @@ -293,10 +305,11 @@ async fn test_backup_describe_list_delete_and_account_scope() { } let details = engine - .create_backup( + .create( &table.key_info.account_id, &table.key_info.table_name, "daily", + JobPermit::detached(), ) .await .expect("create backup"); @@ -316,7 +329,7 @@ async fn test_backup_describe_list_delete_and_account_scope() { assert_eq!(details.backup_creation_date_time, embedded_seconds); let description = engine - .describe_backup(&table.key_info.account_id, &details.backup_arn) + .describe(&table.key_info.account_id, &details.backup_arn) .await .expect("describe backup"); assert_eq!( @@ -334,14 +347,14 @@ async fn test_backup_describe_list_delete_and_account_scope() { ); let table_backups = engine - .list_backups(&table.key_info.account_id, Some(&table.key_info.table_name)) + .list(&table.key_info.account_id, Some(&table.key_info.table_name)) .await .expect("list table backups"); assert_eq!(table_backups.len(), 1); assert_eq!(table_backups[0].backup_arn, details.backup_arn); assert_eq!(table_backups[0].table_id, table.key_info.table_id); let account_backups = engine - .list_backups(&table.key_info.account_id, None) + .list(&table.key_info.account_id, None) .await .expect("list account backups"); assert_eq!(account_backups.len(), 1); @@ -353,14 +366,14 @@ async fn test_backup_describe_list_delete_and_account_scope() { let foreign = unique_test_account(); let foreign_error = engine - .describe_backup(&foreign, &details.backup_arn) + .describe(&foreign, &details.backup_arn) .await .expect_err("another account must not resolve the backup"); assert!( matches!(foreign_error, StorageError::Validation(message) if message.contains("Backup not found")) ); let foreign_source = engine - .backup_restore_source(&foreign, &details.backup_arn) + .restore_source(&foreign, &details.backup_arn) .await .expect_err("another account must not resolve the restore source"); assert!( @@ -368,25 +381,25 @@ async fn test_backup_describe_list_delete_and_account_scope() { ); let deleted = engine - .delete_backup(&table.key_info.account_id, &details.backup_arn) + .delete(&table.key_info.account_id, &details.backup_arn) .await .expect("delete backup"); assert_eq!(deleted.backup_details.backup_status, "DELETED"); assert!( engine - .describe_backup(&table.key_info.account_id, &details.backup_arn) + .describe(&table.key_info.account_id, &details.backup_arn) .await .is_err() ); assert!( engine - .list_backups(&table.key_info.account_id, None) + .list(&table.key_info.account_id, None) .await .expect("list after delete") .is_empty() ); let gone = engine - .backup_restore_source(&table.key_info.account_id, &details.backup_arn) + .restore_source(&table.key_info.account_id, &details.backup_arn) .await .expect_err("a deleted backup is not a restore source"); assert!(matches!(gone, StorageError::Validation(m) if m.contains("Backup not found"))); @@ -412,7 +425,12 @@ async fn test_restore_uses_immutable_snapshot_with_sort_key() { .await; let backup = engine - .create_backup(&account_id, &table.key_info.table_name, "immutable") + .create( + &account_id, + &table.key_info.table_name, + "immutable", + JobPermit::detached(), + ) .await .expect("create backup"); @@ -488,7 +506,7 @@ async fn test_restore_uses_immutable_snapshot_with_sort_key() { .await .expect("delete restored table"); engine - .delete_backup(&account_id, &backup.backup_arn) + .delete(&account_id, &backup.backup_arn) .await .expect("delete backup"); } @@ -518,7 +536,12 @@ async fn test_table_name_filter_spans_table_recreation() { .await .expect("seed original table"); let backup = engine - .create_backup(&account.account_id, table_name, "before-recreate") + .create( + &account.account_id, + table_name, + "before-recreate", + JobPermit::detached(), + ) .await .expect("backup original table"); let original_table_id = source.key_info.table_id.clone(); @@ -542,7 +565,7 @@ async fn test_table_name_filter_spans_table_recreation() { assert_eq!(replacement.key_info.key_schema.len(), 2); let listed = engine - .list_backups(&account.account_id, Some(table_name)) + .list(&account.account_id, Some(table_name)) .await .expect("list backups across table recreation"); assert_eq!(listed.len(), 1); @@ -591,7 +614,7 @@ async fn test_table_name_filter_spans_table_recreation() { .await .expect("delete restored table"); engine - .delete_backup(&account.account_id, &backup.backup_arn) + .delete(&account.account_id, &backup.backup_arn) .await .expect("delete backup"); } @@ -691,11 +714,16 @@ async fn test_restore_reproduces_gsi_and_lsi_contents() { put_all(&engine, &source, items).await; let backup = engine - .create_backup(&account_id, "IndexedSource", "indexed") + .create( + &account_id, + "IndexedSource", + "indexed", + JobPermit::detached(), + ) .await .expect("create backup"); let recorded = engine - .backup_restore_source(&account_id, &backup.backup_arn) + .restore_source(&account_id, &backup.backup_arn) .await .expect("restore source"); let definition = recorded.definition.as_ref().expect("definition recorded"); @@ -808,7 +836,7 @@ async fn test_restore_applies_billing_overrides() { ) .await; let backup = engine - .create_backup(&account_id, "OnDemandSource", "od") + .create(&account_id, "OnDemandSource", "od", JobPermit::detached()) .await .expect("backup on-demand source"); let overrides = RestoreTableOverrides { @@ -867,11 +895,16 @@ async fn test_restore_applies_billing_overrides() { ) .await; let backup = engine - .create_backup(&account_id, "ProvisionedSource", "pv") + .create( + &account_id, + "ProvisionedSource", + "pv", + JobPermit::detached(), + ) .await .expect("backup provisioned source"); let source = engine - .backup_restore_source(&account_id, &backup.backup_arn) + .restore_source(&account_id, &backup.backup_arn) .await .expect("restore source"); assert_eq!(source.billing_mode, "PROVISIONED"); @@ -996,12 +1029,12 @@ async fn test_dropped_restore_request_still_reaches_active() { backed_up_plain_table(&engine, "DroppedSource", ITEMS).await; let account_id = account.account_id.clone(); let source = engine - .backup_restore_source(&account_id, &backup_arn) + .restore_source(&account_id, &backup_arn) .await .expect("restore source"); source.ensure_restorable().expect("restorable"); let input = source.restore_input("DroppedTarget", &RestoreTableOverrides::default()); - let request = engine.restore_table_from_backup( + let request = engine.restore( &account_id, RestoreRequest { backup_arn: backup_arn.clone(), @@ -1009,6 +1042,7 @@ async fn test_dropped_restore_request_still_reaches_active() { restore_date_time: source.backup_creation_date_time, input, }, + JobPermit::detached(), ); assert!( request.now_or_never().is_none(), @@ -1069,7 +1103,7 @@ async fn test_backup_and_target_are_held_while_restoring() { let (account, source, backup_arn) = backed_up_plain_table(&engine, "HeldSource", ITEMS).await; let account_id = account.account_id.clone(); let source_backup = engine - .backup_restore_source(&account_id, &backup_arn) + .restore_source(&account_id, &backup_arn) .await .expect("restore source"); @@ -1084,7 +1118,7 @@ async fn test_backup_and_target_are_held_while_restoring() { .expect("restore"); let in_use = engine - .delete_backup(&account_id, &backup_arn) + .delete(&account_id, &backup_arn) .await .expect_err("DeleteBackup during the copy"); assert!( @@ -1157,7 +1191,7 @@ async fn test_backup_and_target_are_held_while_restoring() { .await .expect("DeleteTable after ACTIVE"); engine - .delete_backup(&account_id, &backup_arn) + .delete(&account_id, &backup_arn) .await .expect("DeleteBackup after ACTIVE"); } @@ -1183,7 +1217,12 @@ async fn test_a_restore_is_refused_while_a_target_of_its_backup_is_creating() { let (account, source, b1) = backed_up_plain_table(&engine, "InUseSource", ITEMS).await; let account_id = account.account_id.clone(); let b2 = engine - .create_backup(&account_id, &source.table_name, "second") + .create( + &account_id, + &source.table_name, + "second", + JobPermit::detached(), + ) .await .expect("second backup") .backup_arn; @@ -1236,11 +1275,11 @@ async fn test_a_restore_is_refused_while_a_target_of_its_backup_is_creating() { // The name is checked before the backup: a backup that does not exist, // restored into that name, is refused as the name too. let source = engine - .backup_restore_source(&account_id, &b2) + .restore_source(&account_id, &b2) .await .expect("source"); let missing = engine - .restore_table_from_backup( + .restore( &account_id, RestoreRequest { backup_arn: flip_last_hex(&b2), @@ -1248,6 +1287,7 @@ async fn test_a_restore_is_refused_while_a_target_of_its_backup_is_creating() { restore_date_time: source.backup_creation_date_time, input: source.restore_input("InUseTarget", &overrides), }, + JobPermit::detached(), ) .await; match missing { @@ -1379,7 +1419,7 @@ async fn test_backup_without_definition_restores_as_legacy() { ) .await; let backup = engine - .create_backup(&account_id, "LegacySource", "legacy") + .create(&account_id, "LegacySource", "legacy", JobPermit::detached()) .await .expect("create backup"); @@ -1398,7 +1438,7 @@ async fn test_backup_without_definition_restores_as_legacy() { .expect("strip definition"); let legacy = engine - .backup_restore_source(&account_id, &backup.backup_arn) + .restore_source(&account_id, &backup.backup_arn) .await .expect("legacy restore source"); assert!(legacy.definition.is_none()); @@ -1433,12 +1473,15 @@ async fn test_backup_without_definition_restores_as_legacy() { ); } -/// A restore whose copy stopped heartbeating is removed by the sweep, and -/// the backup it held becomes deletable. Ownership is taken away from the -/// running copy (as if its process had died and another had claimed it) by -/// rewriting the owner and back-dating the heartbeat. +/// A restore whose owner died is found by `list_dead_restore_jobs` only once +/// its heartbeat is older than the stale window, and is then failed and +/// removed through `fail_restore_job` and `remove_restore_target`, after +/// which the backup it held becomes deletable. Ownership is taken away from +/// the running copy (as if its process had died and another had claimed the +/// target) by rewriting the generation and back-dating the heartbeat; the +/// copy notices and does not resurrect the target. #[tokio::test] -async fn test_sweep_removes_abandoned_restore_target() { +async fn test_abandoned_restore_is_removed_only_when_its_heartbeat_is_stale() { if crate::helpers::skip_without_cassandra() { return; } @@ -1456,45 +1499,56 @@ async fn test_sweep_removes_abandoned_restore_target() { ) .await .expect("restore"); + orphan_restore_target(&engine, &account_id, "AbandonedTarget", &target.table_id).await; + let key = RestoreTargetKey { + account_id: account_id.clone(), + table_name: "AbandonedTarget".to_owned(), + table_id: target.table_id.clone(), + }; - let stale = extenddb_storage_cassandra::cassandra_util::now_millis() - 120_000; - let orphan = format!( - "UPDATE {}.tables SET restore_owner = ?, restore_heartbeat = ? \ - WHERE account_id = ? AND table_name = ? IF table_id = ? AND table_status = 'CREATING'", - engine.catalog_keyspace() - ); - let result = extenddb_storage_cassandra::cassandra_util::query_lwt( - engine.session(), - &orphan, - cdrs_tokio::query_values!( - uuid::Uuid::new_v4(), - stale, - account_id.as_str(), - "AbandonedTarget", - target.table_id.as_str() - ), - ) - .await - .expect("orphan the target"); - assert!( - extenddb_storage_cassandra::cassandra_util::lwt_applied(&result).unwrap(), - "the copy must still be running when the target is orphaned" - ); - - // A fresh heartbeat is not swept. + // A heartbeat two minutes old is not stale under a ten-minute window. let not_yet = engine - .sweep_abandoned_restores(Duration::from_secs(600)) + .list_dead_restore_jobs(Duration::from_secs(600)) .await - .expect("sweep"); - assert!(!not_yet.contains(&"AbandonedTarget".to_owned())); - - let removed = engine - .sweep_abandoned_restores(Duration::from_secs(60)) + .expect("list"); + assert!( + !not_yet.iter().any(|job| job.target == key), + "a heartbeat younger than the window is not dead: {not_yet:?}" + ); + let dead = engine + .list_dead_restore_jobs(Duration::from_secs(60)) .await - .expect("sweep"); + .expect("list"); + let job = dead + .iter() + .find(|job| job.target == key) + .unwrap_or_else(|| panic!("the orphaned target is dead under a 60 s window: {dead:?}")); + assert_eq!(job.source_backup_arn, backup_arn); + assert_eq!(job.status, JobStatus::Running); + assert!( + engine + .fail_restore_job(&key, "owner dead") + .await + .expect("fail"), + "the record was RUNNING" + ); + assert!( + !engine + .fail_restore_job(&key, "owner dead") + .await + .expect("fail again"), + "a FAILED record is not RUNNING" + ); + assert!( + engine.remove_restore_target(&key).await.expect("remove"), + "the target was a CREATING restore target" + ); assert!( - removed.contains(&"AbandonedTarget".to_owned()), - "removed {removed:?}" + !engine + .remove_restore_target(&key) + .await + .expect("remove again"), + "a removed target is not one" ); let gone = engine .describe_table( @@ -1504,7 +1558,7 @@ async fn test_sweep_removes_abandoned_restore_target() { }, ) .await - .expect_err("swept target is gone"); + .expect_err("removed target is gone"); assert!(matches!(gone, StorageError::TableNotFound(_))); // The orphaned copy notices it lost the target and does not resurrect it. tokio::time::sleep(Duration::from_secs(2)).await; @@ -1520,7 +1574,7 @@ async fn test_sweep_removes_abandoned_restore_target() { .is_err() ); engine - .delete_backup(&account_id, &backup_arn) + .delete(&account_id, &backup_arn) .await .expect("the abandoned restore no longer holds the backup"); } @@ -1535,16 +1589,16 @@ async fn test_restore_of_deleted_backup_is_not_found() { let engine = setup_engine().await; let (account, _source, backup_arn) = backed_up_plain_table(&engine, "GoneSource", 3).await; let source = engine - .backup_restore_source(&account.account_id, &backup_arn) + .restore_source(&account.account_id, &backup_arn) .await .expect("source"); let input = source.restore_input("GoneTarget", &RestoreTableOverrides::default()); engine - .delete_backup(&account.account_id, &backup_arn) + .delete(&account.account_id, &backup_arn) .await .expect("delete backup"); let error = engine - .restore_table_from_backup( + .restore( &account.account_id, RestoreRequest { backup_arn: backup_arn.clone(), @@ -1552,6 +1606,7 @@ async fn test_restore_of_deleted_backup_is_not_found() { restore_date_time: source.backup_creation_date_time, input, }, + JobPermit::detached(), ) .await .expect_err("restore of a deleted backup"); @@ -1571,9 +1626,10 @@ async fn test_restore_of_deleted_backup_is_not_found() { // ── Isolated catalogs (own keyspace prefix) ──────────────────────────────── -/// Catalog migrations V001-V004: the catalog shape before V005 added -/// `backups_by_arn.definition/active_restores` and `tables.restore_*`. -const CATALOG_THROUGH_V004: &[(&str, &str)] = &[ +/// Catalog migrations V001-V005: the catalog shape before V006 added the +/// backup catalog index (`backups_by_arn.store_*`, `backup_index_by_status`, +/// the `tables.restore_*` job record, `backup_maintenance`, `catalog_id`). +const CATALOG_THROUGH_V005: &[(&str, &str)] = &[ ( "V001__initial_schema.cql", include_str!("../../migrations/catalog/V001__initial_schema.cql"), @@ -1590,15 +1646,21 @@ const CATALOG_THROUGH_V004: &[(&str, &str)] = &[ "V004__ttl_backfill_cursor.cql", include_str!("../../migrations/catalog/V004__ttl_backfill_cursor.cql"), ), + ( + "V005__backup_definitions_and_restore_registration.cql", + include_str!( + "../../migrations/catalog/V005__backup_definitions_and_restore_registration.cql" + ), + ), ]; -/// An engine on its own keyspace prefix (so its sweeps and workers touch no -/// other test's tables) with one provisioned account. With `through_v004` -/// the catalog has run only catalog migrations V001-V004, as a catalog not -/// yet migrated by `extenddb migrate --yes`; otherwise every migration. +/// An engine on its own keyspace prefix (so its catalog scans see no other +/// test's tables) with one provisioned account. With `through_v005` the +/// catalog has run only catalog migrations V001-V005, as a catalog not yet +/// migrated by `extenddb migrate --yes`; otherwise every migration. async fn isolated_engine( tag: &str, - through_v004: bool, + through_v005: bool, ) -> ( std::sync::Arc, String, @@ -1621,11 +1683,11 @@ async fn isolated_engine( .await .unwrap(); } - if through_v004 { + if through_v005 { // Statement by statement, each qualified with the catalog keyspace by - // the function the migration runner uses, recording versions 1-4 in - // schema_history as the runner did for a 0.0.3 catalog. - for (filename, sql) in CATALOG_THROUGH_V004 { + // the function the migration runner uses, recording versions 1-5 in + // schema_history as the runner did for a 0.0.4 catalog. + for (filename, sql) in CATALOG_THROUGH_V005 { let statements = extenddb_storage_cassandra::migrations::migration_statements( &catalog_ks, filename, @@ -1633,20 +1695,20 @@ async fn isolated_engine( ) .expect(filename); for statement in statements { - session.query(statement).await.expect("V001-V004 statement"); + session.query(statement).await.expect("V001-V005 statement"); } } - for version in 1..=4_i32 { + for version in 1..=5_i32 { session .query_with_values( format!( "INSERT INTO {catalog_ks}.schema_history (version, description, applied_at) \ - VALUES (?, 'through_v004', toTimestamp(now()))" + VALUES (?, 'through_v005', toTimestamp(now()))" ), cdrs_tokio::query_values!(version), ) .await - .expect("record V001-V004"); + .expect("record V001-V005"); } } else { extenddb_storage_cassandra::migrations::run_catalog_migrations(&session, &catalog_ks) @@ -1669,8 +1731,9 @@ async fn isolated_engine( (engine, account_id, drop_keyspaces) } -/// The columns catalog migration V005 adds are absent from `keyspace`. -async fn assert_catalog_lacks_v005(engine: &CassandraEngine, keyspace: &str) { +/// The columns and tables catalog migration V006 adds are absent from +/// `keyspace`, and V005's are present. +async fn assert_catalog_lacks_v006(engine: &CassandraEngine, keyspace: &str) { let rows = extenddb_storage_cassandra::cassandra_util::query_rows::( &engine.session_arc(), "SELECT column_name FROM system_schema.columns WHERE keyspace_name = ?", @@ -1691,26 +1754,69 @@ async fn assert_catalog_lacks_v005(engine: &CassandraEngine, keyspace: &str) { }) .collect(); assert!(columns.iter().any(|c| c == "table_status"), "{columns:?}"); - for v005 in [ - "restore_source_backup_arn", - "restore_heartbeat", - "active_restores", - "definition", + for v005 in ["restore_source_backup_arn", "active_restores", "definition"] { + assert!(columns.iter().any(|c| c == v005), "{v005} absent"); + } + for v006 in [ + "restore_status", + "restore_owner_generation", + "restore_heartbeat_at", + "store_kind", + "source_table_details_json", ] { - assert!(!columns.iter().any(|c| c == v005), "{v005} present"); + assert!(!columns.iter().any(|c| c == v006), "{v006} present"); + } + let tables = extenddb_storage_cassandra::cassandra_util::query_rows::( + &engine.session_arc(), + "SELECT table_name FROM system_schema.tables WHERE keyspace_name = ?", + cdrs_tokio::query_values!(keyspace), + "tables", + ) + .await + .expect("read schema"); + let names: Vec = tables + .iter() + .map(|row| { + extenddb_storage_cassandra::cassandra_util::get_column::( + row, + "table_name", + "tables", + ) + .unwrap() + }) + .collect(); + for v006 in ["backup_index_by_status", "backup_maintenance"] { + assert!(!names.iter().any(|n| n == v006), "{v006} present"); } + assert!( + engine.catalog_id().await.is_err(), + "a catalog without V006 has no catalog id" + ); + // `serve` does not start on it: the factory's guard refuses the 0.0.4 + // catalog with the command that brings it forward. + let refusal = engine + .require_served_catalog() + .await + .expect_err("serve refuses a catalog behind the binary") + .to_string(); + assert!( + refusal.contains("at version 0.0.4") + && refusal.contains(extenddb_storage_cassandra::CATALOG_VERSION) + && refusal.contains("extenddb migrate --yes"), + "{refusal}" + ); } -/// On a catalog that has not run V005, ordinary table and item operations -/// work; only backup and restore need V005. (CreateTable used to name the -/// V005 restore columns in its INSERT for every table, which CQL rejects.) +/// On a catalog that has not run V006, ordinary table and item operations +/// work; only backup and restore need V006. (CreateTable used to name the +/// restore columns in its INSERT for every table, which CQL rejects.) #[tokio::test] -async fn test_table_operations_work_on_a_catalog_without_v005() { +async fn test_table_operations_work_on_a_catalog_without_v006() { if crate::helpers::skip_without_cassandra() { return; } - let (engine, account_id, _drop_keyspaces) = isolated_engine("v4", true).await; - assert_catalog_lacks_v005(&engine, &engine.catalog_keyspace()).await; + let (engine, account_id, _drop_keyspaces) = isolated_engine("v5", true).await; + assert_catalog_lacks_v006(&engine, &engine.catalog_keyspace()).await; let created = engine .create_table( @@ -1765,15 +1871,15 @@ async fn test_table_operations_work_on_a_catalog_without_v005() { assert!(got.is_some()); assert_eq!(scan_count(&engine, &key_info).await, 1); - // Backup needs V005. CreateBackup, DeleteBackup and RestoreTableFromBackup + // Backup needs V006. CreateBackup, DeleteBackup and RestoreTableFromBackup // are refused with a message naming the file and `migrate` (they failed // with an internal `Undefined column name ...`); nothing is created. let assert_refused = |what: &str, result: Result| match result { Err(StorageError::Unsupported(m)) => { assert!( - m.contains("V005__backup_definitions_and_restore_registration.cql") + m.contains("V006__backup_catalog_index.cql") && m.contains("extenddb migrate --yes") - && !m.contains("V004"), + && !m.contains("V005"), "{what}: {m}" ); } @@ -1782,7 +1888,12 @@ async fn test_table_operations_work_on_a_catalog_without_v005() { assert_refused( "CreateBackup", engine - .create_backup(&account_id, "Unmigrated", "needs-v005") + .create( + &account_id, + "Unmigrated", + "needs-v006", + JobPermit::detached(), + ) .await .map(|d| d.backup_arn), ); @@ -1795,14 +1906,14 @@ async fn test_table_operations_work_on_a_catalog_without_v005() { assert_refused( "DeleteBackup", engine - .delete_backup(&account_id, &some_arn) + .delete(&account_id, &some_arn) .await .map(|d| d.backup_details.backup_arn), ); assert_refused( "RestoreTableFromBackup", engine - .restore_table_from_backup( + .restore( &account_id, RestoreRequest { backup_arn: some_arn.clone(), @@ -1816,6 +1927,7 @@ async fn test_table_operations_work_on_a_catalog_without_v005() { ..Default::default() }, }, + JobPermit::detached(), ) .await .map(|d| d.table_name), @@ -1833,23 +1945,36 @@ async fn test_table_operations_work_on_a_catalog_without_v005() { "a refused restore creates no target" ); - // Liveness: once V005 is applied (as `migrate --yes` does, the runner + // Liveness: once V006 is applied (as `migrate --yes` does, the runner // skipping the recorded files), the same engine backs up without a - // restart. + // restart, and the catalog has its id. extenddb_storage_cassandra::migrations::run_catalog_migrations( &engine.session_arc(), &engine.catalog_keyspace(), ) .await - .expect("apply V005"); + .expect("apply V006"); + assert!( + uuid::Uuid::parse_str(&engine.catalog_id().await.expect("catalog id")).is_ok(), + "the migration writes a catalog id" + ); + engine + .require_served_catalog() + .await + .expect("serve accepts the migrated catalog"); let backup = engine - .create_backup(&account_id, "Unmigrated", "after-v005") + .create( + &account_id, + "Unmigrated", + "after-v006", + JobPermit::detached(), + ) .await - .expect("CreateBackup after V005"); + .expect("CreateBackup after V006"); engine - .delete_backup(&account_id, &backup.backup_arn) + .delete(&account_id, &backup.backup_arn) .await - .expect("DeleteBackup after V005"); + .expect("DeleteBackup after V006"); engine .delete_table( @@ -1859,12 +1984,13 @@ async fn test_table_operations_work_on_a_catalog_without_v005() { }, ) .await - .expect("DeleteTable on a catalog without V005"); + .expect("DeleteTable on a catalog without V006"); } /// Back-date the heartbeat of the CREATING restore target `name` by two -/// minutes and give it a foreign owner, as if the process copying into it had -/// died; asserts the copy was still running (the LWT applied). +/// minutes and give its job record another generation, as if the process +/// copying into it had died and another had claimed it; asserts the copy was +/// still running (the LWT applied). async fn orphan_restore_target( engine: &CassandraEngine, account_id: &str, @@ -1873,14 +1999,15 @@ async fn orphan_restore_target( ) { let stale = extenddb_storage_cassandra::cassandra_util::now_millis() - 120_000; let orphan = format!( - "UPDATE {}.tables SET restore_owner = ?, restore_heartbeat = ? \ + "UPDATE {}.tables SET restore_owner_generation = ?, restore_heartbeat_at = ? \ WHERE account_id = ? AND table_name = ? IF table_id = ? AND table_status = 'CREATING'", engine.catalog_keyspace() ); + let foreign_generation = 1_i64; let result = extenddb_storage_cassandra::cassandra_util::query_lwt( engine.session(), &orphan, - cdrs_tokio::query_values!(uuid::Uuid::new_v4(), stale, account_id, name, table_id), + cdrs_tokio::query_values!(foreign_generation, stale, account_id, name, table_id), ) .await .expect("orphan the target"); @@ -1890,90 +2017,11 @@ async fn orphan_restore_target( ); } -/// The server's sweep worker sweeps once as soon as it starts (not one -/// interval later) and returns promptly once shutdown is signalled. -#[tokio::test] -async fn test_restore_sweep_worker_sweeps_at_startup_and_stops_on_shutdown() { - if crate::helpers::skip_without_cassandra() { - return; - } - let (engine, account_id, _drop_keyspaces) = isolated_engine("sw", false).await; - let source = create_active_table( - &engine, - &account_id, - CreateTableInput { - table_name: "SweepSource".to_owned(), - key_schema: vec![ks("id", KeyType::Hash)], - attribute_definitions: vec![attr("id", ScalarAttributeType::S)], - billing_mode: Some(BillingMode::PayPerRequest), - ..Default::default() - }, - ) - .await; - put_all( - &engine, - &source, - (0..1_500) - .map(|i| item(&format!("item-{i:05}"), None, &"x".repeat(200))) - .collect(), - ) - .await; - let backup = engine - .create_backup(&account_id, "SweepSource", "sweep") - .await - .expect("create backup"); - let target = restore( - &engine, - &account_id, - &backup.backup_arn, - "SweepTarget", - &RestoreTableOverrides::default(), - ) - .await - .expect("restore"); - orphan_restore_target(&engine, &account_id, "SweepTarget", &target.table_id).await; - - let shutdown = extenddb_storage::CancellationToken::new(); - let started = Instant::now(); - let worker = tokio::spawn(extenddb_storage_cassandra::poll_abandoned_restores( - std::sync::Arc::clone(&engine), - shutdown.clone(), - )); - // Well inside RESTORE_SWEEP_INTERVAL (15 s): only a sweep at startup - // removes the target this soon. - let deadline = started + Duration::from_secs(8); - loop { - let described = engine - .describe_table( - &account_id, - DescribeTableInput { - table_name: "SweepTarget".to_owned(), - }, - ) - .await; - if matches!(described, Err(StorageError::TableNotFound(_))) { - break; - } - assert!( - Instant::now() < deadline, - "the abandoned target was not swept within 8 s of the worker starting: {:?}", - described.map(|d| d.table_status) - ); - tokio::time::sleep(Duration::from_millis(100)).await; - } - - shutdown.cancel(); - tokio::time::timeout(Duration::from_secs(3), worker) - .await - .expect("the sweep worker did not stop within 3 s of shutdown") - .expect("sweep worker panicked"); -} - /// The restore heartbeat runs while the target's data and index tables are /// created, not only once the copy starts: with many indexes that DDL (each -/// statement waiting for schema agreement) can outlast RESTORE_STALE_AFTER on -/// a slow cluster, and a heartbeat still at its insert value would let a -/// sweep remove the live target. +/// statement waiting for schema agreement) can outlast the reconciler's +/// stale window on a slow cluster, and a heartbeat still at its insert value +/// would let the reconciler remove the live target. #[tokio::test] async fn test_restore_heartbeat_runs_during_target_ddl() { if crate::helpers::skip_without_cassandra() { @@ -2025,12 +2073,13 @@ async fn test_restore_heartbeat_runs_during_target_ddl() { ) .await; let backup = engine - .create_backup(&account_id, "ManyIndexes", "ddl") + .create(&account_id, "ManyIndexes", "ddl", JobPermit::detached()) .await .expect("create backup"); // Watch the target row while the restore request runs: when it first - // appears, and when its heartbeat first moves past its insert value. + // appears, and when its heartbeat is first written after the insert (the + // column's write time moves past the row's, which the insert set). let watching = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(true)); let watcher = { let engine = std::sync::Arc::clone(&engine); @@ -2038,8 +2087,8 @@ async fn test_restore_heartbeat_runs_during_target_ddl() { let watching = std::sync::Arc::clone(&watching); tokio::spawn(async move { let query = format!( - "SELECT created_at, restore_heartbeat FROM {}.tables \ - WHERE account_id = ? AND table_name = ?", + "SELECT WRITETIME(restore_heartbeat_at) AS beat, WRITETIME(table_id) AS inserted \ + FROM {}.tables WHERE account_id = ? AND table_name = ?", engine.catalog_keyspace() ); let (mut row_seen, mut refreshed) = (None, None); @@ -2056,10 +2105,11 @@ async fn test_restore_heartbeat_runs_during_target_ddl() { use cdrs_tokio::types::IntoRustByName; let observed = Instant::now(); row_seen.get_or_insert(observed); - let created: Option = row.get_by_name("created_at").ok().flatten(); - let heartbeat: Option = - row.get_by_name("restore_heartbeat").ok().flatten(); - if heartbeat > created { + let beat: Option = row.get_by_name("beat").ok().flatten(); + let inserted: Option = row.get_by_name("inserted").ok().flatten(); + if let (Some(beat), Some(inserted)) = (beat, inserted) + && beat > inserted + { refreshed = Some(observed); } } @@ -2107,3 +2157,2137 @@ async fn test_restore_heartbeat_runs_during_target_ddl() { ) .await; } + +// ── The catalog index ─────────────────────────────────────────────────────── + +/// A store identity no other test's rows carry, so a store-scoped listing +/// in the shared catalog keyspace sees only this test's rows. +fn unique_store() -> StoreIdentity { + StoreIdentity { + store_kind: StoreKind::Filesystem, + store_root: format!( + "/var/lib/extenddb/backups-{}", + crate::helpers::unique_test_id() + ), + store_id: "store-1".to_owned(), + } +} + +/// A row for a backup of `table` in `account` minted now, with the manifest +/// fixture's source table. +fn index_row(account: &str, table: &str, name: &str, store: &StoreIdentity) -> BackupRow { + let backup_id = extenddb_storage::util::new_backup_id(); + let backup_arn = extenddb_storage::util::backup_arn("us-east-1", account, table, &backup_id); + let manifest: serde_json::Value = serde_json::from_str(include_str!( + "../../../core/testdata/extenddb-manifest/v1.json" + )) + .expect("fixture"); + BackupRow { + backup_id: backup_id.clone(), + account_id: account.to_owned(), + table_name: table.to_owned(), + backup_arn, + backup_name: name.to_owned(), + status: BackupStatus::Creating, + store: store.clone(), + size_bytes: 0, + item_count: 0, + created_at: extenddb_storage::util::backup_creation_date_time(&backup_id).unwrap(), + completed_at: None, + failure_reason: None, + source_table_details_json: manifest["source_table"].clone(), + owner_generation: 0, + heartbeat_at: 0.0, + } +} + +fn arn_of(row: &BackupRow) -> BackupArn { + BackupArn::parse(&row.backup_arn).unwrap() +} + +/// Run `cql` with `values` as a lightweight transaction and assert it applied. +async fn apply(engine: &CassandraEngine, cql: &str, values: cdrs_tokio::query::QueryValues) { + let result = + extenddb_storage_cassandra::cassandra_util::query_lwt(engine.session(), cql, values) + .await + .expect("lwt"); + assert!( + extenddb_storage_cassandra::cassandra_util::lwt_applied(&result).unwrap(), + "{cql}" + ); +} + +/// The claim: a second claim of the same backup is refused while the row +/// exists, the heartbeat confirms the claim and refreshes the row, a status +/// write is conditioned on the generation and the `from` status, and the row +/// is read back by account, backup id, table and ARN. +#[tokio::test] +async fn test_claim_backup_excludes_a_second_owner_and_fences_its_writes() { + if crate::helpers::skip_without_cassandra() { + return; + } + let engine = setup_engine().await; + let account = TestAccount::new(&engine, "extenddb_ttl_test").await; + let account_id = account.account_id.clone(); + assert!( + uuid::Uuid::parse_str(&engine.catalog_id().await.expect("id")).is_ok(), + "the catalog has an id from the migration" + ); + assert!(!engine.check_maintenance_claim().await.expect("check")); + let store = unique_store(); + let row = index_row(&account_id, "Music", "nightly", &store); + let claim = engine.claim_backup(&row).await.expect("claim"); + assert_ne!(claim.generation(), UNOWNED_GENERATION); + let second = engine.claim_backup(&row).await.err(); + assert!( + matches!(second, Some(StorageError::AlreadyExists(_))), + "{second:?}" + ); + let stored = engine + .read_backup_row(&arn_of(&row)) + .await + .expect("read") + .expect("the row exists"); + assert_eq!(stored.status, BackupStatus::Creating); + assert_eq!(stored.owner_generation, claim.generation()); + assert_eq!(stored.store, store); + assert_eq!(stored.created_at, row.created_at); + assert_eq!(stored.backup_id, row.backup_id); + assert_eq!( + stored.source_table_details_json, + row.source_table_details_json + ); + let before = stored.heartbeat_at; + assert!(before > 0.0, "the insert stamps the heartbeat"); + tokio::time::sleep(Duration::from_millis(20)).await; + assert_eq!( + claim.heartbeat().await.expect("heartbeat"), + Heartbeat::Confirmed + ); + let after = engine + .read_backup_row(&arn_of(&row)) + .await + .unwrap() + .unwrap() + .heartbeat_at; + assert!(after > before, "{after} > {before}"); + + // The same id under another account, table or region finds nothing; the + // legacy reader does not see the row either. + let other_account = unique_test_account(); + let other_account_arn = + BackupArn::new("aws", "us-east-1", &other_account, "Music", &row.backup_id).unwrap(); + assert!( + engine + .read_backup_row(&other_account_arn) + .await + .unwrap() + .is_none() + ); + let other_table = + BackupArn::new("aws", "us-east-1", &account_id, "Other", &row.backup_id).unwrap(); + assert!( + engine + .read_backup_row(&other_table) + .await + .unwrap() + .is_none() + ); + let other_region = + BackupArn::new("aws", "eu-west-1", &account_id, "Music", &row.backup_id).unwrap(); + assert!( + engine + .read_backup_row(&other_region) + .await + .unwrap() + .is_none() + ); + assert!( + matches!( + engine.describe(&account_id, &row.backup_arn).await, + Err(StorageError::Validation(ref m)) if m.contains("Backup not found") + ), + "an index row is not an in-catalog backup" + ); + + // A write from the wrong state matches nothing; the right one matches + // once, and the row then carries what it set. + let fields = StatusFields { + size_bytes: Some(1234), + item_count: Some(5), + failure_reason: None, + completed: true, + }; + assert_eq!( + claim + .set_status(JobStatus::Running, JobStatus::Available, &fields) + .await + .unwrap(), + WriteOutcome::NotMatched + ); + assert_eq!( + claim + .set_status(JobStatus::Creating, JobStatus::Available, &fields) + .await + .unwrap(), + WriteOutcome::Matched + ); + assert_eq!( + claim + .set_status(JobStatus::Creating, JobStatus::Available, &fields) + .await + .unwrap(), + WriteOutcome::NotMatched + ); + let read = claim.read().await.expect("read"); + assert_eq!(read.status, JobStatus::Available); + assert_eq!(read.owner_generation, claim.generation()); + let stored = engine + .read_backup_row(&arn_of(&row)) + .await + .unwrap() + .unwrap(); + assert_eq!(stored.status, BackupStatus::Available); + assert_eq!(stored.size_bytes, 1234); + assert_eq!(stored.item_count, 5); + assert!(stored.completed_at.is_some()); + // The status index followed the row. + let available: Vec = engine + .list_backup_rows(Some(BackupStatus::Available), &store) + .try_collect() + .await + .unwrap(); + assert_eq!(available.len(), 1, "{available:?}"); + assert!( + engine + .list_backup_rows(Some(BackupStatus::Creating), &store) + .try_collect::>() + .await + .unwrap() + .is_empty() + ); + // The legacy listing does not show the index row. + assert!( + engine + .list(&account_id, None) + .await + .expect("legacy list") + .is_empty() + ); + + // A row another generation took over confirms nothing for this claim. + apply( + &engine, + &format!( + "UPDATE {}.backups_by_arn SET owner_generation = ? \ + WHERE account_id = ? AND backup_arn = ? IF owner_generation = ?", + engine.catalog_keyspace() + ), + cdrs_tokio::query_values!( + i64::try_from(claim.generation()).unwrap() + 1, + account_id.as_str(), + row.backup_arn.as_str(), + i64::try_from(claim.generation()).unwrap() + ), + ) + .await; + assert_eq!(claim.heartbeat().await.expect("heartbeat"), Heartbeat::Lost); + assert_eq!( + claim + .set_status( + JobStatus::Available, + JobStatus::Failed, + &StatusFields::default() + ) + .await + .unwrap(), + WriteOutcome::NotMatched + ); + drop(claim); + // The row is the claim: a successor can take it only through its insert, + // which the existing row refuses. + let third = engine.claim_backup(&row).await.err(); + assert!( + matches!(third, Some(StorageError::AlreadyExists(_))), + "{third:?}" + ); +} + +/// DeleteBackup's first step and the tombstone writes: only an AVAILABLE row +/// moves to DELETING; a CREATING row answers its status; a backup a CREATING +/// restore target reads answers the target's name; DELETING moves to DELETED +/// once; a DELETED row is removed once, with its list and status entries; a +/// missing row is "Backup not found". +#[tokio::test] +async fn test_start_backup_delete_checks_the_status_and_the_in_use_rule_under_the_lock() { + if crate::helpers::skip_without_cassandra() { + return; + } + let engine = setup_engine().await; + let account = TestAccount::new(&engine, "extenddb_ttl_test").await; + let account_id = account.account_id.clone(); + let store = unique_store(); + let row = index_row(&account_id, "Music", "nightly", &store); + let key = row.key(); + assert!(matches!( + engine.start_backup_delete(&key).await, + Err(StorageError::Validation(m)) if m.contains("Backup not found") + )); + let claim = engine.claim_backup(&row).await.expect("claim"); + assert_eq!( + engine.start_backup_delete(&key).await.unwrap(), + DeleteStart::Status(BackupStatus::Creating) + ); + assert_eq!( + claim + .set_status( + JobStatus::Creating, + JobStatus::Available, + &StatusFields::default() + ) + .await + .unwrap(), + WriteOutcome::Matched + ); + assert_eq!(engine.backup_in_use(&key).await.unwrap(), None); + + // A CREATING restore target registered from the backup: in use, naming + // the target; an ACTIVE one is not, and its registration is cleared. + let target = create_active_table( + &engine, + &account_id, + CreateTableInput { + table_name: "DelTarget".to_owned(), + key_schema: vec![ks("id", KeyType::Hash)], + attribute_definitions: vec![attr("id", ScalarAttributeType::S)], + billing_mode: Some(BillingMode::PayPerRequest), + ..Default::default() + }, + ) + .await; + let catalog = engine.catalog_keyspace(); + apply( + &engine, + &format!( + "UPDATE {catalog}.tables SET table_status = 'CREATING', status_transition_at = null, \ + restore_source_backup_arn = ? WHERE account_id = ? AND table_name = ? IF table_id = ?" + ), + cdrs_tokio::query_values!( + row.backup_arn.as_str(), + account_id.as_str(), + "DelTarget", + target.table_id.as_str() + ), + ) + .await; + apply( + &engine, + &format!( + "UPDATE {catalog}.backups_by_arn SET active_restores[?] = ? \ + WHERE account_id = ? AND backup_arn = ? IF EXISTS" + ), + cdrs_tokio::query_values!( + target.table_id.as_str(), + "DelTarget", + account_id.as_str(), + row.backup_arn.as_str() + ), + ) + .await; + assert_eq!( + engine.backup_in_use(&key).await.unwrap().as_deref(), + Some("DelTarget") + ); + assert_eq!( + engine.start_backup_delete(&key).await.unwrap(), + DeleteStart::InUse("DelTarget".to_owned()) + ); + assert_eq!( + engine + .read_backup_row(&arn_of(&row)) + .await + .unwrap() + .unwrap() + .status, + BackupStatus::Available, + "a refused delete leaves the row AVAILABLE" + ); + apply( + &engine, + &format!( + "UPDATE {catalog}.tables SET table_status = 'ACTIVE' \ + WHERE account_id = ? AND table_name = ? IF table_id = ?" + ), + cdrs_tokio::query_values!(account_id.as_str(), "DelTarget", target.table_id.as_str()), + ) + .await; + assert_eq!(engine.backup_in_use(&key).await.unwrap(), None); + + assert_eq!( + engine.start_backup_delete(&key).await.unwrap(), + DeleteStart::Deleting + ); + assert_eq!( + engine.start_backup_delete(&key).await.unwrap(), + DeleteStart::Status(BackupStatus::Deleting), + "a retry finds the row DELETING" + ); + assert!( + !engine.remove_backup_row(&key).await.unwrap(), + "DELETING is not a tombstone" + ); + assert!(engine.set_backup_deleted(&key).await.unwrap()); + assert!(!engine.set_backup_deleted(&key).await.unwrap()); + let stored = engine + .read_backup_row(&arn_of(&row)) + .await + .unwrap() + .unwrap(); + assert_eq!(stored.status, BackupStatus::Deleted); + assert!(stored.completed_at.is_some()); + let deleted: Vec = engine + .list_backup_rows(Some(BackupStatus::Deleted), &store) + .try_collect() + .await + .unwrap(); + assert_eq!(deleted.len(), 1, "the status index followed: {deleted:?}"); + assert!(engine.remove_backup_row(&key).await.unwrap()); + assert!(!engine.remove_backup_row(&key).await.unwrap()); + assert!( + engine + .read_backup_row(&arn_of(&row)) + .await + .unwrap() + .is_none() + ); + assert!( + engine + .list_backup_rows(None, &store) + .try_collect::>() + .await + .unwrap() + .is_empty(), + "the status entry went with the row" + ); + assert!( + engine + .list_account_backup_rows(&account_id, None, &TimeBounds::default()) + .try_collect::>() + .await + .unwrap() + .is_empty(), + "the list entries went with the row" + ); + drop(claim); + engine + .delete_table( + &account_id, + DeleteTableInput { + table_name: "DelTarget".to_owned(), + }, + ) + .await + .expect("delete the target"); +} + +/// The listings: by account with an optional table and the time bounds +/// (lower inclusive, upper exclusive) in (created_at, ARN) order, and by +/// store and status. +#[tokio::test] +async fn test_the_catalog_listings_are_scoped_and_ordered() { + if crate::helpers::skip_without_cassandra() { + return; + } + let engine = setup_engine().await; + let account = TestAccount::new(&engine, "extenddb_ttl_test").await; + let other = TestAccount::new(&engine, "extenddb_ttl_test").await; + let account_id = account.account_id.clone(); + let other_id = other.account_id.clone(); + let store = unique_store(); + let mut rows = Vec::new(); + for (account, table, name) in [ + (account_id.as_str(), "Music", "a"), + (account_id.as_str(), "Music", "b"), + (account_id.as_str(), "Other", "c"), + (other_id.as_str(), "Music", "d"), + ] { + tokio::time::sleep(Duration::from_millis(5)).await; + let row = index_row(account, table, name, &store); + let claim = engine.claim_backup(&row).await.expect("claim"); + if name == "b" { + assert_eq!( + claim + .set_status( + JobStatus::Creating, + JobStatus::Failed, + &StatusFields::default() + ) + .await + .unwrap(), + WriteOutcome::Matched + ); + } + rows.push((row, claim)); + } + let names = |listed: Vec| -> Vec { + listed.into_iter().map(|r| r.backup_name).collect() + }; + let all: Vec = engine + .list_account_backup_rows(&account_id, None, &TimeBounds::default()) + .try_collect() + .await + .unwrap(); + assert_eq!( + names(all.clone()), + ["a", "b", "c"], + "every status, in creation order" + ); + assert_eq!(all[1].status, BackupStatus::Failed); + let music: Vec = engine + .list_account_backup_rows(&account_id, Some("Music"), &TimeBounds::default()) + .try_collect() + .await + .unwrap(); + assert_eq!(names(music), ["a", "b"]); + let bounds = TimeBounds { + lower: Some(rows[1].0.created_at), + upper: Some(rows[2].0.created_at), + }; + let bounded: Vec = engine + .list_account_backup_rows(&account_id, None, &bounds) + .try_collect() + .await + .unwrap(); + assert_eq!(names(bounded), ["b"], "lower inclusive, upper exclusive"); + let others: Vec = engine + .list_account_backup_rows(&other_id, None, &TimeBounds::default()) + .try_collect() + .await + .unwrap(); + assert_eq!(names(others), ["d"]); + + let by_store: Vec = engine + .list_backup_rows(None, &store) + .try_collect() + .await + .unwrap(); + assert_eq!(names(by_store), ["a", "b", "c", "d"]); + let failed: Vec = engine + .list_backup_rows(Some(BackupStatus::Failed), &store) + .try_collect() + .await + .unwrap(); + assert_eq!(names(failed), ["b"]); + let elsewhere = StoreIdentity { + store_id: "store-2".to_owned(), + ..store.clone() + }; + let none: Vec = engine + .list_backup_rows(None, &elsewhere) + .try_collect() + .await + .unwrap(); + assert!(none.is_empty(), "another store's rows are not listed"); + drop(rows); +} + +/// The restore job record of a target, as `(status, kind, generation, +/// heartbeat_at ms, completed_at ms)`. +async fn job_record( + engine: &CassandraEngine, + account_id: &str, + name: &str, +) -> ( + Option, + Option, + Option, + Option, + Option, +) { + use cdrs_tokio::types::IntoRustByName; + let rows = extenddb_storage_cassandra::cassandra_util::query_rows_quorum::( + &engine.session_arc(), + &format!( + "SELECT restore_status, restore_kind, restore_owner_generation, restore_heartbeat_at, \ + restore_completed_at FROM {}.tables WHERE account_id = ? AND table_name = ?", + engine.catalog_keyspace() + ), + cdrs_tokio::query_values!(account_id, name), + "job record", + ) + .await + .expect("read the record"); + let row = rows.first().expect("the target exists"); + ( + row.get_by_name("restore_status").ok().flatten(), + row.get_by_name("restore_kind").ok().flatten(), + row.get_by_name("restore_owner_generation").ok().flatten(), + row.get_by_name("restore_heartbeat_at").ok().flatten(), + row.get_by_name("restore_completed_at").ok().flatten(), + ) +} + +/// A catalog at the shape before V006 holding a finished restore and a +/// restore the stop interrupted (its target CREATING) migrates with the +/// registrations backfilled: the finished one DONE, the interrupted one +/// RUNNING with the heartbeat at the epoch and the unowned generation, so the +/// reconciler's first pass fails it and removes its target. A second run of +/// the migration keeps the catalog id. +#[tokio::test] +async fn test_migration_v006_backfills_the_registrations_and_the_reconciler_removes_the_interrupted_one() + { + if crate::helpers::skip_without_cassandra() { + return; + } + let (engine, account_id, _drop_keyspaces) = isolated_engine("bf", false).await; + let source = create_active_table( + &engine, + &account_id, + CreateTableInput { + table_name: "BfSource".to_owned(), + key_schema: vec![ks("id", KeyType::Hash)], + attribute_definitions: vec![attr("id", ScalarAttributeType::S)], + billing_mode: Some(BillingMode::PayPerRequest), + ..Default::default() + }, + ) + .await; + put_all( + &engine, + &source, + (0..40).map(|i| item(&format!("i{i}"), None, "v")).collect(), + ) + .await; + let backup = engine + .create(&account_id, "BfSource", "b", JobPermit::detached()) + .await + .expect("backup"); + let overrides = RestoreTableOverrides::default(); + let finished = restore( + &engine, + &account_id, + &backup.backup_arn, + "BfDone", + &overrides, + ) + .await + .expect("restore"); + wait_active(&engine, &account_id, "BfDone", Duration::from_secs(60)).await; + let interrupted = restore( + &engine, + &account_id, + &backup.backup_arn, + "BfCut", + &overrides, + ) + .await + .expect("restore"); + wait_active(&engine, &account_id, "BfCut", Duration::from_secs(60)).await; + let catalog_id = engine.catalog_id().await.expect("id"); + + // Back to the shape before V006: the registrations as the previous + // release wrote them (no job record), the interrupted target CREATING, + // and the migration forgotten. + let catalog = engine.catalog_keyspace(); + for (name, table_id) in [ + ("BfDone", finished.table_id.as_str()), + ("BfCut", interrupted.table_id.as_str()), + ] { + apply( + &engine, + &format!( + "UPDATE {catalog}.tables SET restore_status = null, restore_kind = null, \ + restore_owner_generation = null, restore_heartbeat_at = null, \ + restore_completed_at = null WHERE account_id = ? AND table_name = ? \ + IF table_id = ?" + ), + cdrs_tokio::query_values!(account_id.as_str(), name, table_id), + ) + .await; + } + apply( + &engine, + &format!( + "UPDATE {catalog}.tables SET table_status = 'CREATING', status_transition_at = null \ + WHERE account_id = ? AND table_name = ? IF table_id = ?" + ), + cdrs_tokio::query_values!(account_id.as_str(), "BfCut", interrupted.table_id.as_str()), + ) + .await; + engine + .session() + .query(format!( + "DELETE FROM {catalog}.schema_history WHERE version = 6" + )) + .await + .expect("forget V006"); + extenddb_storage_cassandra::migrations::run_catalog_migrations(&engine.session_arc(), &catalog) + .await + .expect("migrate again"); + + let (status, kind, generation, heartbeat, completed) = + job_record(&engine, &account_id, "BfCut").await; + assert_eq!(status.as_deref(), Some("RUNNING")); + assert_eq!(kind.as_deref(), Some("legacy")); + assert_eq!(generation, Some(i64::try_from(UNOWNED_GENERATION).unwrap())); + assert_eq!(heartbeat, Some(0), "the epoch, not null"); + assert_eq!(completed, None); + let (status, kind, _, _, completed) = job_record(&engine, &account_id, "BfDone").await; + assert_eq!(status.as_deref(), Some("DONE")); + assert_eq!(kind.as_deref(), Some("legacy")); + #[allow(clippy::cast_possible_truncation)] + let created_ms = (finished.creation_date_time * 1_000.0).round() as i64; + assert_eq!( + completed, + Some(created_ms), + "completed when the table was created" + ); + assert_eq!( + engine.catalog_id().await.expect("id"), + catalog_id, + "a second run keeps the catalog id" + ); + + // The first reconciler pass: the interrupted restore is dead. + let dead = engine + .list_dead_restore_jobs(Duration::from_secs(300)) + .await + .expect("list"); + assert_eq!(dead.len(), 1, "{dead:?}"); + assert_eq!( + dead[0].target, + RestoreTargetKey { + account_id: account_id.clone(), + table_name: "BfCut".to_owned(), + table_id: interrupted.table_id.clone(), + } + ); + assert_eq!(dead[0].owner_generation, UNOWNED_GENERATION); + assert_eq!(dead[0].source_backup_arn, backup.backup_arn); + assert!( + engine + .fail_restore_job(&dead[0].target, "owner dead") + .await + .unwrap() + ); + assert!(engine.remove_restore_target(&dead[0].target).await.unwrap()); + assert!(matches!( + engine + .describe_table( + &account_id, + DescribeTableInput { + table_name: "BfCut".to_owned(), + }, + ) + .await, + Err(StorageError::TableNotFound(_)) + )); + assert!( + engine.table_key_info(&account_id, "BfCut").await.is_err(), + "the target's data went with it" + ); + assert_eq!( + describe(&engine, &account_id, "BfDone").await.table_status, + TableStatus::Active + ); + assert_eq!( + describe(&engine, &account_id, "BfSource") + .await + .table_status, + TableStatus::Active + ); + engine + .delete(&account_id, &backup.backup_arn) + .await + .expect("the removed target released the backup"); +} + +/// The legacy restore registers through the job record: RUNNING under the +/// claim's generation with a fresh heartbeat while the copy runs, DONE with +/// the ACTIVE flip. +#[tokio::test] +async fn test_a_legacy_restore_registers_a_running_job_record_and_finishes_it_done() { + if crate::helpers::skip_without_cassandra() { + return; + } + const ITEMS: usize = 1_500; + let engine = setup_engine().await; + let (account, _source, backup_arn) = backed_up_plain_table(&engine, "JrSource", ITEMS).await; + let account_id = account.account_id.clone(); + let target = restore( + &engine, + &account_id, + &backup_arn, + "JrTarget", + &RestoreTableOverrides::default(), + ) + .await + .expect("registers"); + let (status, kind, generation, heartbeat, completed) = + job_record(&engine, &account_id, "JrTarget").await; + assert_eq!(status.as_deref(), Some("RUNNING")); + assert_eq!(kind.as_deref(), Some("legacy")); + assert_ne!(generation, Some(i64::try_from(UNOWNED_GENERATION).unwrap())); + assert!(generation.is_some_and(|g| g > 0)); + let now = extenddb_storage_cassandra::cassandra_util::now_millis(); + let age = now - heartbeat.expect("a heartbeat"); + assert!(age.abs() < 60_000, "a fresh heartbeat: {age} ms old"); + assert_eq!(completed, None); + let key = RestoreTargetKey { + account_id: account_id.clone(), + table_name: "JrTarget".to_owned(), + table_id: target.table_id.clone(), + }; + assert!( + !engine + .list_dead_restore_jobs(Duration::from_secs(300)) + .await + .unwrap() + .iter() + .any(|job| job.target == key), + "a running restore is not dead" + ); + wait_active(&engine, &account_id, "JrTarget", Duration::from_secs(180)).await; + let (status, _, generation_after, _, completed) = + job_record(&engine, &account_id, "JrTarget").await; + assert_eq!(status.as_deref(), Some("DONE")); + assert_eq!(generation_after, generation, "DONE under the same claim"); + assert!(completed.is_some()); + assert!( + !engine.fail_restore_job(&key, "late").await.unwrap(), + "a DONE record is not RUNNING" + ); + assert!( + !engine.remove_restore_target(&key).await.unwrap(), + "an ACTIVE table is not a CREATING restore target" + ); +} + +/// An in-catalog CreateBackup whose process died leaves a CREATING row whose +/// heartbeat goes stale. `abandoned_creating_backups` reports it only once +/// the heartbeat is older than the window, exactly once (the DELETING scan +/// runs before the CREATING one, so the row it just fenced is not read +/// again), fencing the row to DELETING under a fresh generation so the dead +/// create's heartbeat no longer applies; the reader's `delete` then removes +/// the row, its list entries and every payload bucket the row did not yet +/// account for. +#[tokio::test] +async fn test_an_abandoned_in_catalog_create_is_reported_once_stale_and_deleted() { + if crate::helpers::skip_without_cassandra() { + return; + } + const ITEMS: usize = 200; + const DEAD_GENERATION: u64 = 4242; + let engine = setup_engine().await; + let (account, _source, backup_arn) = + backed_up_plain_table(&engine, "AbandonedCreate", ITEMS).await; + let account_id = account.account_id.clone(); + let abandoned = extenddb_storage::backup_catalog::AbandonedLegacyBackup { + account_id: account_id.clone(), + backup_arn: backup_arn.clone(), + }; + let reported = |stale_after: Duration| { + let engine = &engine; + let abandoned = &abandoned; + async move { + engine + .abandoned_creating_backups(stale_after) + .await + .unwrap() + .into_iter() + .filter(|row| row == abandoned) + .count() + } + }; + // A live, finished backup is not reported. + assert_eq!(reported(Duration::from_secs(60)).await, 0); + // What a create that died mid-copy leaves: a CREATING row counting no + // items yet, owned by the dead create's generation, its payload buckets + // written, its heartbeat two minutes old. + let catalog = engine.catalog_keyspace(); + let stale = extenddb_storage_cassandra::cassandra_util::now_millis() - 120_000; + apply( + &engine, + &format!( + "UPDATE {catalog}.backups_by_arn SET backup_status = 'CREATING', item_count = 0, \ + owner_generation = ? WHERE account_id = ? AND backup_arn = ? \ + IF backup_status = 'AVAILABLE'" + ), + cdrs_tokio::query_values!( + i64::try_from(DEAD_GENERATION).unwrap(), + account_id.as_str(), + backup_arn.as_str() + ), + ) + .await; + assert!( + extenddb_storage_cassandra::heartbeat_backup_row( + &engine, + &account_id, + &backup_arn, + DEAD_GENERATION + ) + .await + .unwrap(), + "until the row is fenced, the create's own heartbeat applies" + ); + apply( + &engine, + &format!( + "UPDATE {catalog}.backups_by_arn SET heartbeat_at = ? \ + WHERE account_id = ? AND backup_arn = ? IF backup_status = 'CREATING'" + ), + cdrs_tokio::query_values!(stale, account_id.as_str(), backup_arn.as_str()), + ) + .await; + assert!( + matches!( + engine.delete(&account_id, &backup_arn).await, + Err(StorageError::Validation(ref m)) if m.contains("Backup not found") + ), + "a client's delete of a backup still being created is not found" + ); + assert_eq!( + reported(Duration::from_secs(600)).await, + 0, + "a heartbeat younger than the window is not dead" + ); + assert_eq!( + reported(Duration::from_secs(60)).await, + 1, + "a heartbeat older than the window is dead, reported once" + ); + assert!( + !extenddb_storage_cassandra::heartbeat_backup_row( + &engine, + &account_id, + &backup_arn, + DEAD_GENERATION + ) + .await + .unwrap(), + "the fence took the row under a fresh generation: the dead create's heartbeat is lost" + ); + let deleted = engine + .delete(&account_id, &backup_arn) + .await + .expect("the reconciler's delete finishes the fenced row"); + assert_eq!(deleted.backup_details.backup_status, "DELETED"); + assert_eq!( + reported(Duration::from_secs(60)).await, + 0, + "a deleted row is not reported again" + ); + assert!(engine.list(&account_id, None).await.unwrap().is_empty()); + // The payload is gone, bucket by bucket past the count the row carried. + let probe = format!( + "SELECT item_index FROM {}.backup_items WHERE backup_arn = ? AND bucket = ? LIMIT 1", + engine.account_keyspace(&account_id) + ); + for bucket in 0..4_i32 { + let rows = extenddb_storage_cassandra::cassandra_util::query_rows_quorum::( + &engine.session_arc(), + &probe, + cdrs_tokio::query_values!(backup_arn.as_str(), bucket), + "probe", + ) + .await + .unwrap(); + assert!(rows.is_empty(), "bucket {bucket} still holds items"); + } +} + +/// The `restore_completed_at` of a target's job record, in epoch ms. +async fn completed_at(engine: &CassandraEngine, account_id: &str, name: &str) -> Option { + job_record(engine, account_id, name).await.4 +} + +/// The dead jobs under `stale_after` that name `key`. +async fn dead_jobs_of( + engine: &CassandraEngine, + key: &RestoreTargetKey, + stale_after: Duration, +) -> Vec { + engine + .list_dead_restore_jobs(stale_after) + .await + .expect("list dead jobs") + .into_iter() + .filter(|job| &job.target == key) + .collect() +} + +/// A restore whose FAILED mark landed but whose target removal did not (the +/// target still CREATING) is listed by `list_dead_restore_jobs` as FAILED +/// once the record is stale: its `restore_completed_at`, or its heartbeat +/// where none was stamped, older than the window. While fresh it is left to +/// the writer still on its failure path. `remove_restore_target` then removes +/// the target, with the FAILED record as it was. A FAILED record whose table +/// is ACTIVE is not listed; one whose table is DELETING with its registration +/// is. +#[tokio::test] +async fn test_a_failed_record_whose_target_removal_did_not_land_is_listed_once_stale_and_removed() { + if crate::helpers::skip_without_cassandra() { + return; + } + const ITEMS: usize = 1_500; + let engine = setup_engine().await; + let (account, _source, backup_arn) = backed_up_plain_table(&engine, "FtSource", ITEMS).await; + let account_id = account.account_id.clone(); + let catalog = engine.catalog_keyspace(); + + // A finished restore: an ACTIVE table whose record is then marked FAILED + // and aged by hand. Not a CREATING target, so never listed. + let done = restore( + &engine, + &account_id, + &backup_arn, + "FtDone", + &RestoreTableOverrides::default(), + ) + .await + .expect("restore"); + wait_active(&engine, &account_id, "FtDone", Duration::from_secs(120)).await; + let done_key = RestoreTargetKey { + account_id: account_id.clone(), + table_name: "FtDone".to_owned(), + table_id: done.table_id.clone(), + }; + apply( + &engine, + &format!( + "UPDATE {catalog}.tables SET restore_status = 'FAILED', restore_completed_at = 0 \ + WHERE account_id = ? AND table_name = ? IF table_id = ?" + ), + cdrs_tokio::query_values!(account_id.as_str(), "FtDone", done.table_id.as_str()), + ) + .await; + assert!( + dead_jobs_of(&engine, &done_key, Duration::from_secs(1)) + .await + .is_empty(), + "a FAILED record of an ACTIVE table is not listed" + ); + + // The shape the duty exists for: the owner died, the reconciler's FAILED + // write landed, the removal did not. + let target = restore( + &engine, + &account_id, + &backup_arn, + "FtTarget", + &RestoreTableOverrides::default(), + ) + .await + .expect("restore"); + orphan_restore_target(&engine, &account_id, "FtTarget", &target.table_id).await; + let key = RestoreTargetKey { + account_id: account_id.clone(), + table_name: "FtTarget".to_owned(), + table_id: target.table_id.clone(), + }; + assert!( + engine + .fail_restore_job(&key, "owner dead") + .await + .expect("fail") + ); + let failed_at = completed_at(&engine, &account_id, "FtTarget") + .await + .expect("the FAILED write stamps completed_at"); + assert!( + dead_jobs_of(&engine, &key, Duration::from_secs(60)) + .await + .is_empty(), + "a fresh FAILED record is left to the writer on its failure path" + ); + + // Two minutes later (on the record's clock): stale under a 60 s window, + // not under a 10 minute one. + let stale = failed_at - 120_000; + let set_completed = |value: Option| { + let engine = &engine; + let catalog = &catalog; + let account_id = &account_id; + let table_id = &target.table_id; + async move { + apply( + engine, + &format!( + "UPDATE {catalog}.tables SET restore_completed_at = ? \ + WHERE account_id = ? AND table_name = ? IF table_id = ?" + ), + cdrs_tokio::query_values!( + value, + account_id.as_str(), + "FtTarget", + table_id.as_str() + ), + ) + .await; + } + }; + set_completed(Some(stale)).await; + assert!( + dead_jobs_of(&engine, &key, Duration::from_secs(600)) + .await + .is_empty(), + "not stale under a window longer than its age" + ); + let listed = dead_jobs_of(&engine, &key, Duration::from_secs(60)).await; + assert_eq!(listed.len(), 1, "{listed:?}"); + assert_eq!(listed[0].status, JobStatus::Failed); + assert_eq!( + listed[0].kind, + extenddb_storage::backup_catalog::RestoreKind::Legacy + ); + assert_eq!(listed[0].source_backup_arn, backup_arn); + + // No completion stamp: the record is aged by its heartbeat (two minutes + // old since it was orphaned). + set_completed(None).await; + assert!( + dead_jobs_of(&engine, &key, Duration::from_secs(600)) + .await + .is_empty(), + "aged by the heartbeat: not stale under 10 minutes" + ); + assert_eq!( + dead_jobs_of(&engine, &key, Duration::from_secs(60)) + .await + .len(), + 1, + "aged by the heartbeat: stale under 60 s" + ); + + // A DELETING table with its registration (the removal's first write + // landed) is listed too; the removal of that state is + // `test_a_failed_record_whose_target_is_deleting_is_listed_once_stale_and_its_removal_resumed`. + apply( + &engine, + &format!( + "UPDATE {catalog}.tables SET table_status = 'DELETING' \ + WHERE account_id = ? AND table_name = ? IF table_id = ?" + ), + cdrs_tokio::query_values!(account_id.as_str(), "FtTarget", target.table_id.as_str()), + ) + .await; + assert_eq!( + dead_jobs_of(&engine, &key, Duration::from_secs(60)) + .await + .len(), + 1, + "a FAILED record of a DELETING table with its registration is listed" + ); + apply( + &engine, + &format!( + "UPDATE {catalog}.tables SET table_status = 'CREATING' \ + WHERE account_id = ? AND table_name = ? IF table_id = ?" + ), + cdrs_tokio::query_values!(account_id.as_str(), "FtTarget", target.table_id.as_str()), + ) + .await; + + // The duty: the target removed, the FAILED record not written again. + assert!( + !engine + .fail_restore_job(&key, "again") + .await + .expect("fail again"), + "a FAILED record is not RUNNING: no second FAILED write" + ); + assert!( + engine.remove_restore_target(&key).await.expect("remove"), + "the target was a CREATING restore target" + ); + assert!(matches!( + engine + .describe_table( + &account_id, + DescribeTableInput { + table_name: "FtTarget".to_owned(), + }, + ) + .await, + Err(StorageError::TableNotFound(_)) + )); + assert!( + dead_jobs_of(&engine, &key, Duration::from_secs(60)) + .await + .is_empty(), + "a removed target has no record" + ); + engine + .delete(&account_id, &backup_arn) + .await + .expect("the removed target released the backup"); + engine + .delete_table( + &account_id, + DeleteTableInput { + table_name: "FtDone".to_owned(), + }, + ) + .await + .expect("delete the finished target"); +} + +/// The removal of a FAILED record's target is two writes: the claim from +/// CREATING to DELETING, then the drop. A target the claim moved to DELETING +/// whose drop then failed (the row DELETING with its registration and FAILED +/// record) is listed by `list_dead_restore_jobs` as FAILED once the record is +/// stale, left alone while fresh, and `remove_restore_target` resumes the +/// removal from the drop: the table, its data table and its record are gone, +/// the backup is released and the name is free. Three DELETING rows the +/// resume rule must leave alone: a restored table with a DONE record that a +/// client is deleting (its registration still on the row), which is neither +/// listed nor removed and keeps its data; the DELETING+FAILED row named by a +/// key with another `table_id`, on which `remove_restore_target` drops +/// nothing; and a DELETING row without a registration (a client's +/// DeleteTable in flight), neither listed nor removed. +#[tokio::test] +async fn test_a_failed_record_whose_target_is_deleting_is_listed_once_stale_and_its_removal_resumed() + { + if crate::helpers::skip_without_cassandra() { + return; + } + const ITEMS: usize = 1_500; + let engine = setup_engine().await; + let (account, _source, backup_arn) = backed_up_plain_table(&engine, "PaSource", ITEMS).await; + let account_id = account.account_id.clone(); + let catalog = engine.catalog_keyspace(); + + // A restored table, ACTIVE with a DONE record, that a client's + // DeleteTable moved to DELETING: the registration is on the row and the + // record is not FAILED, so the row belongs to that delete. + let done = restore( + &engine, + &account_id, + &backup_arn, + "PaDone", + &RestoreTableOverrides::default(), + ) + .await + .expect("restore the table a client deletes"); + wait_active(&engine, &account_id, "PaDone", Duration::from_secs(60)).await; + let done_key = RestoreTargetKey { + account_id: account_id.clone(), + table_name: "PaDone".to_owned(), + table_id: done.table_id.clone(), + }; + assert_eq!( + job_record(&engine, &account_id, "PaDone") + .await + .0 + .as_deref(), + Some("DONE"), + "the activated restore's record is DONE" + ); + apply( + &engine, + &format!( + "UPDATE {catalog}.tables SET table_status = 'DELETING' \ + WHERE account_id = ? AND table_name = ? IF table_id = ? AND table_status = 'ACTIVE'" + ), + cdrs_tokio::query_values!(account_id.as_str(), "PaDone", done.table_id.as_str()), + ) + .await; + let done_data_table = format!("items_{}", done.table_id.replace('-', "_")); + assert!( + dead_jobs_of(&engine, &done_key, Duration::from_secs(1)) + .await + .is_empty(), + "a DONE record of a DELETING table is not listed" + ); + assert!( + !engine + .remove_restore_target(&done_key) + .await + .expect("remove a restored table a client is deleting"), + "a DELETING restored table whose record is DONE belongs to the client's delete" + ); + assert!( + data_table_exists(&engine, &account_id, &done_data_table).await, + "the client's delete keeps the restored table's data" + ); + apply( + &engine, + &format!( + "UPDATE {catalog}.tables SET table_status = 'ACTIVE' \ + WHERE account_id = ? AND table_name = ? IF table_id = ?" + ), + cdrs_tokio::query_values!(account_id.as_str(), "PaDone", done.table_id.as_str()), + ) + .await; + engine + .delete_table( + &account_id, + DeleteTableInput { + table_name: "PaDone".to_owned(), + }, + ) + .await + .expect("delete the restored table"); + + let target = restore( + &engine, + &account_id, + &backup_arn, + "PaTarget", + &RestoreTableOverrides::default(), + ) + .await + .expect("restore"); + orphan_restore_target(&engine, &account_id, "PaTarget", &target.table_id).await; + let key = RestoreTargetKey { + account_id: account_id.clone(), + table_name: "PaTarget".to_owned(), + table_id: target.table_id.clone(), + }; + assert!( + engine + .fail_restore_job(&key, "owner dead") + .await + .expect("fail") + ); + let failed_at = completed_at(&engine, &account_id, "PaTarget") + .await + .expect("the FAILED write stamps completed_at"); + // The removal's claim landed and its drop did not. + apply( + &engine, + &format!( + "UPDATE {catalog}.tables SET table_status = 'DELETING' \ + WHERE account_id = ? AND table_name = ? IF table_id = ? AND table_status = 'CREATING'" + ), + cdrs_tokio::query_values!(account_id.as_str(), "PaTarget", target.table_id.as_str()), + ) + .await; + assert!( + dead_jobs_of(&engine, &key, Duration::from_secs(60)) + .await + .is_empty(), + "a fresh FAILED record of a DELETING target is left to the remover on its path" + ); + + apply( + &engine, + &format!( + "UPDATE {catalog}.tables SET restore_completed_at = ? \ + WHERE account_id = ? AND table_name = ? IF table_id = ?" + ), + cdrs_tokio::query_values!( + failed_at - 120_000, + account_id.as_str(), + "PaTarget", + target.table_id.as_str() + ), + ) + .await; + assert!( + dead_jobs_of(&engine, &key, Duration::from_secs(600)) + .await + .is_empty(), + "not stale under a window longer than its age" + ); + let listed = dead_jobs_of(&engine, &key, Duration::from_secs(60)).await; + assert_eq!( + listed.len(), + 1, + "a stale FAILED record of a DELETING target with its registration is listed: {listed:?}" + ); + assert_eq!(listed[0].status, JobStatus::Failed); + assert_eq!(listed[0].source_backup_arn, backup_arn); + + // The duty resumes the drop, on the row's own table id only: a key with + // another id (a table re-created under the name) touches nothing. + let data_table = format!("items_{}", target.table_id.replace('-', "_")); + assert!( + data_table_exists(&engine, &account_id, &data_table).await, + "the target's data table exists before the removal" + ); + let other_id_key = RestoreTargetKey { + table_id: "not-this-id".to_owned(), + ..key.clone() + }; + assert!( + !engine + .remove_restore_target(&other_id_key) + .await + .expect("remove under another table id"), + "a key naming another table id removes nothing from a DELETING target" + ); + assert!( + data_table_exists(&engine, &account_id, &data_table).await, + "the data table is kept under a key with another table id" + ); + assert_eq!( + dead_jobs_of(&engine, &key, Duration::from_secs(60)) + .await + .len(), + 1, + "the record is still listed after the mismatched removal" + ); + assert!( + engine.remove_restore_target(&key).await.expect("remove"), + "a DELETING target with its registration and FAILED record is removed" + ); + assert!(matches!( + engine + .describe_table( + &account_id, + DescribeTableInput { + table_name: "PaTarget".to_owned(), + }, + ) + .await, + Err(StorageError::TableNotFound(_)) + )); + assert!( + !data_table_exists(&engine, &account_id, &data_table).await, + "the target's data table is dropped" + ); + assert!( + dead_jobs_of(&engine, &key, Duration::from_secs(60)) + .await + .is_empty(), + "a removed target has no record" + ); + assert!( + !engine + .remove_restore_target(&key) + .await + .expect("remove again"), + "nothing left to remove" + ); + // The name is free, and the backup released. + let reused = create_active_table( + &engine, + &account_id, + CreateTableInput { + table_name: "PaTarget".to_owned(), + key_schema: vec![ks("id", KeyType::Hash)], + attribute_definitions: vec![attr("id", ScalarAttributeType::S)], + billing_mode: Some(BillingMode::PayPerRequest), + ..Default::default() + }, + ) + .await; + assert_ne!(reused.table_id, target.table_id); + engine + .delete(&account_id, &backup_arn) + .await + .expect("the removed target released the backup"); + + // A DELETING row without a registration is a client's DeleteTable in + // flight: not a restore target, so not listed and not removed. + let plain_key = RestoreTargetKey { + account_id: account_id.clone(), + table_name: "PaTarget".to_owned(), + table_id: reused.table_id.clone(), + }; + apply( + &engine, + &format!( + "UPDATE {catalog}.tables SET table_status = 'DELETING' \ + WHERE account_id = ? AND table_name = ? IF table_id = ? AND table_status = 'ACTIVE'" + ), + cdrs_tokio::query_values!(account_id.as_str(), "PaTarget", reused.table_id.as_str()), + ) + .await; + assert!( + dead_jobs_of(&engine, &plain_key, Duration::from_secs(1)) + .await + .is_empty(), + "a DELETING table that is not a restore target is not listed" + ); + assert!( + !engine + .remove_restore_target(&plain_key) + .await + .expect("remove a plain table"), + "a DELETING table without a registration is not removed" + ); + assert!( + data_table_exists( + &engine, + &account_id, + &format!("items_{}", reused.table_id.replace('-', "_")) + ) + .await, + "the client's delete keeps its table" + ); + apply( + &engine, + &format!( + "UPDATE {catalog}.tables SET table_status = 'ACTIVE' \ + WHERE account_id = ? AND table_name = ? IF table_id = ?" + ), + cdrs_tokio::query_values!(account_id.as_str(), "PaTarget", reused.table_id.as_str()), + ) + .await; + engine + .delete_table( + &account_id, + DeleteTableInput { + table_name: "PaTarget".to_owned(), + }, + ) + .await + .expect("delete the re-created table"); +} + +/// Whether `table` exists in `account_id`'s keyspace. +async fn data_table_exists(engine: &CassandraEngine, account_id: &str, table: &str) -> bool { + !extenddb_storage_cassandra::cassandra_util::query_rows::( + &engine.session_arc(), + "SELECT table_name FROM system_schema.tables WHERE keyspace_name = ? AND table_name = ?", + cdrs_tokio::query_values!(engine.account_keyspace(account_id), table), + "schema", + ) + .await + .expect("read the schema") + .is_empty() +} + +#[cfg(feature = "test-hooks")] +/// The write times of a `backups_by_arn` row's `heartbeat_at` and of a column +/// only the insert writes (`backup_name`), as `(heartbeat, inserted)`, for +/// the one row of `account_id`, with the row's status. +async fn create_row_write_times( + engine: &CassandraEngine, + account_id: &str, +) -> Option<(Option, Option, String)> { + use cdrs_tokio::types::IntoRustByName; + let rows = extenddb_storage_cassandra::cassandra_util::query_rows_quorum::( + &engine.session_arc(), + &format!( + "SELECT WRITETIME(heartbeat_at) AS beat, WRITETIME(backup_name) AS inserted, \ + backup_status FROM {}.backups_by_arn WHERE account_id = ? ALLOW FILTERING", + engine.catalog_keyspace() + ), + cdrs_tokio::query_values!(account_id), + "create row", + ) + .await + .expect("read the create's row"); + let row = rows.first()?; + Some(( + row.get_by_name("beat").ok().flatten(), + row.get_by_name("inserted").ok().flatten(), + row.get_r_by_name("backup_status").expect("status"), + )) +} + +#[cfg(feature = "test-hooks")] +/// The in-catalog create heartbeats its CREATING row while the items copy: +/// `heartbeat_at` is written again after the insert, while the row is still +/// CREATING, and the published row carries the last beat. +#[tokio::test] +async fn test_the_in_catalog_create_heartbeats_its_row_while_the_items_copy() { + if crate::helpers::skip_without_cassandra() { + return; + } + const ITEMS: usize = 3_000; + let engine = std::sync::Arc::new(setup_engine().await); + let account = TestAccount::new(&engine, "extenddb_ttl_test").await; + let account_id = account.account_id.clone(); + extenddb_storage_cassandra::test_hooks::set_heartbeat_interval( + &account_id, + Duration::from_millis(100), + ); + let key_info = create_active_table( + &engine, + &account_id, + CreateTableInput { + table_name: "Beating".to_owned(), + key_schema: vec![ks("id", KeyType::Hash)], + attribute_definitions: vec![attr("id", ScalarAttributeType::S)], + billing_mode: Some(BillingMode::PayPerRequest), + ..Default::default() + }, + ) + .await; + put_all( + &engine, + &key_info, + (0..ITEMS) + .map(|i| item(&format!("item-{i:05}"), None, &"x".repeat(200))) + .collect(), + ) + .await; + + // Watch the row while the create runs: the first time its heartbeat's + // write time passes the insert's while the row is still CREATING. + let watching = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(true)); + let watcher = { + let engine = std::sync::Arc::clone(&engine); + let account_id = account_id.clone(); + let watching = std::sync::Arc::clone(&watching); + tokio::spawn(async move { + let mut refreshed_while_creating = false; + while watching.load(std::sync::atomic::Ordering::Acquire) && !refreshed_while_creating { + if let Some((Some(beat), Some(inserted), status)) = + create_row_write_times(&engine, &account_id).await + && beat > inserted + && status == "CREATING" + { + refreshed_while_creating = true; + } + tokio::time::sleep(Duration::from_millis(20)).await; + } + refreshed_while_creating + }) + }; + let created = engine + .create(&account_id, "Beating", "beats", JobPermit::detached()) + .await + .expect("create backup"); + watching.store(false, std::sync::atomic::Ordering::Release); + assert!( + watcher.await.expect("watcher"), + "heartbeat_at was written again while the row was CREATING" + ); + let (beat, inserted, status) = create_row_write_times(&engine, &account_id) + .await + .expect("the published row"); + assert_eq!(status, "AVAILABLE"); + assert!( + beat.zip(inserted) + .is_some_and(|(beat, inserted)| beat > inserted), + "the published row carries the last beat: {beat:?} vs {inserted:?}" + ); + engine + .delete(&account_id, &created.backup_arn) + .await + .expect("delete backup"); +} + +#[cfg(feature = "test-hooks")] +/// Items of `backup_arn`'s payload in `account_id`'s keyspace, all buckets. +async fn payload_item_count(engine: &CassandraEngine, account_id: &str, backup_arn: &str) -> i64 { + use cdrs_tokio::types::IntoRustByName; + let rows = extenddb_storage_cassandra::cassandra_util::query_rows_quorum::( + &engine.session_arc(), + &format!( + "SELECT COUNT(*) AS n FROM {}.backup_items WHERE backup_arn = ? ALLOW FILTERING", + engine.account_keyspace(account_id) + ), + cdrs_tokio::query_values!(backup_arn), + "payload count", + ) + .await + .expect("count the payload"); + rows.first() + .and_then(|row| row.get_by_name("n").ok().flatten()) + .unwrap_or(0) +} + +/// The `backup_status` of the `backups_by_arn` row `backup_arn`, if there is +/// one. +#[cfg(feature = "test-hooks")] +async fn backup_status_of( + engine: &CassandraEngine, + account_id: &str, + backup_arn: &str, +) -> Option { + use cdrs_tokio::types::IntoRustByName; + extenddb_storage_cassandra::cassandra_util::query_rows_quorum::( + &engine.session_arc(), + &format!( + "SELECT backup_status FROM {}.backups_by_arn WHERE account_id = ? AND backup_arn = ?", + engine.catalog_keyspace() + ), + cdrs_tokio::query_values!(account_id, backup_arn), + "backup status", + ) + .await + .expect("read the row") + .first() + .and_then(|row| row.get_by_name("backup_status").ok().flatten()) +} + +#[cfg(feature = "test-hooks")] +/// Three heartbeat errors in a row stop an in-catalog create within the +/// bucket it is writing: the create fails naming the unconfirmed claim, its +/// row stays CREATING for the reconciler, and the payload ends at a bucket +/// boundary short of the table. Two errors followed by a confirmed beat do +/// not stop it. +#[tokio::test] +async fn test_three_heartbeat_errors_stop_a_create_within_a_bucket_and_two_do_not() { + if crate::helpers::skip_without_cassandra() { + return; + } + const ITEMS: i64 = 3_000; + const BUCKET: i64 = 64; + let engine = std::sync::Arc::new(setup_engine().await); + let account = TestAccount::new(&engine, "extenddb_ttl_test").await; + let account_id = account.account_id.clone(); + extenddb_storage_cassandra::test_hooks::set_heartbeat_interval( + &account_id, + Duration::from_millis(10), + ); + let key_info = create_active_table( + &engine, + &account_id, + CreateTableInput { + table_name: "Erring".to_owned(), + key_schema: vec![ks("id", KeyType::Hash)], + attribute_definitions: vec![attr("id", ScalarAttributeType::S)], + billing_mode: Some(BillingMode::PayPerRequest), + ..Default::default() + }, + ) + .await; + put_all( + &engine, + &key_info, + (0..ITEMS) + .map(|i| item(&format!("item-{i:05}"), None, &"x".repeat(200))) + .collect(), + ) + .await; + + // The create starts with confirmed beats; once its first bucket is on + // disk, the next three beats fail. + let create = { + let engine = std::sync::Arc::clone(&engine); + let account_id = account_id.clone(); + tokio::spawn(async move { + engine + .create(&account_id, "Erring", "errs", JobPermit::detached()) + .await + }) + }; + let backup_arn = first_written_backup_arn(&engine, &account_id).await; + extenddb_storage_cassandra::test_hooks::fail_heartbeats(&account_id, 3); + let error = create + .await + .expect("create task") + .expect_err("three heartbeat errors in a row stop the create"); + assert!( + matches!(error, StorageError::Internal(ref m) if m.contains("could not be confirmed")), + "{error}" + ); + assert_eq!( + backup_status_of(&engine, &account_id, &backup_arn).await, + Some("CREATING".to_owned()), + "the row is left for the reconciler" + ); + let written = payload_item_count(&engine, &account_id, &backup_arn).await; + assert!( + written > 0 && written < ITEMS, + "the copy stopped part way: {written} of {ITEMS}" + ); + assert_eq!( + written % BUCKET, + 0, + "the copy polls its claim per bucket, so it stopped at a bucket boundary: {written}" + ); + // The reconciler finishes the row once its heartbeat is stale. + let stale = extenddb_storage_cassandra::cassandra_util::now_millis() - 120_000; + apply( + &engine, + &format!( + "UPDATE {}.backups_by_arn SET heartbeat_at = ? \ + WHERE account_id = ? AND backup_arn = ? IF backup_status = 'CREATING'", + engine.catalog_keyspace() + ), + cdrs_tokio::query_values!(stale, account_id.as_str(), backup_arn.as_str()), + ) + .await; + assert_eq!( + engine + .abandoned_creating_backups(Duration::from_secs(60)) + .await + .unwrap() + .iter() + .filter(|row| row.backup_arn == backup_arn) + .count(), + 1 + ); + engine + .delete(&account_id, &backup_arn) + .await + .expect("the reconciler's delete finishes the stopped create"); + assert_eq!( + payload_item_count(&engine, &account_id, &backup_arn).await, + 0 + ); + + // Two errors, then confirmed beats: the create finishes. The errors are + // armed once the copy is writing, and both are seen taken by a beat + // while the payload is still short of the table, so the create met two + // errors in a row where three would have stopped it. + let create = { + let engine = std::sync::Arc::clone(&engine); + let account_id = account_id.clone(); + tokio::spawn(async move { + engine + .create(&account_id, "Erring", "two-errors", JobPermit::detached()) + .await + }) + }; + let second_arn = first_written_backup_arn(&engine, &account_id).await; + extenddb_storage_cassandra::test_hooks::fail_heartbeats(&account_id, 2); + let deadline = Instant::now() + Duration::from_secs(30); + while extenddb_storage_cassandra::test_hooks::pending_heartbeat_errors(&account_id) > 0 { + assert!( + Instant::now() < deadline, + "the two erring beats never fired" + ); + tokio::time::sleep(Duration::from_millis(2)).await; + } + let written_at_second_error = payload_item_count(&engine, &account_id, &second_arn).await; + assert!( + written_at_second_error < ITEMS, + "both erring beats fired while the copy was still writing: {written_at_second_error} \ + of {ITEMS}" + ); + let created = create + .await + .expect("create task") + .expect("two heartbeat errors do not stop a create"); + assert_eq!(created.backup_arn, second_arn); + assert_eq!(created.backup_status, "AVAILABLE"); + assert_eq!( + payload_item_count(&engine, &account_id, &created.backup_arn).await, + ITEMS + ); + engine + .delete(&account_id, &created.backup_arn) + .await + .expect("delete backup"); +} + +#[cfg(feature = "test-hooks")] +/// Wait for the first `backup_items` row in `account_id`'s keyspace and +/// return its backup ARN: the create whose copy has begun writing. +async fn first_written_backup_arn(engine: &CassandraEngine, account_id: &str) -> String { + let deadline = Instant::now() + Duration::from_secs(30); + loop { + assert!( + Instant::now() < deadline, + "the create never started writing" + ); + if let Some(arn) = extenddb_storage_cassandra::cassandra_util::query_rows::( + &engine.session_arc(), + &format!( + "SELECT backup_arn FROM {}.backup_items LIMIT 1", + engine.account_keyspace(account_id) + ), + cdrs_tokio::query_values!(), + "first item", + ) + .await + .expect("probe") + .first() + .map(|row| { + extenddb_storage_cassandra::cassandra_util::get_column::( + row, + "backup_arn", + "first item", + ) + .unwrap() + }) { + return arn; + } + tokio::time::sleep(Duration::from_millis(5)).await; + } +} + +#[cfg(feature = "test-hooks")] +/// A restore that registers on the backup between DeleteBackup's in-use read +/// and its DELETING claim is what the claim's own `active_restores = null` +/// condition is for: the claim does not apply, the delete answers +/// BackupInUse naming the target, and the backup stays AVAILABLE; the same +/// for `start_backup_delete` on a catalog index row. +#[tokio::test] +async fn test_a_restore_registering_between_the_in_use_read_and_the_claim_keeps_the_backup() { + if crate::helpers::skip_without_cassandra() { + return; + } + const ITEMS: usize = 1_500; + let engine = std::sync::Arc::new(setup_engine().await); + let (account, _source, backup_arn) = backed_up_plain_table(&engine, "RaceSource", ITEMS).await; + let account_id = account.account_id.clone(); + + // The legacy delete, paused after its in-use read found no restore. + let pause = extenddb_storage_cassandra::test_hooks::pause_before_delete_claim(&backup_arn); + let delete = { + let engine = std::sync::Arc::clone(&engine); + let (account_id, backup_arn) = (account_id.clone(), backup_arn.clone()); + tokio::spawn(async move { engine.delete(&account_id, &backup_arn).await }) + }; + pause.reached().await; + let target = restore( + &engine, + &account_id, + &backup_arn, + "RaceTarget", + &RestoreTableOverrides::default(), + ) + .await + .expect("the restore registers while the delete is between its read and its claim"); + assert_eq!(target.table_status, TableStatus::Creating); + pause.release(); + let refused = delete + .await + .expect("delete task") + .expect_err("the claim's active_restores = null condition refuses the delete"); + assert!( + matches!(refused, StorageError::BackupInUse(ref m) if m.contains("RaceTarget")), + "{refused}" + ); + assert_eq!( + backup_status_of(&engine, &account_id, &backup_arn).await, + Some("AVAILABLE".to_owned()), + "a refused delete leaves the row" + ); + wait_active(&engine, &account_id, "RaceTarget", Duration::from_secs(120)).await; + engine + .delete(&account_id, &backup_arn) + .await + .expect("the finished restore released the backup"); + + // `start_backup_delete` on an index row: the registration lands in the + // same gap; the claim answers InUse and the row stays AVAILABLE. + let store = unique_store(); + let row = index_row(&account_id, "RaceSource", "nightly", &store); + let key = row.key(); + let claim = engine.claim_backup(&row).await.expect("claim"); + assert_eq!( + claim + .set_status( + JobStatus::Creating, + JobStatus::Available, + &StatusFields::default() + ) + .await + .unwrap(), + WriteOutcome::Matched + ); + let pause = extenddb_storage_cassandra::test_hooks::pause_before_delete_claim(&row.backup_arn); + let start = { + let engine = std::sync::Arc::clone(&engine); + let key = key.clone(); + tokio::spawn(async move { engine.start_backup_delete(&key).await }) + }; + pause.reached().await; + let catalog = engine.catalog_keyspace(); + let registered = create_active_table( + &engine, + &account_id, + CreateTableInput { + table_name: "RaceIndexTarget".to_owned(), + key_schema: vec![ks("id", KeyType::Hash)], + attribute_definitions: vec![attr("id", ScalarAttributeType::S)], + billing_mode: Some(BillingMode::PayPerRequest), + ..Default::default() + }, + ) + .await; + apply( + &engine, + &format!( + "UPDATE {catalog}.tables SET table_status = 'CREATING', status_transition_at = null, \ + restore_source_backup_arn = ? WHERE account_id = ? AND table_name = ? IF table_id = ?" + ), + cdrs_tokio::query_values!( + row.backup_arn.as_str(), + account_id.as_str(), + "RaceIndexTarget", + registered.table_id.as_str() + ), + ) + .await; + apply( + &engine, + &format!( + "UPDATE {catalog}.backups_by_arn SET active_restores[?] = ? \ + WHERE account_id = ? AND backup_arn = ? IF backup_status = 'AVAILABLE'" + ), + cdrs_tokio::query_values!( + registered.table_id.as_str(), + "RaceIndexTarget", + account_id.as_str(), + row.backup_arn.as_str() + ), + ) + .await; + pause.release(); + assert_eq!( + start.await.expect("start task").unwrap(), + DeleteStart::InUse("RaceIndexTarget".to_owned()) + ); + assert_eq!( + engine + .read_backup_row(&arn_of(&row)) + .await + .unwrap() + .unwrap() + .status, + BackupStatus::Available, + "a refused delete leaves the row AVAILABLE" + ); + apply( + &engine, + &format!( + "UPDATE {catalog}.tables SET table_status = 'ACTIVE' \ + WHERE account_id = ? AND table_name = ? IF table_id = ?" + ), + cdrs_tokio::query_values!( + account_id.as_str(), + "RaceIndexTarget", + registered.table_id.as_str() + ), + ) + .await; + assert_eq!( + engine.start_backup_delete(&key).await.unwrap(), + DeleteStart::Deleting + ); + assert!(engine.set_backup_deleted(&key).await.unwrap()); + assert!(engine.remove_backup_row(&key).await.unwrap()); + drop(claim); + engine + .delete_table( + &account_id, + DeleteTableInput { + table_name: "RaceIndexTarget".to_owned(), + }, + ) + .await + .expect("delete the target"); +} + +/// A V005 catalog whose V006 step fails (a `tables` row carrying a restore +/// registration and no `table_id`, which the backfill cannot key) stays at +/// 0.0.4 with V006 unrecorded, so `serve` refuses it and `migrate` runs the +/// file again; once the cause is gone the second run lands at 0.0.5 with +/// V006 recorded. A fresh catalog lands there in one run. +#[tokio::test] +async fn test_a_migration_that_fails_in_its_step_leaves_the_version_behind_and_resumes() { + if crate::helpers::skip_without_cassandra() { + return; + } + async fn catalog_state(engine: &CassandraEngine, catalog: &str) -> (Option, Vec) { + use cdrs_tokio::types::IntoRustByName; + let session = engine.session_arc(); + let version = extenddb_storage_cassandra::cassandra_util::query_rows::( + &session, + &format!("SELECT value FROM {catalog}.settings WHERE key = 'catalog_version'"), + cdrs_tokio::query_values!(), + "version", + ) + .await + .unwrap() + .first() + .and_then(|row| { + let value: Option = row.get_by_name("value").ok().flatten(); + value + }); + let mut recorded: Vec = + extenddb_storage_cassandra::cassandra_util::query_rows::( + &session, + &format!("SELECT version FROM {catalog}.schema_history"), + cdrs_tokio::query_values!(), + "history", + ) + .await + .unwrap() + .iter() + .map(|row| { + let version: i32 = row.get_r_by_name("version").unwrap(); + version + }) + .collect(); + recorded.sort_unstable(); + (version, recorded) + } + + let (fresh, _account, _drop_fresh) = isolated_engine("mv", false).await; + assert_eq!( + catalog_state(&fresh, &fresh.catalog_keyspace()).await, + ( + Some(extenddb_storage_cassandra::CATALOG_VERSION.to_owned()), + vec![1, 2, 3, 4, 5, 6] + ), + "a fresh catalog is at the binary's version with every file recorded" + ); + + let (engine, account_id, _drop_keyspaces) = isolated_engine("mf", true).await; + let catalog = engine.catalog_keyspace(); + assert_eq!( + catalog_state(&engine, &catalog).await, + (Some("0.0.4".to_owned()), vec![1, 2, 3, 4, 5]) + ); + // A registration the step cannot key. + engine + .session() + .query_with_values( + format!( + "INSERT INTO {catalog}.tables (account_id, table_name, table_status, \ + restore_source_backup_arn) VALUES (?, 'Unkeyed', 'ACTIVE', 'arn:aws:dynamodb:us-east-1:000000000000:table/T/backup/01700000000000-abc')" + ), + cdrs_tokio::query_values!(account_id.as_str()), + ) + .await + .expect("a row without a table_id"); + let _failed = extenddb_storage_cassandra::migrations::run_catalog_migrations( + &engine.session_arc(), + &catalog, + ) + .await + .expect_err("the step fails on the unkeyed row"); + assert_eq!( + catalog_state(&engine, &catalog).await, + (Some("0.0.4".to_owned()), vec![1, 2, 3, 4, 5]), + "a failed step leaves the version and the history where they were" + ); + let refusal = engine + .require_served_catalog() + .await + .expect_err("serve refuses the half-migrated catalog") + .to_string(); + assert!( + refusal.contains("at version 0.0.4") && refusal.contains("extenddb migrate --yes"), + "{refusal}" + ); + assert!( + matches!( + engine.create(&account_id, "Unkeyed", "b", JobPermit::detached()).await, + Err(StorageError::Unsupported(ref m)) if m.contains("V006__backup_catalog_index.cql") + ), + "a backup operation is refused until the migration is recorded" + ); + + // The cause gone, the next run applies the file again and lands. + engine + .session() + .query_with_values( + format!("DELETE FROM {catalog}.tables WHERE account_id = ? AND table_name = 'Unkeyed'"), + cdrs_tokio::query_values!(account_id.as_str()), + ) + .await + .expect("remove the row"); + extenddb_storage_cassandra::migrations::run_catalog_migrations(&engine.session_arc(), &catalog) + .await + .expect("the second run succeeds"); + assert_eq!( + catalog_state(&engine, &catalog).await, + ( + Some(extenddb_storage_cassandra::CATALOG_VERSION.to_owned()), + vec![1, 2, 3, 4, 5, 6] + ) + ); + engine + .require_served_catalog() + .await + .expect("serve accepts the migrated catalog"); + assert!( + uuid::Uuid::parse_str(&engine.catalog_id().await.expect("catalog id")).is_ok(), + "the step wrote the catalog id" + ); +} diff --git a/crates/storage-mongodb/src/backup_catalog.rs b/crates/storage-mongodb/src/backup_catalog.rs new file mode 100644 index 000000000..6633f8486 --- /dev/null +++ b/crates/storage-mongodb/src/backup_catalog.rs @@ -0,0 +1,920 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! The catalog index on MongoDB: the `backup_index` collection, the restore +//! job record (`restore_job`) on a restore target's `tables` document, the +//! `backup_maintenance` collection and the `catalog_id` settings document. +//! +//! The claim of a backup job is the row itself: it is born owned by an +//! `update_one` with `upsert: true`, a filter on `owner_generation: {$exists: +//! false}` and an update pipeline that sets the generation and `heartbeat_at: +//! "$$NOW"`, so no row ever exists unowned and a second claim of the same +//! backup trips the unique index on (`account_id`, `backup_id`). Every write +//! the job makes carries its generation in the filter; a heartbeat that +//! matches other than exactly one document means the claim is lost. +//! +//! The lock on a row for DeleteBackup is a session transaction that `$inc`s +//! `lock_seq` on the row before it reads the status and the in-use rule, so +//! two such transactions on one row conflict and order wholly one before the +//! other; a restore's registration over a `backup_index` row takes the same +//! lock. Staleness is measured on the server's clock (`$$NOW`). +//! +//! A conditional write whose error is the driver's network class +//! (`ErrorKind::Io`, `ConnectionPoolCleared`) has an unknown outcome +//! (`WriteOutcome::Unknown`: the statement may have reached the server); +//! a server error (a command or write error) is `Err`, as a statement error +//! is on PostgreSQL. + +use std::time::Duration; + +use extenddb_core::backup_format::BackupArn; +use extenddb_storage::backup_catalog::{ + BackupCatalog, BackupKey, BackupRow, BackupStatus, DeleteStart, Heartbeat, JobClaim, JobRow, + JobStatus, RestoreJobRow, RestoreKind, RestoreTargetKey, StatusFields, StoreIdentity, + StoreKind, TimeBounds, WriteOutcome, new_generation, +}; +use extenddb_storage::error::StorageError; +use futures::TryStreamExt; +use futures::future::BoxFuture; +use futures::stream::BoxStream; +use mongodb::bson::{self, Bson, Document, doc}; + +use crate::MongoEngine; +use crate::backup_engine::{age_is_stale, restoring_target_filter}; +use crate::data_engine::{ + CommitOutcome, TRANSIENT_RETRY_ATTEMPTS, TxErr, backoff_sleep, is_duplicate_key, +}; + +/// The catalog collection of backup rows. +pub(crate) const BACKUP_INDEX: &str = "backup_index"; +/// The catalog collection of the maintenance claim record. +pub(crate) const BACKUP_MAINTENANCE: &str = "backup_maintenance"; +/// The settings document holding the catalog's random id. +pub(crate) const CATALOG_ID: &str = "catalog_id"; +/// The counter a row-locking transaction `$inc`s so two of them conflict. +pub(crate) const LOCK_SEQ: &str = "lock_seq"; + +/// The restore job record, a subdocument of a restore target's `tables` +/// document, with the field names the job record has on every backend. +pub(crate) const RESTORE_JOB: &str = "restore_job"; +pub(crate) const RESTORE_JOB_STATUS: &str = "restore_job.status"; +pub(crate) const RESTORE_JOB_GENERATION: &str = "restore_job.owner_generation"; +pub(crate) const RESTORE_JOB_HEARTBEAT: &str = "restore_job.heartbeat_at"; +pub(crate) const RESTORE_JOB_FAILURE_REASON: &str = "restore_job.failure_reason"; +pub(crate) const RESTORE_JOB_COMPLETED_AT: &str = "restore_job.completed_at"; + +fn internal(e: impl std::fmt::Display) -> StorageError { + StorageError::Internal(e.to_string()) +} + +fn not_found(backup_arn: &str) -> StorageError { + extenddb_storage::backup_catalog::backup_not_found(backup_arn) +} + +/// The error for a backup a CREATING restore target still reads: the shared +/// text, [`extenddb_storage::backup_catalog::backup_in_use`]. +pub(crate) fn backup_in_use(table: &str, backup_arn: &str) -> StorageError { + extenddb_storage::backup_catalog::backup_in_use(table, backup_arn) +} + +/// An `owner_generation` as the catalog stores it (a 64-bit integer); +/// generations are minted below `i64::MAX`. +pub(crate) fn generation_field(generation: u64) -> i64 { + i64::try_from(generation).unwrap_or(i64::MAX) +} + +fn generation_from_field(doc: &Document, key: &str) -> u64 { + match doc.get(key) { + Some(Bson::Int64(g)) => u64::try_from(*g).unwrap_or_default(), + Some(Bson::Int32(g)) => u64::try_from(*g).unwrap_or_default(), + Some(Bson::Double(g)) => { + #[allow(clippy::cast_possible_truncation, clippy::cast_sign_loss)] + let g = *g as u64; + g + } + _ => 0, + } +} + +#[allow(clippy::cast_precision_loss)] +fn datetime_epoch_secs(dt: bson::DateTime) -> f64 { + dt.timestamp_millis() as f64 / 1000.0 +} + +#[allow(clippy::cast_possible_truncation)] +fn epoch_secs_datetime(secs: f64) -> bson::DateTime { + bson::DateTime::from_millis((secs * 1000.0).round() as i64) +} + +fn optional_datetime(doc: &Document, key: &str) -> Option { + doc.get_datetime(key) + .ok() + .map(|dt| datetime_epoch_secs(*dt)) +} + +fn required_str(doc: &Document, key: &str) -> Result { + doc.get_str(key) + .map(str::to_owned) + .map_err(|_| StorageError::Internal(format!("backup row is missing {key}"))) +} + +/// Whether a write's outcome cannot be told from its error: the statement +/// may have reached the server before the connection failed. +fn outcome_unknown(e: &mongodb::error::Error) -> bool { + matches!( + *e.kind, + mongodb::error::ErrorKind::Io(_) | mongodb::error::ErrorKind::ConnectionPoolCleared { .. } + ) +} + +/// One `backup_index` document as a [`BackupRow`]. +fn row_from_doc(doc: &Document) -> Result { + let backup_arn = required_str(doc, "backup_arn")?; + let store_kind = required_str(doc, "store_kind")?; + let store_kind = StoreKind::parse(&store_kind).ok_or_else(|| { + StorageError::Internal(format!( + "backup {backup_arn} carries an unknown store kind {store_kind:?}" + )) + })?; + let source_table_details_json: serde_json::Value = doc + .get("source_table_details_json") + .cloned() + .map(bson::from_bson) + .transpose() + .map_err(|e| StorageError::Internal(format!("parse source_table_details_json: {e}")))? + .ok_or_else(|| { + StorageError::Internal(format!( + "backup {backup_arn} is missing source_table_details_json" + )) + })?; + Ok(BackupRow { + backup_id: required_str(doc, "backup_id")?, + account_id: required_str(doc, "account_id")?, + table_name: required_str(doc, "table_name")?, + backup_arn, + backup_name: required_str(doc, "backup_name")?, + status: BackupStatus::parse(&required_str(doc, "status")?)?, + store: StoreIdentity { + store_kind, + store_root: required_str(doc, "store_root")?, + store_id: required_str(doc, "store_id")?, + }, + size_bytes: doc.get_i64("size_bytes").unwrap_or(0), + item_count: doc.get_i64("item_count").unwrap_or(0), + created_at: optional_datetime(doc, "created_at").unwrap_or(0.0), + completed_at: optional_datetime(doc, "completed_at"), + failure_reason: doc.get_str("failure_reason").ok().map(str::to_owned), + source_table_details_json, + owner_generation: generation_from_field(doc, "owner_generation"), + heartbeat_at: optional_datetime(doc, "heartbeat_at").unwrap_or(0.0), + }) +} + +/// The fields of `row` the claim's insert sets, status CREATING; the +/// generation and the heartbeat are added by the caller. +fn row_fields(row: &BackupRow) -> Result { + let source = bson::to_bson(&row.source_table_details_json) + .map_err(|e| StorageError::Internal(format!("serialize source_table_details_json: {e}")))?; + Ok(doc! { + "table_name": &row.table_name, + "backup_arn": &row.backup_arn, + "backup_name": &row.backup_name, + "status": BackupStatus::Creating.as_str(), + "store_kind": row.store.store_kind.as_str(), + "store_root": &row.store.store_root, + "store_id": &row.store.store_id, + "size_bytes": row.size_bytes, + "item_count": row.item_count, + "created_at": epoch_secs_datetime(row.created_at), + "source_table_details_json": source, + LOCK_SEQ: 0_i64, + }) +} + +/// The filter for the row `key` names. +fn key_filter(key: &BackupKey) -> Document { + doc! { "account_id": &key.account_id, "backup_id": &key.backup_id } +} + +/// The sort every listing applies: `(created_at, backup_arn)`. +fn listing_sort() -> Document { + doc! { "created_at": 1, "backup_arn": 1 } +} + +/// The rows `read` fetches, as a stream: the read runs once, on the first +/// poll, and a failed read is one `Err` item. +fn rows_stream<'a, F>(read: F) -> BoxStream<'a, Result> +where + F: std::future::Future, StorageError>> + Send + 'a, +{ + use futures::StreamExt; + Box::pin( + futures::stream::once(async move { + match read.await { + Ok(docs) => docs.iter().map(row_from_doc).collect::>(), + Err(e) => vec![Err(e)], + } + }) + .flat_map(futures::stream::iter), + ) +} + +/// The filter of the restore job records the reconciler acts on: a RUNNING +/// record whose heartbeat is older than `stale_after` on the server's clock +/// (a dead owner), and a FAILED record whose table is still a CREATING +/// restore target once `completed_at` (or, where none was stamped, the +/// heartbeat) is that old: the FAILED write landed and the removal did not, +/// and the delay leaves a task still on its own failure path to finish it. +pub(crate) fn dead_restore_job_filter(stale_after: Duration) -> Document { + doc! { + "$or": [ + { + RESTORE_JOB_STATUS: JobStatus::Running.as_str(), + "$expr": age_is_stale(&[RESTORE_JOB_HEARTBEAT], stale_after), + }, + { + RESTORE_JOB_STATUS: JobStatus::Failed.as_str(), + "table_status": "CREATING", + "$expr": age_is_stale( + &[RESTORE_JOB_COMPLETED_AT, RESTORE_JOB_HEARTBEAT], + stale_after, + ), + }, + ] + } +} + +/// A backup job's handle on its `backup_index` document. +struct MongoBackupClaim { + index: mongodb::Collection, + key: BackupKey, + generation: u64, +} + +impl MongoBackupClaim { + /// The filter of the row while it carries this claim. + fn owned_filter(&self) -> Document { + let mut filter = key_filter(&self.key); + filter.insert("owner_generation", generation_field(self.generation)); + filter + } +} + +impl JobClaim for MongoBackupClaim { + fn generation(&self) -> u64 { + self.generation + } + + fn heartbeat(&self) -> BoxFuture<'_, Result> { + Box::pin(async move { + let result = self + .index + .update_one( + self.owned_filter(), + vec![doc! { "$set": { "heartbeat_at": "$$NOW" } }], + ) + .await + .map_err(|e| StorageError::Internal(format!("backup heartbeat: {e}")))?; + Ok(if result.matched_count == 1 { + Heartbeat::Confirmed + } else { + Heartbeat::Lost + }) + }) + } + + fn set_status<'a>( + &'a self, + from: JobStatus, + to: JobStatus, + fields: &'a StatusFields, + ) -> BoxFuture<'a, Result> { + Box::pin(async move { + let mut filter = self.owned_filter(); + filter.insert("status", from.as_str()); + let mut set = doc! { "status": to.as_str() }; + if let Some(size) = fields.size_bytes { + set.insert("size_bytes", size); + } + if let Some(count) = fields.item_count { + set.insert("item_count", count); + } + if let Some(reason) = &fields.failure_reason { + set.insert("failure_reason", reason.as_str()); + } + if fields.completed { + set.insert("completed_at", "$$NOW"); + } + // A pipeline update, so `$$NOW` is the server's clock; the + // literal strings are wrapped so the pipeline does not read them + // as field paths. + let set: Document = set + .into_iter() + .map(|(k, v)| { + let v = match v { + Bson::String(s) if s == "$$NOW" => Bson::String(s), + other => doc! { "$literal": other }.into(), + }; + (k, v) + }) + .collect(); + match self + .index + .update_one(filter, vec![doc! { "$set": set }]) + .await + { + Ok(result) if result.matched_count == 1 => Ok(WriteOutcome::Matched), + Ok(_) => Ok(WriteOutcome::NotMatched), + Err(e) if outcome_unknown(&e) => Ok(WriteOutcome::Unknown), + Err(e) => Err(internal(e)), + } + }) + } + + fn read(&self) -> BoxFuture<'_, Result> { + Box::pin(async move { + let doc = self + .index + .find_one(key_filter(&self.key)) + .await + .map_err(internal)? + .ok_or_else(|| not_found(&self.key.backup_arn))?; + Ok(JobRow { + status: JobStatus::parse(&required_str(&doc, "status")?)?, + owner_generation: generation_from_field(&doc, "owner_generation"), + heartbeat_at: optional_datetime(&doc, "heartbeat_at").unwrap_or(0.0), + failure_reason: doc.get_str("failure_reason").ok().map(str::to_owned), + }) + }) + } +} + +impl MongoEngine { + pub(crate) fn backup_index(&self) -> mongodb::Collection { + self.catalog_db.collection::(BACKUP_INDEX) + } + + fn tables(&self) -> mongodb::Collection { + self.catalog_db.collection::("tables") + } + + /// The name of a CREATING restore target registered from `backup_arn`, + /// if any: the one in-use rule DeleteBackup, a restore's registration and + /// `backup_in_use` share. Reads the target's status, not the job's. The + /// driver's error is returned as it is, so a read inside a transaction + /// keeps its transient label for the caller's retry. + pub(crate) async fn creating_restore_target( + &self, + account_id: &str, + backup_arn: &str, + session: Option<&mut mongodb::ClientSession>, + ) -> Result, mongodb::error::Error> { + let filter = restoring_target_filter(account_id, backup_arn); + let tables = self.tables(); + let find = tables.find_one(filter); + let target = match session { + Some(session) => find.session(session).await, + None => find.await, + }?; + Ok(target.map(|t| { + t.get_document("_id") + .ok() + .and_then(|id| id.get_str("table_name").ok()) + .unwrap_or("?") + .to_owned() + })) + } + + /// DeleteBackup's first step as one session transaction: `$inc` the + /// row's `lock_seq` (the lock), read the status, run the in-use rule, + /// write DELETING. A transaction that lost a write conflict is re-run. + async fn start_backup_delete_tx(&self, key: &BackupKey) -> Result { + let index = self.backup_index(); + let mut session = self.client.start_session().await.map_err(internal)?; + let tx_options = self.write_transaction_options(); + for attempt in 0..TRANSIENT_RETRY_ATTEMPTS { + session + .start_transaction() + .with_options(tx_options.clone()) + .await + .map_err(internal)?; + let body: Result = async { + let locked = index + .find_one_and_update(key_filter(key), doc! { "$inc": { LOCK_SEQ: 1_i64 } }) + .session(&mut session) + .await + .map_err(TxErr::from)? + .ok_or_else(|| TxErr::Fatal(not_found(&key.backup_arn)))?; + let status = BackupStatus::parse(&required_str(&locked, "status")?)?; + if status != BackupStatus::Available { + return Ok(DeleteStart::Status(status)); + } + if let Some(table) = self + .creating_restore_target(&key.account_id, &key.backup_arn, Some(&mut session)) + .await + .map_err(TxErr::from)? + { + return Ok(DeleteStart::InUse(table)); + } + let mut filter = key_filter(key); + filter.insert("status", BackupStatus::Available.as_str()); + index + .update_one( + filter, + doc! { "$set": { "status": BackupStatus::Deleting.as_str() } }, + ) + .session(&mut session) + .await + .map_err(TxErr::from)?; + Ok(DeleteStart::Deleting) + } + .await; + match body { + Ok(outcome) => match self.commit_write_transaction(&mut session, None).await? { + CommitOutcome::Committed => return Ok(outcome), + CommitOutcome::RerunBody => backoff_sleep(attempt).await, + }, + Err(TxErr::Transient) => { + let _ = session.abort_transaction().await; + backoff_sleep(attempt).await; + } + Err(TxErr::Fatal(e)) => { + let _ = session.abort_transaction().await; + return Err(e); + } + } + } + Err(StorageError::Internal(format!( + "DeleteBackup of {} did not commit after {TRANSIENT_RETRY_ATTEMPTS} conflicting \ + attempts", + key.backup_arn + ))) + } +} + +impl BackupCatalog for MongoEngine { + fn catalog_id(&self) -> BoxFuture<'_, Result> { + Box::pin(async move { + let id = self + .catalog_db + .collection::("settings") + .find_one(doc! { "_id": CATALOG_ID }) + .await + .map_err(internal)? + .and_then(|d| d.get_str("value").ok().map(str::to_owned)); + id.ok_or_else(|| { + StorageError::Internal( + "the catalog has no catalog_id; run extenddb migrate --yes".to_owned(), + ) + }) + }) + } + + fn claim_backup<'a>( + &'a self, + row: &'a BackupRow, + ) -> BoxFuture<'a, Result, StorageError>> { + Box::pin(async move { + let generation = new_generation(); + let mut filter = key_filter(&row.key()); + filter.insert("owner_generation", doc! { "$exists": false }); + // The row is the claim. The upsert inserts it owned, with the + // server's clock on the heartbeat; an existing row never matches + // the filter (every row carries a generation), so the upsert's + // insert hits the unique index and the claim is refused. + let mut set = row_fields(row)?; + set.insert("owner_generation", generation_field(generation)); + set.insert("heartbeat_at", "$$NOW"); + let set: Document = set + .into_iter() + .map(|(k, v)| { + let v = match v { + Bson::String(s) if s == "$$NOW" => Bson::String(s), + other => doc! { "$literal": other }.into(), + }; + (k, v) + }) + .collect(); + match self + .backup_index() + .update_one(filter, vec![doc! { "$set": set }]) + .upsert(true) + .await + { + Ok(result) if result.upserted_id.is_some() => {} + Ok(_) => { + return Err(StorageError::AlreadyExists(format!( + "backup {} already has a row", + row.backup_arn + ))); + } + Err(e) if is_duplicate_key(&e) => { + return Err(StorageError::AlreadyExists(format!( + "a live job owns backup {}", + row.backup_arn + ))); + } + Err(e) => return Err(internal(e)), + } + Ok(Box::new(MongoBackupClaim { + index: self.backup_index(), + key: row.key(), + generation, + }) as Box) + }) + } + + fn list_backup_rows<'a>( + &'a self, + status: Option, + store: &'a StoreIdentity, + ) -> BoxStream<'a, Result> { + rows_stream(async move { + let mut filter = doc! { + "store_kind": store.store_kind.as_str(), + "store_root": &store.store_root, + "store_id": &store.store_id, + }; + if let Some(status) = status { + filter.insert("status", status.as_str()); + } + self.backup_index() + .find(filter) + .sort(listing_sort()) + .await + .map_err(internal)? + .try_collect() + .await + .map_err(internal) + }) + } + + fn list_account_backup_rows<'a>( + &'a self, + account_id: &'a str, + table_name: Option<&'a str>, + bounds: &'a TimeBounds, + ) -> BoxStream<'a, Result> { + rows_stream(async move { + let mut filter = doc! { "account_id": account_id }; + if let Some(table_name) = table_name { + filter.insert("table_name", table_name); + } + let mut created = Document::new(); + if let Some(lower) = bounds.lower { + created.insert("$gte", epoch_secs_datetime(lower)); + } + if let Some(upper) = bounds.upper { + created.insert("$lt", epoch_secs_datetime(upper)); + } + if !created.is_empty() { + filter.insert("created_at", created); + } + self.backup_index() + .find(filter) + .sort(listing_sort()) + .await + .map_err(internal)? + .try_collect() + .await + .map_err(internal) + }) + } + + fn read_backup_row<'a>( + &'a self, + arn: &'a BackupArn, + ) -> BoxFuture<'a, Result, StorageError>> { + Box::pin(async move { + let doc = self + .backup_index() + .find_one(doc! { + "account_id": arn.account_id(), + "backup_id": arn.backup_id(), + "table_name": arn.table_name(), + "backup_arn": arn.to_arn_string(), + }) + .await + .map_err(internal)?; + doc.as_ref().map(row_from_doc).transpose() + }) + } + + fn start_backup_delete<'a>( + &'a self, + key: &'a BackupKey, + ) -> BoxFuture<'a, Result> { + Box::pin(self.start_backup_delete_tx(key)) + } + + fn set_backup_deleted<'a>( + &'a self, + key: &'a BackupKey, + ) -> BoxFuture<'a, Result> { + Box::pin(async move { + let mut filter = key_filter(key); + filter.insert("status", BackupStatus::Deleting.as_str()); + let result = self + .backup_index() + .update_one( + filter, + vec![doc! { "$set": { + "status": { "$literal": BackupStatus::Deleted.as_str() }, + "completed_at": "$$NOW", + } }], + ) + .await + .map_err(internal)?; + Ok(result.matched_count == 1) + }) + } + + fn remove_backup_row<'a>( + &'a self, + key: &'a BackupKey, + ) -> BoxFuture<'a, Result> { + Box::pin(async move { + let mut filter = key_filter(key); + filter.insert( + "status", + doc! { "$in": [BackupStatus::Deleted.as_str(), BackupStatus::Failed.as_str()] }, + ); + let result = self + .backup_index() + .delete_one(filter) + .await + .map_err(internal)?; + Ok(result.deleted_count == 1) + }) + } + + fn backup_in_use<'a>( + &'a self, + key: &'a BackupKey, + ) -> BoxFuture<'a, Result, StorageError>> { + Box::pin(async move { + self.creating_restore_target(&key.account_id, &key.backup_arn, None) + .await + .map_err(internal) + }) + } + + fn remove_restore_target<'a>( + &'a self, + key: &'a RestoreTargetKey, + ) -> BoxFuture<'a, Result> { + Box::pin(async move { + // The key must name a CREATING restore target with this table id; + // a table re-created under the name has another id and is not + // touched. The delete is one atomic step on the document, so it + // and the ACTIVE flip (conditioned on CREATING) exclude each + // other. The catalog document goes first: once it is gone no + // provenance keeps the source backup in use, whatever the + // collection drops do, and the target counts as removed: a drop + // that fails then is logged, naming the collections, and leaves + // them for `backup sync` (no record remains for a later cycle to + // retry from). + let removed = self + .tables() + .find_one_and_delete(doc! { + "_id": { "account_id": &key.account_id, "table_name": &key.table_name }, + "table_id": &key.table_id, + "table_status": "CREATING", + "restore_source_backup_arn": { "$exists": true }, + }) + .await + .map_err(internal)?; + if removed.is_none() { + return Ok(false); + } + let dropped = self.remove_target_collections(&key.table_id).await; + #[cfg(test)] + let dropped = if self.backup_liveness.target_drop_fails { + Err(StorageError::Internal( + "test: the reply to the target's collection drops was lost".to_owned(), + )) + } else { + dropped + }; + if let Err(e) = dropped { + tracing::error!( + "the restore target {} (account {}) was removed from the catalog, but \ + dropping its data collection {} or the index collections of table id {} \ + failed; they are left in the data database for `backup sync` to sweep: {e}", + key.table_name, + key.account_id, + crate::data::data_collection_name(&key.table_id), + key.table_id + ); + } + Ok(true) + }) + } + + fn list_dead_restore_jobs( + &self, + stale_after: Duration, + ) -> BoxFuture<'_, Result, StorageError>> { + Box::pin(async move { + let docs: Vec = self + .tables() + .find(dead_restore_job_filter(stale_after)) + .await + .map_err(internal)? + .try_collect() + .await + .map_err(internal)?; + docs.iter() + .map(|doc| { + let id = doc + .get_document("_id") + .map_err(|_| StorageError::Internal("table is missing _id".to_owned()))?; + let job = doc.get_document(RESTORE_JOB).map_err(|_| { + StorageError::Internal( + "restore target is missing its job record".to_owned(), + ) + })?; + Ok(RestoreJobRow { + target: RestoreTargetKey { + account_id: required_str(id, "account_id")?, + table_name: required_str(id, "table_name")?, + table_id: required_str(doc, "table_id")?, + }, + source_backup_arn: required_str(doc, "restore_source_backup_arn")?, + kind: RestoreKind::parse(&required_str(job, "kind")?)?, + status: JobStatus::parse(&required_str(job, "status")?)?, + owner_generation: generation_from_field(job, "owner_generation"), + heartbeat_at: optional_datetime(job, "heartbeat_at").unwrap_or(0.0), + }) + }) + .collect() + }) + } + + fn fail_restore_job<'a>( + &'a self, + key: &'a RestoreTargetKey, + reason: &'a str, + ) -> BoxFuture<'a, Result> { + Box::pin(async move { + let result = self + .tables() + .update_one( + doc! { + "_id": { "account_id": &key.account_id, "table_name": &key.table_name }, + "table_id": &key.table_id, + RESTORE_JOB_STATUS: JobStatus::Running.as_str(), + }, + vec![doc! { "$set": { + RESTORE_JOB_STATUS: { "$literal": JobStatus::Failed.as_str() }, + RESTORE_JOB_FAILURE_REASON: { "$literal": reason }, + RESTORE_JOB_COMPLETED_AT: "$$NOW", + } }], + ) + .await + .map_err(internal)?; + Ok(result.matched_count == 1) + }) + } + + fn check_maintenance_claim(&self) -> BoxFuture<'_, Result> { + Box::pin(async move { + let held = self + .catalog_db + .collection::(BACKUP_MAINTENANCE) + .find_one(doc! {}) + .await + .map_err(internal)?; + Ok(held.is_some()) + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn fixture_source_table() -> serde_json::Value { + let manifest: serde_json::Value = serde_json::from_str(include_str!( + "../../core/testdata/extenddb-manifest/v1.json" + )) + .expect("fixture parses"); + manifest["source_table"].clone() + } + + fn row() -> BackupRow { + BackupRow { + backup_id: "01757500000123-0badcafe".to_owned(), + account_id: "123456789012".to_owned(), + table_name: "Music".to_owned(), + backup_arn: + "arn:aws:dynamodb:us-east-1:123456789012:table/Music/backup/01757500000123-0badcafe" + .to_owned(), + backup_name: "music-nightly".to_owned(), + status: BackupStatus::Creating, + store: StoreIdentity { + store_kind: StoreKind::S3, + store_root: "s3://bucket/prefix".to_owned(), + store_id: "store-1".to_owned(), + }, + size_bytes: 61828, + item_count: 100_000, + created_at: 1_757_500_000.123, + completed_at: None, + failure_reason: None, + source_table_details_json: fixture_source_table(), + owner_generation: 7, + heartbeat_at: 1_757_500_001.0, + } + } + + /// The document the claim writes reads back as the row it was built + /// from, the source table object included, with the times kept to the + /// millisecond. + #[test] + fn a_row_round_trips_through_its_document() { + let row = row(); + let mut doc = row_fields(&row).unwrap(); + doc.insert("account_id", &row.account_id); + doc.insert("backup_id", &row.backup_id); + doc.insert("owner_generation", generation_field(row.owner_generation)); + doc.insert("heartbeat_at", epoch_secs_datetime(row.heartbeat_at)); + let read = row_from_doc(&doc).unwrap(); + assert_eq!(read, row); + assert_eq!(read.source_table().unwrap().table_name, "Music"); + } + + #[test] + fn a_document_with_an_unknown_store_kind_or_status_is_internal() { + let row = row(); + let mut doc = row_fields(&row).unwrap(); + doc.insert("account_id", &row.account_id); + doc.insert("backup_id", &row.backup_id); + doc.insert("owner_generation", 7_i64); + doc.insert("heartbeat_at", epoch_secs_datetime(1.0)); + let mut bad_kind = doc.clone(); + bad_kind.insert("store_kind", "gcs"); + assert!(matches!( + row_from_doc(&bad_kind), + Err(StorageError::Internal(_)) + )); + let mut bad_status = doc; + bad_status.insert("status", "ACTIVE"); + assert!(matches!( + row_from_doc(&bad_status), + Err(StorageError::Internal(_)) + )); + } + + #[test] + fn only_a_connection_class_failure_leaves_a_write_unknown() { + let io: mongodb::error::Error = mongodb::error::ErrorKind::Io(std::sync::Arc::new( + std::io::Error::new(std::io::ErrorKind::ConnectionReset, "reset"), + )) + .into(); + assert!(outcome_unknown(&io)); + let command: mongodb::error::Error = mongodb::error::ErrorKind::Command( + bson::from_document(doc! { "code": 112, "codeName": "WriteConflict", "errmsg": "x" }) + .expect("a command error document"), + ) + .into(); + assert!(!outcome_unknown(&command)); + } + + /// The dead-job filter selects, by the server's clock, RUNNING records + /// with a stale heartbeat and FAILED records of a CREATING table whose + /// `completed_at`, or heartbeat where none, is stale. + #[test] + fn the_dead_job_filter_reads_both_kinds_of_record_by_the_server_clock() { + let filter = dead_restore_job_filter(Duration::from_secs(300)); + let arms = filter.get_array("$or").expect("$or"); + assert_eq!(arms.len(), 2); + let running = arms[0].as_document().expect("arm"); + assert_eq!(running.get_str(RESTORE_JOB_STATUS), Ok("RUNNING")); + assert!(running.get("table_status").is_none()); + let expr = running.get_document("$expr").expect("$expr").to_string(); + assert!(expr.contains("$restore_job.heartbeat_at"), "{expr}"); + assert!(!expr.contains("completed_at"), "{expr}"); + assert!(expr.contains("$$NOW") && expr.contains("300000"), "{expr}"); + let failed = arms[1].as_document().expect("arm"); + assert_eq!(failed.get_str(RESTORE_JOB_STATUS), Ok("FAILED")); + assert_eq!(failed.get_str("table_status"), Ok("CREATING")); + let expr = failed.get_document("$expr").expect("$expr").to_string(); + let completed = expr + .find("$restore_job.completed_at") + .expect("completed_at"); + let heartbeat = expr + .find("$restore_job.heartbeat_at") + .expect("heartbeat_at"); + assert!(completed < heartbeat, "completed_at is read first: {expr}"); + assert!(expr.contains("$$NOW") && expr.contains("300000"), "{expr}"); + } + + #[test] + fn generations_round_trip_through_the_integer_field() { + let g = new_generation(); + let doc = doc! { "owner_generation": generation_field(g) }; + assert_eq!(generation_from_field(&doc, "owner_generation"), g); + assert_eq!(generation_from_field(&doc! {}, "owner_generation"), 0); + } +} diff --git a/crates/storage-mongodb/src/backup_engine.rs b/crates/storage-mongodb/src/backup_engine.rs index beedee97c..d4561e1f7 100644 --- a/crates/storage-mongodb/src/backup_engine.rs +++ b/crates/storage-mongodb/src/backup_engine.rs @@ -1,12 +1,12 @@ // Copyright 2026 ExtendDB contributors // SPDX-License-Identifier: Apache-2.0 -//! `BackupEngine` implementation for `MongoDB`. +//! The in-catalog backup format on `MongoDB`: its `LegacyBackupReader`. //! //! Backups are stored as one MongoDB collection per backup, plus a `backups` //! metadata collection in the catalog. `CreateBackup` uses MongoDB's //! server-side aggregation `$out` stage to clone the source data collection -//! into `_backup_` in the data database — no per-item traffic +//! into `_backup_` in the data database, with no per-item traffic //! between the driver and the server. `RestoreTableFromBackup` uses the same //! stage in reverse, in a task detached from the request. `DeleteBackup` drops //! the collection. @@ -15,48 +15,72 @@ //! from that id so the `backup_arn` (which contains slashes and colons) never //! appears in a collection name. //! -//! # Claims, restore targets, and liveness +//! # The restore job record, DeleteBackup and the claim //! -//! MongoDB has no transaction spanning the claim and the target's collections, -//! so a restore and a DeleteBackup exclude each other through markers: +//! A restore registers through the job record on its target's `tables` +//! document (`restore_job`: `status`, `kind = legacy`, `owner_generation`, +//! `heartbeat_at`, [`crate::backup_catalog`]), beside the provenance fields +//! `DescribeTable` reports. The registration is one session transaction on +//! the catalog database: it `$inc`s `lock_seq` on the backup's `backups` +//! document while that document is AVAILABLE, checks that no CREATING target +//! is restoring the backup already, and inserts the CREATING target. The +//! legacy DeleteBackup takes the same lock in its own transaction before it +//! re-checks for a CREATING target and writes DELETING (on the delete's side +//! the DELETING write is itself a write to the document, so the `$inc` there +//! is what makes a refused delete, `Status` or `InUse`, conflict too), so +//! the two order wholly one before the other: a DeleteBackup that wins leaves the +//! registration a document that is not AVAILABLE (`Backup not found`, no +//! target), and a registration that wins makes the DeleteBackup `BackupInUse`. +//! From then on the CREATING target is what keeps DeleteBackup and other +//! restores of the backup out, until the target is ACTIVE, as the service +//! answers. Both in-use checks read at the transaction read concern +//! (`transaction_read_concern`, `snapshot` by default), which is +//! majority-committed data, while `DescribeTable` reads the catalog at +//! `local`: a client whose `DescribeTable` has just shown the target ACTIVE +//! can be answered `BackupInUse` by a RestoreTableFromBackup or DeleteBackup +//! issued within the majority-commit lag (the journal flush, and any +//! concurrent batch write still holding an earlier oplog slot), and the +//! retry succeeds. That lock replaces the claim markers earlier releases wrote on +//! the `backups` document (`restore_in_progress`, `delete_in_progress` and +//! their heartbeats): nothing writes or reads them now, and a marker a dead +//! process left no longer blocks the backup until a sweep clears it. Once +//! DeleteBackup has written DELETING the backup never reads AVAILABLE again: +//! the drop that follows may have run on the server even if the request +//! stops before its reply, and a later DeleteBackup of a DELETING backup +//! resumes the drop and the DELETED write instead of answering that the +//! backup is in use; `describe` and `list` leave DELETING out as they leave +//! DELETED out. //! -//! - a restore claims the backup document (`restore_in_progress`), checks that -//! no CREATING target is restoring the backup already, inserts its CREATING -//! target with provenance (`restore_source_backup_arn`, -//! `restore_operation_id`), then releases the claim; from then on the -//! CREATING target is what keeps DeleteBackup and other restores of the -//! backup out (`BackupInUse` until the target is ACTIVE, as the service -//! answers). A claim another restore released before inserting anything is -//! retried for up to [`RESTORE_CLAIM_RETRY_FOR`]. The exclusion is bounded -//! by the heartbeat: a restore that stalls between its claim and its insert -//! for longer than [`BACKUP_HEARTBEAT_STALE_AFTER`] has its claim swept, a -//! second restore of the backup can then claim and insert, and when the -//! first resumes it inserts too (its release of the claim returns `false` -//! and it goes on while the backup is AVAILABLE), leaving two CREATING -//! targets of one backup that both finish. The other three backends have no -//! such window: PostgreSQL and SQLite register a restore in one transaction, -//! and Cassandra's registration is a lightweight transaction on the -//! backup's row conditioned on no other registration; -//! - DeleteBackup claims the same document (`delete_in_progress`) under the -//! inverse predicate, re-checks for a CREATING target, and only then drops -//! the backup's data. Once the drop may have started its claim is never -//! released; if DeleteBackup stops after that, the sweep finishes it. -//! -//! Every marker carries its owner's operation id and a heartbeat timestamp -//! (`_heartbeat` on a claim, `restore_heartbeat_at` on a target) that -//! a companion task refreshes at once and then every -//! [`BACKUP_HEARTBEAT_INTERVAL`] with the server's clock (`$$NOW`); a restore -//! also refreshes its target's right after inserting it, replacing the insert's -//! local-clock value. [`MongoEngine::sweep_stale_backup_claims`], -//! run at startup and periodically by the GSI backfill worker, removes only -//! markers whose heartbeat is older than [`BACKUP_HEARTBEAT_STALE_AFTER`], so -//! several processes can share one catalog without clearing each other's live -//! work. A restore target stops heartbeating once its base copy is done and -//! `restore_backfill_pending` is set; from then the backfill worker owns it and -//! finishes it after a restart, so the sweep never touches it. - -use std::collections::HashMap; -use std::sync::Arc; +//! The restore's claim is the job record: the target is inserted RUNNING +//! under a fresh `owner_generation`, and the copy task refreshes +//! `heartbeat_at` with the server's clock (`$$NOW`) at once and then every +//! [`JOB_HEARTBEAT_INTERVAL`], the one constant every backend's heartbeat +//! loop reads and the stale floor is computed from (`[backup] +//! heartbeat_interval_secs` reaches the backends in a later release); a +//! refresh that matches other than exactly one document, or fails, means the +//! claim is lost and the copy stops without touching the target (the FAILED +//! write and the removal, each conditioned on the generation, are what +//! follow); the `$out` already sent runs to its end on the server, a write +//! stage the server does not interrupt for a client's disconnect, and then +//! fails rather than recreate the data collection the removal dropped, since +//! `$out` refuses to replace a target whose indexes changed while it ran (see +//! `copy_restore`), so a lost heartbeat leaves no orphan collection. The +//! copy's hand-over to the GSI backfill worker marks the +//! record DONE in the same write, conditioned on the generation and on +//! RUNNING; a failure marks it FAILED under the same condition and removes +//! the target only when that matched. A target whose record is RUNNING with +//! a stale heartbeat has a dead owner: the reconciler +//! (`BackupCatalog::list_dead_restore_jobs`) fails it and removes it; a +//! FAILED record whose target is still CREATING (the removal did not land) +//! is listed once the record is stale, and the reconciler removes the target +//! with no second FAILED write. The removal deletes the target's `tables` +//! document first and then drops its data and index collections; once the +//! document is gone the target is removed (the name and the backup are +//! free, and no record is left for a later cycle), so a drop that fails +//! after it is logged with the collections' names and leaves them in the +//! data database for `backup sync` to sweep, as nothing in the catalog +//! points at them. + use std::time::{Duration, Instant}; use futures::TryStreamExt; @@ -64,7 +88,6 @@ use futures::future::BoxFuture; use mongodb::bson::{self, Bson, Document, doc}; use serde::Serialize; use serde::de::DeserializeOwned; -use tokio::sync::{OwnedSemaphorePermit, Semaphore}; use extenddb_core::types::{ AttributeDefinition, BackupDescription, BackupDetails, BackupSummary, @@ -72,55 +95,74 @@ use extenddb_core::types::{ PointInTimeRecoveryDescription, Projection, ProvisionedThroughput, SourceTableDetails, StreamSpecification, TableDescription, TimeToLiveDescription, TimeToLiveStatus, }; +use extenddb_storage::backup_catalog::{ + AbandonedLegacyBackup, BackupCatalog, JOB_HEARTBEAT_INTERVAL, JobStatus, RestoreKind, + RestoreTargetKey, new_generation, +}; use extenddb_storage::backup_definition::{BACKUP_DEFINITION_VERSION, BackupTableDefinition}; use extenddb_storage::error::StorageError; -use extenddb_storage::{BackupEngine, BackupRestoreSource, RestoreRequest, util}; +use extenddb_storage::{BackupRestoreSource, JobPermit, LegacyBackupReader, RestoreRequest, util}; use crate::MongoEngine; +use crate::backup_catalog::{ + LOCK_SEQ, RESTORE_JOB, RESTORE_JOB_COMPLETED_AT, RESTORE_JOB_FAILURE_REASON, + RESTORE_JOB_GENERATION, RESTORE_JOB_HEARTBEAT, RESTORE_JOB_STATUS, backup_in_use, + generation_field, +}; use crate::data::data_collection_name; - -/// How often a live restore or DeleteBackup refreshes its heartbeats. -pub(crate) const BACKUP_HEARTBEAT_INTERVAL: Duration = Duration::from_secs(15); -/// A claim or restore target whose heartbeat is older than this belongs to an -/// operation that stopped (its process died); the sweep removes it. Four -/// missed refreshes, so a slow catalog write does not make live work look dead. -pub(crate) const BACKUP_HEARTBEAT_STALE_AFTER: Duration = Duration::from_secs(60); - -pub(crate) const RESTORE_CLAIM: &str = "restore_in_progress"; -pub(crate) const RESTORE_CLAIM_HEARTBEAT: &str = "restore_in_progress_heartbeat"; -pub(crate) const DELETE_CLAIM: &str = "delete_in_progress"; -pub(crate) const DELETE_CLAIM_HEARTBEAT: &str = "delete_in_progress_heartbeat"; -pub(crate) const TARGET_OPERATION: &str = "restore_operation_id"; -pub(crate) const TARGET_HEARTBEAT: &str = "restore_heartbeat_at"; +use crate::data_engine::{CommitOutcome, TRANSIENT_RETRY_ATTEMPTS, TxErr, backoff_sleep}; /// The definition version stored on a backup document written by this build. /// Backup documents written before it carry the same fields without the /// marker and are read as version 1. const DEFINITION_VERSION_FIELD: &str = "definition_version"; -/// Heartbeat timing for backup claims and restore targets. Production uses -/// the defaults; tests shorten both so liveness can be observed in seconds. +/// How long a request waits for its detached task to register the CREATING +/// target before it answers `Internal`. The bound is what turns a connection +/// pool that has stopped handing out connections into a 500 instead of a +/// request that never returns (the driver does not enforce +/// `waitQueueTimeoutMS`, so a checkout waits for as long as every connection +/// stays checked out). The task goes on: a request that hit the bound may +/// leave a CREATING target that appears after its error, holds the backup in +/// use (`BackupInUseException` to DeleteBackup) and takes the name until the +/// copy ends, and DescribeTable is where a client finds it. +pub(crate) const RESTORE_REGISTRATION_TIMEOUT: Duration = Duration::from_secs(60); + +/// Heartbeat timing of a restore's job record. Production uses the default, +/// [`JOB_HEARTBEAT_INTERVAL`]; tests shorten it so liveness can be observed +/// in seconds. #[derive(Debug, Clone)] pub(crate) struct BackupLiveness { pub(crate) heartbeat_interval: Duration, - pub(crate) stale_after: Duration, - /// How long a restore waits for its account slot and a server slot - /// together before it is refused with `LimitExceeded` - /// ([`RESTORE_SLOT_WAIT`] in production; tests shorten it). - pub(crate) slot_wait: Duration, + /// Test only: when set, a restore's task waits for one permit after the + /// name check and before it registers its target, so a test can order a + /// registration after a DeleteBackup of the same backup. + #[cfg(test)] + pub(crate) registration_gate: Option>, /// Test only: when set, a restore's copy task waits for one permit after /// registering its target and before its base copy, so a test can observe /// the CREATING window deterministically. #[cfg(test)] pub(crate) copy_gate: Option>, - /// Test only: what a DeleteBackup or sweep does right after it dropped a - /// backup's data collection. + /// Test only: what a DeleteBackup does right after it dropped a backup's + /// data collection. #[cfg(test)] pub(crate) after_backup_drop: Option, + /// Test only: when set, `remove_restore_target` treats the reply to the + /// target's collection drops as lost (an error after the drops ran). + #[cfg(test)] + pub(crate) target_drop_fails: bool, /// Test only: how far this process's clock is behind the server's, applied /// to the one heartbeat written with the local clock (the target insert). #[cfg(test)] pub(crate) local_clock_behind: Duration, + /// Test only: when set, the restore heartbeat fails at its next tick once + /// the test has armed the cell (an injected error, as a network fault on + /// the heartbeat's connection would be), and records that it did: + /// [`HeartbeatFault::ARMED`] is the test's request, + /// [`HeartbeatFault::INJECTED`] the loop's answer. + #[cfg(test)] + pub(crate) heartbeat_fault: Option>, } /// Test only: the two ways a backup drop can end after the server ran it. @@ -134,128 +176,40 @@ pub(crate) enum AfterBackupDrop { Hang, } +/// Test only: the states of [`BackupLiveness::heartbeat_fault`]. +#[cfg(test)] +pub(crate) struct HeartbeatFault; + +#[cfg(test)] +impl HeartbeatFault { + /// Nothing requested. + pub(crate) const IDLE: u8 = 0; + /// The next heartbeat tick fails. + pub(crate) const ARMED: u8 = 1; + /// A heartbeat failed as requested. + pub(crate) const INJECTED: u8 = 2; +} + impl Default for BackupLiveness { fn default() -> Self { Self { - heartbeat_interval: BACKUP_HEARTBEAT_INTERVAL, - stale_after: BACKUP_HEARTBEAT_STALE_AFTER, - slot_wait: RESTORE_SLOT_WAIT, + heartbeat_interval: JOB_HEARTBEAT_INTERVAL, + #[cfg(test)] + registration_gate: None, #[cfg(test)] copy_gate: None, #[cfg(test)] after_backup_drop: None, #[cfg(test)] + target_drop_fails: false, + #[cfg(test)] local_clock_behind: Duration::ZERO, + #[cfg(test)] + heartbeat_fault: None, } } } -/// Restores one account may have in flight at once in this process. A -/// restore beyond the limit waits for one of its own account's slots, so one -/// account cannot hold every other account's restores back. Same figure as -/// the PostgreSQL backend. -const RESTORE_SLOTS_PER_ACCOUNT: usize = 4; - -/// Upper limit on [`restore_server_slot_count`]. -const RESTORE_SERVER_SLOTS_MAX: usize = 16; - -/// How long a restore waits for its account slot and a server slot, -/// together, before it is refused with `LimitExceeded`. -pub(crate) const RESTORE_SLOT_WAIT: Duration = Duration::from_secs(30); - -/// How long a request waits for its detached task to register the CREATING -/// target before it answers `Internal`. The wait for a slot -/// ([`RESTORE_SLOT_WAIT`]) runs inside the task, so it counts against this -/// bound too. The task goes on; the bound is what turns a connection pool -/// that has stopped handing out connections into a 500 instead of a request -/// that never returns (the driver enforces no checkout timeout, see -/// [`crate::effective_pool_size`]). -pub(crate) const RESTORE_REGISTRATION_TIMEOUT: Duration = Duration::from_secs(60); - -/// Restores the whole process may have in flight at once, across accounts: -/// `min(16, pool_size / 2)`, at least 1, where `pool_size` is the driver pool -/// of the engine's client (its `maxPoolSize`, default 10). -/// -/// A restore's detached task holds up to one pooled connection at a time -/// (the `$out` copy, a catalog write) and its heartbeat task takes one for -/// each refresh, so without a process-wide bound enough concurrent restores -/// can hold every pooled connection and every other request waits on a -/// checkout the driver never times out. Half the pool stays free for them. -/// A restore takes its account slot first, then one of these; the two waits -/// share one deadline of [`BackupLiveness::slot_wait`]. -pub(crate) fn restore_server_slot_count(pool_size: u32) -> usize { - usize::try_from(pool_size / 2) - .unwrap_or(usize::MAX) - .clamp(1, RESTORE_SERVER_SLOTS_MAX) -} - -/// The two slots a running restore holds until its copy ends. -pub(crate) struct RestorePermits { - _account: OwnedSemaphorePermit, - _server: OwnedSemaphorePermit, -} - -/// Take an account slot, then a server slot, waiting at most `wait` for -/// both together. Running out of time on either is `LimitExceeded`, with a -/// message naming the limit that was full. -async fn acquire_restore_permits( - account: Arc, - server: Arc, - server_slots: usize, - wait: Duration, -) -> Result { - let deadline = tokio::time::Instant::now() + wait; - let closed = - |e: tokio::sync::AcquireError| StorageError::Internal(format!("restore slot: {e}")); - let account_permit = tokio::time::timeout_at(deadline, account.acquire_owned()) - .await - .map_err(|_| { - StorageError::LimitExceeded(format!( - "Too many restores are in progress for this account (at most \ - {RESTORE_SLOTS_PER_ACCOUNT} at once); retry later" - )) - })? - .map_err(closed)?; - let server_permit = tokio::time::timeout_at(deadline, server.acquire_owned()) - .await - .map_err(|_| { - StorageError::LimitExceeded(format!( - "Too many restores are in progress on this server (at most {server_slots} at \ - once); retry later" - )) - })? - .map_err(closed)?; - Ok(RestorePermits { - _account: account_permit, - _server: server_permit, - }) -} - -/// Per-account restore slots of one engine (shared by its clones, so one -/// per server process). An entry nobody holds (no permit, no waiter) is -/// pruned the next time any account takes a slot, so the map does not grow -/// with every account that ever restored. -#[derive(Clone, Default)] -pub(crate) struct RestoreAccountSlots(Arc>>>); - -impl RestoreAccountSlots { - /// The slot semaphore for `account_id`. - pub(crate) fn for_account(&self, account_id: &str) -> Arc { - let mut slots = self - .0 - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner); - // A permit and a waiter each hold a clone, so a count of one means - // the account has nothing in flight. - slots.retain(|_, s| Arc::strong_count(s) > 1); - Arc::clone( - slots - .entry(account_id.to_owned()) - .or_insert_with(|| Arc::new(Semaphore::new(RESTORE_SLOTS_PER_ACCOUNT))), - ) - } -} - /// Return the MongoDB collection name that holds items for a given backup. /// /// The collection lives in the data database. The name is derived from the @@ -377,33 +331,11 @@ fn restore_gsi_throughput( }) } -/// A restore first claims backup metadata, then transfers protection to the -/// provenance-bearing CREATING target. DeleteBackup claims the same metadata -/// before dropping data, so neither operation can pass the other between its -/// check and destructive action. The heartbeat fields do not take part in -/// either predicate: a stale claim still excludes the other operation until -/// the sweep removes it. -fn restore_claim_filter(account_id: &str, backup_arn: &str) -> Document { - doc! { - "_id": backup_arn, - "account_id": account_id, - "backup_status": "AVAILABLE", - "restore_in_progress": { "$exists": false }, - "delete_in_progress": { "$exists": false }, - } -} - -fn delete_claim_filter(account_id: &str, backup_arn: &str) -> Document { - doc! { - "_id": backup_arn, - "account_id": account_id, - "backup_status": "AVAILABLE", - "restore_in_progress": { "$exists": false }, - "delete_in_progress": { "$exists": false }, - } -} - -fn restoring_target_filter(account_id: &str, backup_arn: &str) -> Document { +/// The filter of a CREATING restore target registered from `backup_arn` in +/// `account_id`: the in-use rule, which reads the target's status, not the +/// job's, so it also covers the window between the job's DONE write and the +/// backfill worker's ACTIVE flip. +pub(crate) fn restoring_target_filter(account_id: &str, backup_arn: &str) -> Document { doc! { "_id.account_id": account_id, "restore_source_backup_arn": backup_arn, @@ -411,631 +343,78 @@ fn restoring_target_filter(account_id: &str, backup_arn: &str) -> Document { } } -/// How long a restore retries a backup claim held by another restore. -pub(crate) const RESTORE_CLAIM_RETRY_FOR: Duration = Duration::from_secs(2); - -/// Why a restore's claim found no backup document to claim, from a re-read of -/// the document (`None`: no document for this ARN and account). -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -enum RestoreClaimMiss { - /// Missing, or not AVAILABLE. - NotFound, - /// AVAILABLE and claimed by a DeleteBackup. - DeleteClaimed, - /// AVAILABLE and claimed by another restore only. - RestoreClaimed, - /// AVAILABLE and unclaimed: the other claim was released meanwhile. - Free, -} - -fn restore_claim_miss(doc: Option<&Document>) -> RestoreClaimMiss { - let Some(doc) = doc else { - return RestoreClaimMiss::NotFound; - }; - if doc.get_str("backup_status") != Ok("AVAILABLE") { - RestoreClaimMiss::NotFound - } else if doc.contains_key(DELETE_CLAIM) { - RestoreClaimMiss::DeleteClaimed - } else if doc.contains_key(RESTORE_CLAIM) { - RestoreClaimMiss::RestoreClaimed - } else { - RestoreClaimMiss::Free - } -} - -/// The error for a DeleteBackup whose claim found no document to claim. -/// `restoring`: a CREATING restore target of the backup exists; `miss`: the -/// re-read backup document, classified. A backup another DeleteBackup has -/// claimed is in use (it reads AVAILABLE until that delete finishes), not -/// missing. -fn delete_claim_miss_error( - restoring: bool, - miss: RestoreClaimMiss, - backup_arn: &str, -) -> StorageError { - match miss { - RestoreClaimMiss::DeleteClaimed => { - StorageError::BackupInUse(format!("Backup is being deleted: {backup_arn}")) - } - RestoreClaimMiss::RestoreClaimed => StorageError::BackupInUse(format!( - "Backup is being used to restore another table: {backup_arn}" - )), - RestoreClaimMiss::NotFound | RestoreClaimMiss::Free if restoring => { - StorageError::BackupInUse(format!( - "Backup is being used to restore another table: {backup_arn}" - )) - } - RestoreClaimMiss::NotFound | RestoreClaimMiss::Free => { - StorageError::Validation(format!("Backup not found: {backup_arn}")) - } - } -} - -/// Install `field = operation_id` and its heartbeat (server clock) in one -/// update. A pipeline update, so `$$NOW` is the server's time and processes -/// with skewed clocks agree on staleness. -fn claim_update(field: &str, heartbeat_field: &str, operation_id: &str) -> Vec { - let mut set = Document::new(); - set.insert(field, doc! { "$literal": operation_id }); - set.insert(heartbeat_field, "$$NOW"); - vec![doc! { "$set": set }] -} - -/// `$expr` that is true when `heartbeat_field` is missing (a marker written -/// before heartbeats existed) or older than `stale_after` by the server clock. -fn heartbeat_is_stale(heartbeat_field: &str, stale_after: Duration) -> Document { +/// `$expr` that is true when the first of `fields` that is set is older than +/// `stale_after` by the server clock; a document with none of them set is as +/// old as the epoch. The reconciler's two ages: a RUNNING record's heartbeat, +/// and a FAILED record's `completed_at` or, where none was stamped, its +/// heartbeat. +pub(crate) fn age_is_stale(fields: &[&str], stale_after: Duration) -> Document { let stale_ms = i64::try_from(stale_after.as_millis()).unwrap_or(i64::MAX); + let age = fields.iter().rev().fold( + Bson::DateTime(bson::DateTime::from_millis(0)), + |fallback, field| Bson::Document(doc! { "$ifNull": [ format!("${field}"), fallback ] }), + ); doc! { "$lt": [ - { "$ifNull": [ format!("${heartbeat_field}"), bson::DateTime::from_millis(0) ] }, + age, { "$subtract": [ "$$NOW", stale_ms ] }, ] } } -/// Filter for a stale claim of `field` on backup documents. -fn stale_claim_filter(field: &str, heartbeat_field: &str, stale_after: Duration) -> Document { - let mut filter = Document::new(); - filter.insert(field, doc! { "$exists": true }); - filter.insert("$expr", heartbeat_is_stale(heartbeat_field, stale_after)); - filter -} - -/// Filter for an abandoned restore target: CREATING, carrying restore -/// provenance, still before its base copy finished (no backfill flag; after -/// that the backfill worker owns it), and not heartbeating. -fn stale_target_filter(stale_after: Duration) -> Document { - doc! { - "table_status": "CREATING", - "restore_source_backup_arn": { "$exists": true }, - "restore_backfill_pending": { "$exists": false }, - "$expr": heartbeat_is_stale(TARGET_HEARTBEAT, stale_after), - } -} - -fn claim_heartbeat_field(field: &'static str) -> &'static str { - if field == RESTORE_CLAIM { - RESTORE_CLAIM_HEARTBEAT - } else { - DELETE_CLAIM_HEARTBEAT - } -} - -/// Restore provenance written into the target's first catalog document. -pub(crate) struct RestoreProvenance<'a> { +/// What a restore's registration writes into the target's first catalog +/// document: the provenance `DescribeTable` reports and the job record. +pub(crate) struct RestoreRegistration<'a> { pub(crate) backup_arn: &'a str, pub(crate) source_table_arn: &'a str, pub(crate) restore_date_time: bson::DateTime, - pub(crate) operation_id: &'a str, - /// The target's first heartbeat. An insert cannot use `$$NOW`, so this is - /// the inserting process's clock; `register_restore` overwrites it with - /// the server's clock as soon as the insert returns. + /// The claim's fencing token, written as `restore_job.owner_generation`. + pub(crate) owner_generation: u64, + /// The record's first heartbeat. An insert cannot use `$$NOW`, so this is + /// the inserting process's clock; the copy task's first refresh replaces + /// it with the server's as soon as the task starts. pub(crate) first_heartbeat: bson::DateTime, - /// Notified once the target document is inserted, while the rest of the - /// target (collections, indexes) is still being created. - pub(crate) inserted: Option<&'a tokio::sync::Notify>, + /// The session whose transaction the target's insert runs in and which + /// the insert commits; `None` inserts and commits nothing beyond the + /// document itself. + pub(crate) session: Option<&'a mut mongodb::ClientSession>, } -impl RestoreProvenance<'_> { +impl RestoreRegistration<'_> { pub(crate) fn insert_into(&self, table_doc: &mut Document) { table_doc.insert("restore_source_backup_arn", self.backup_arn); table_doc.insert("restore_source_table_arn", self.source_table_arn); table_doc.insert("restore_date_time", self.restore_date_time); - table_doc.insert(TARGET_OPERATION, self.operation_id); - table_doc.insert(TARGET_HEARTBEAT, self.first_heartbeat); - } -} - -struct BackupMetadataClaim { - backups: mongodb::Collection, - account_id: String, - backup_arn: String, - field: &'static str, - operation_id: String, - armed: bool, -} - -impl BackupMetadataClaim { - fn new( - backups: mongodb::Collection, - account_id: &str, - backup_arn: &str, - field: &'static str, - operation_id: String, - ) -> Self { - Self { - backups, - account_id: account_id.to_owned(), - backup_arn: backup_arn.to_owned(), - field, - operation_id, - armed: true, - } - } - - fn filter(&self) -> Document { - let mut filter = doc! { - "_id": &self.backup_arn, - "account_id": &self.account_id, - }; - filter.insert(self.field, &self.operation_id); - filter - } - - fn unset_update(field: &'static str) -> Document { - let mut unset = Document::new(); - unset.insert(field, ""); - unset.insert(claim_heartbeat_field(field), ""); - doc! { "$unset": unset } - } - - /// Release the claim. `Ok(false)` means it was no longer ours to release: - /// a sweep removed it because its heartbeat had lapsed. - async fn release(&mut self) -> Result { - if !self.armed { - return Ok(true); - } - let result = self - .backups - .update_one(self.filter(), Self::unset_update(self.field)) - .await - .map_err(internal)?; - self.armed = false; - Ok(result.matched_count == 1) - } - - fn disarm(&mut self) { - self.armed = false; - } -} - -impl Drop for BackupMetadataClaim { - /// Release a claim whose operation was dropped before its destructive - /// step (a DeleteBackup request cancelled between its claim and its drop; - /// DeleteBackup disarms the claim before dropping, so this never runs - /// after the data may be gone). This runs only when a Tokio runtime is - /// current and the spawned update can still reach the catalog; a process - /// exit, a drop outside a runtime, or a failed update leaves the marker in - /// place. The backstop for all of those is the heartbeat sweep - /// ([`MongoEngine::sweep_stale_backup_claims`]): the companion heartbeat - /// stops with the operation, and once the marker is older than - /// [`BACKUP_HEARTBEAT_STALE_AFTER`] the sweep clears a restore claim and - /// finishes the delete of a delete claim. - fn drop(&mut self) { - if !self.armed { - return; - } - let backups = self.backups.clone(); - let filter = self.filter(); - let update = Self::unset_update(self.field); - let backup_arn = self.backup_arn.clone(); - if let Ok(runtime) = tokio::runtime::Handle::try_current() { - let _cleanup = runtime.spawn(async move { - if let Err(error) = backups.update_one(filter, update).await { - tracing::error!( - "could not clear cancelled backup metadata claim for {backup_arn}: {error}" - ); - } - }); - } - } -} - -/// The companion heartbeat of one restore or DeleteBackup operation. Aborted -/// when dropped, so the heartbeat stops exactly when its owner does. -pub(crate) struct Heartbeat(tokio::task::JoinHandle<()>); - -impl Drop for Heartbeat { - fn drop(&mut self) { - self.0.abort(); - } -} - -/// The updates that refresh every marker `operation_id` owns on `backup_arn` -/// (either claim) and on the restore target, if any, as (collection, marker, -/// filter, pipeline update). Each matches only the owner's marker, so a -/// refresh never revives a marker a sweep already removed. -fn heartbeat_updates( - operation_id: &str, - account_id: &str, - backup_arn: &str, - target_table: Option<&str>, -) -> Vec<(&'static str, &'static str, Document, Vec)> { - let mut updates = Vec::new(); - for (field, heartbeat_field) in [ - (RESTORE_CLAIM, RESTORE_CLAIM_HEARTBEAT), - (DELETE_CLAIM, DELETE_CLAIM_HEARTBEAT), - ] { - let mut filter = doc! { "_id": backup_arn, "account_id": account_id }; - filter.insert(field, operation_id); - let mut set = Document::new(); - set.insert(heartbeat_field, "$$NOW"); - updates.push(("backups", field, filter, vec![doc! { "$set": set }])); - } - if let Some(table_name) = target_table { - let mut filter = doc! { - "_id": { "account_id": account_id, "table_name": table_name }, - "table_status": "CREATING", - }; - filter.insert(TARGET_OPERATION, operation_id); - let mut set = Document::new(); - set.insert(TARGET_HEARTBEAT, "$$NOW"); - updates.push(( - "tables", - TARGET_HEARTBEAT, - filter, - vec![doc! { "$set": set }], - )); - } - updates -} - -/// Run `run` on every item, also after one fails, and return every error. -async fn try_each(items: Vec, mut run: impl FnMut(T) -> Fut) -> Vec -where - Fut: std::future::Future>, -{ - let mut errors = Vec::new(); - for item in items { - if let Err(error) = run(item).await { - errors.push(error); - } + table_doc.insert( + RESTORE_JOB, + doc! { + "status": JobStatus::Running.as_str(), + "kind": RestoreKind::Legacy.as_str(), + "owner_generation": generation_field(self.owner_generation), + "heartbeat_at": self.first_heartbeat, + }, + ); } - errors -} - -/// Refresh every marker `operation_id` owns (see [`heartbeat_updates`]). A -/// failed update does not stop the others; every failure is returned. -async fn refresh_heartbeats( - catalog: &mongodb::Database, - operation_id: &str, - account_id: &str, - backup_arn: &str, - target_table: Option<&str>, -) -> Vec { - let updates = heartbeat_updates(operation_id, account_id, backup_arn, target_table); - try_each(updates, |(collection, marker, filter, update)| { - let collection_handle = catalog.collection::(collection); - async move { - collection_handle - .update_one(filter, update) - .await - .map(|_| ()) - .map_err(|e| StorageError::Internal(format!("{collection}.{marker}: {e}"))) - } - }) - .await -} - -/// What one sweep removed, by name, for the log. -#[derive(Debug, Default, Clone, PartialEq, Eq)] -pub struct BackupSweep { - /// Backup ARNs whose stale restore claim was cleared. - pub restore_claims_cleared: Vec, - /// `/` of abandoned restore targets removed. - pub restore_targets_removed: Vec, - /// Backup ARNs whose interrupted DeleteBackup the sweep finished. - pub delete_claims_finished: Vec, - /// Backup ARNs whose stale delete claim the sweep took over but left held, - /// because a restore target from the backup appeared after its first - /// check; a later sweep finishes the delete. - pub delete_claims_waiting: Vec, } -impl BackupSweep { - #[must_use] - pub fn is_empty(&self) -> bool { - self.restore_claims_cleared.is_empty() - && self.restore_targets_removed.is_empty() - && self.delete_claims_finished.is_empty() - && self.delete_claims_waiting.is_empty() - } +/// Why a legacy registration or DeleteBackup found no AVAILABLE backup +/// document to lock, from a re-read of the document. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum LockMiss { + /// Missing, another account's, DELETED, or in a state no release writes. + NotFound, + /// A DeleteBackup has written DELETING and is dropping the data. + Deleting, } -impl std::fmt::Display for BackupSweep { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - let parts = [ - ( - "cleared abandoned restore claim(s) on", - &self.restore_claims_cleared, - ), - ( - "removed abandoned restore target(s)", - &self.restore_targets_removed, - ), - ( - "finished interrupted DeleteBackup(s) of", - &self.delete_claims_finished, - ), - ( - "left interrupted DeleteBackup(s) to a later sweep (a restore target exists) of", - &self.delete_claims_waiting, - ), - ]; - let mut first = true; - for (label, names) in parts { - if names.is_empty() { - continue; - } - if !first { - f.write_str("; ")?; - } - first = false; - write!(f, "{label} [{}]", names.join(", "))?; - } - Ok(()) +fn lock_miss(doc: Option<&Document>) -> LockMiss { + match doc.and_then(|d| d.get_str("backup_status").ok()) { + Some("DELETING") => LockMiss::Deleting, + _ => LockMiss::NotFound, } } impl MongoEngine { - /// Start the heartbeat of `operation_id`: at once and then every - /// `heartbeat_interval` it refreshes the operation's claim on `backup_arn` - /// and, for a restore, its CREATING target. Refreshes that find no marker - /// do nothing; each failed update is logged. - pub(crate) fn start_backup_heartbeat( - &self, - operation_id: &str, - account_id: &str, - backup_arn: &str, - target_table: Option<&str>, - ) -> Heartbeat { - let catalog = self.catalog_db.clone(); - let interval = self.backup_liveness.heartbeat_interval; - let operation_id = operation_id.to_owned(); - let account_id = account_id.to_owned(); - let backup_arn = backup_arn.to_owned(); - let target_table = target_table.map(str::to_owned); - Heartbeat(tokio::spawn(async move { - loop { - for error in refresh_heartbeats( - &catalog, - &operation_id, - &account_id, - &backup_arn, - target_table.as_deref(), - ) - .await - { - tracing::warn!( - "backup heartbeat for operation {operation_id} on {backup_arn} failed: \ - {error}" - ); - } - tokio::time::sleep(interval).await; - } - })) - } - - /// Remove the backup markers and restore targets whose owner stopped - /// heartbeating. Safe while other processes share the catalog: every step - /// re-checks staleness in the same statement that acts, so a marker whose - /// owner refreshed it meanwhile is left alone. Run at startup and - /// periodically by the GSI backfill worker. - /// - /// - A stale restore claim is removed. If its restore had already created - /// its target, the target carries its own heartbeat and is judged by it. - /// - A stale CREATING restore target that has not finished its base copy - /// is removed completely: catalog document (which is its provenance), data - /// collection, and index collections and documents, so DeleteBackup of - /// the source and the target name are free again. - /// - A stale delete claim whose backup has a CREATING restore target is - /// left as it is; a later sweep finishes the delete once the target is - /// ACTIVE or removed. Otherwise the claim is taken over (re-claimed under - /// a new operation id and heartbeat) and the delete is finished: the - /// caller already received an answer or died waiting for one, the - /// collection drop is idempotent, and leaving the backup AVAILABLE could - /// point at data the crash already dropped. The sweep never releases a - /// delete claim: the stalled owner may still drop the data. - /// - /// Targets are swept before delete claims, so a delete blocked only by a - /// dead restore is finished in the same pass. - pub async fn sweep_stale_backup_claims(&self) -> Result { - let stale_after = self.backup_liveness.stale_after; - let backups_coll = self.catalog_db.collection::("backups"); - let tables_coll = self.catalog_db.collection::("tables"); - let mut sweep = BackupSweep::default(); - - // 1. Stale restore claims. - let stale: Vec = backups_coll - .find(stale_claim_filter( - RESTORE_CLAIM, - RESTORE_CLAIM_HEARTBEAT, - stale_after, - )) - .await - .map_err(internal)? - .try_collect() - .await - .map_err(internal)?; - for meta in stale { - let (Ok(arn), Some(owner)) = (meta.get_str("_id"), meta.get(RESTORE_CLAIM)) else { - continue; - }; - let mut filter = doc! { "_id": arn }; - filter.insert(RESTORE_CLAIM, owner.clone()); - filter.insert( - "$expr", - heartbeat_is_stale(RESTORE_CLAIM_HEARTBEAT, stale_after), - ); - let result = backups_coll - .update_one(filter, BackupMetadataClaim::unset_update(RESTORE_CLAIM)) - .await - .map_err(internal)?; - if result.modified_count == 1 { - tracing::warn!( - "backup sweep: cleared the restore claim on {arn} (operation {owner}); its \ - heartbeat stopped" - ); - sweep.restore_claims_cleared.push(arn.to_owned()); - } - } - - // 2. Abandoned restore targets. - let stale: Vec = tables_coll - .find(stale_target_filter(stale_after)) - .await - .map_err(internal)? - .try_collect() - .await - .map_err(internal)?; - for target in stale { - let (Ok(id), Ok(table_id)) = (target.get_document("_id"), target.get_str("table_id")) - else { - continue; - }; - let mut filter = stale_target_filter(stale_after); - filter.insert("_id", id.clone()); - filter.insert("table_id", table_id); - let removed = tables_coll - .find_one_and_delete(filter) - .await - .map_err(internal)?; - let Some(removed) = removed else { - continue; - }; - let name = format!( - "{}/{}", - id.get_str("account_id").unwrap_or("?"), - id.get_str("table_name").unwrap_or("?") - ); - let source = removed - .get_str("restore_source_backup_arn") - .unwrap_or("?") - .to_owned(); - match self.remove_target_collections(table_id).await { - Ok(()) => tracing::warn!( - "backup sweep: removed abandoned restore target {name} (table_id \ - {table_id}, restoring {source}); its restore stopped heartbeating before \ - its copy finished" - ), - Err(error) => tracing::error!( - "backup sweep: removed abandoned restore target {name} (table_id \ - {table_id}, restoring {source}) from the catalog, but dropping its \ - collections failed: {error}" - ), - } - sweep.restore_targets_removed.push(name); - } - - // 3. Stale delete claims. - let stale: Vec = backups_coll - .find(stale_claim_filter( - DELETE_CLAIM, - DELETE_CLAIM_HEARTBEAT, - stale_after, - )) - .await - .map_err(internal)? - .try_collect() - .await - .map_err(internal)?; - for meta in stale { - let (Ok(arn), Ok(account_id), Some(owner)) = ( - meta.get_str("_id"), - meta.get_str("account_id"), - meta.get(DELETE_CLAIM), - ) else { - continue; - }; - // A CREATING restore target of this backup: leave the claim for a - // later sweep, which finishes the delete once the target is - // ACTIVE or removed. Nothing is written, so the next sweep looks - // again. - if tables_coll - .find_one(restoring_target_filter(account_id, arn)) - .await - .map_err(internal)? - .is_some() - { - tracing::debug!( - "backup sweep: the stale DeleteBackup claim on {arn} waits for a running \ - restore from it" - ); - continue; - } - let sweeper = uuid::Uuid::new_v4().to_string(); - let mut filter = doc! { "_id": arn }; - filter.insert(DELETE_CLAIM, owner.clone()); - filter.insert( - "$expr", - heartbeat_is_stale(DELETE_CLAIM_HEARTBEAT, stale_after), - ); - let taken = backups_coll - .update_one( - filter, - claim_update(DELETE_CLAIM, DELETE_CLAIM_HEARTBEAT, &sweeper), - ) - .await - .map_err(internal)?; - if taken.modified_count != 1 { - continue; - } - // The sweep never releases a delete claim it took over: the - // stalled owner may still be about to drop the data, and only a - // held claim keeps the backup from reading as AVAILABLE once the - // data is gone. Any early return below leaves the claim held by - // `sweeper`; its heartbeat stops when this function returns, and a - // later sweep takes it over again. - let _heartbeat = self.start_backup_heartbeat(&sweeper, account_id, arn, None); - let mut sweeper_filter = doc! { "_id": arn, "account_id": account_id }; - sweeper_filter.insert(DELETE_CLAIM, &sweeper); - // A restore whose claim was swept while it stalled can insert its - // target after the check above; it aborts that target when it sees - // this claim, and a later sweep finishes the delete. - if tables_coll - .find_one(restoring_target_filter(account_id, arn)) - .await - .map_err(internal)? - .is_some() - { - tracing::warn!( - "backup sweep: took over the stale DeleteBackup claim on {arn}, but a \ - restore target from it exists; a later sweep finishes the delete" - ); - sweep.delete_claims_waiting.push(arn.to_owned()); - continue; - } - if let Ok(backup_id) = meta.get_str("backup_id") { - self.drop_backup_collection(backup_id).await?; - } - backups_coll - .update_one( - sweeper_filter, - doc! { - "$set": { "backup_status": "DELETED" }, - "$unset": { DELETE_CLAIM: "", DELETE_CLAIM_HEARTBEAT: "" }, - }, - ) - .await - .map_err(internal)?; - tracing::warn!( - "backup sweep: finished the interrupted DeleteBackup of {arn} (operation {owner})" - ); - sweep.delete_claims_finished.push(arn.to_owned()); - } - Ok(sweep) - } - /// Drop a backup's data collection (missing is success). In test builds /// [`BackupLiveness::after_backup_drop`] can make it fail or hang after the /// drop ran. @@ -1057,7 +436,10 @@ impl MongoEngine { /// Drop a removed restore target's data and index collections and its /// index documents. Tries every step and returns the first error. - async fn remove_target_collections(&self, table_id: &str) -> Result<(), StorageError> { + pub(crate) async fn remove_target_collections( + &self, + table_id: &str, + ) -> Result<(), StorageError> { let mut cleanup_error = None; if let Err(e) = self .drop_collection_if_exists(&data_collection_name(table_id)) @@ -1072,41 +454,6 @@ impl MongoEngine { cleanup_error.map_or(Ok(()), Err) } - /// Remove a partial restore target without allowing its provenance to keep - /// the source backup in use. The full predicate ensures cleanup can only - /// claim the target created by this restore. - async fn abort_backup_restore( - &self, - account_id: &str, - target_table_name: &str, - backup_arn: &str, - restore_operation_id: &str, - ) -> Result { - let removed = self - .catalog_db - .collection::("tables") - .find_one_and_delete(doc! { - "_id": { "account_id": account_id, "table_name": target_table_name }, - "restore_source_backup_arn": backup_arn, - "restore_operation_id": restore_operation_id, - "table_status": "CREATING", - "status_transition_at": Bson::Null, - }) - .await - .map_err(internal)?; - let Some(removed) = removed else { - return Ok(false); - }; - let table_id = removed - .get_str("table_id") - .map_err(|_| StorageError::Internal("restore target missing table_id".to_owned()))?; - - // The catalog row is already gone, so even a failed physical cleanup - // cannot leave provenance that blocks DeleteBackup. - self.remove_target_collections(table_id).await?; - Ok(true) - } - /// This process's clock, for the one heartbeat that cannot use the /// server's (the target insert). Test builds can set it behind. fn local_now(&self) -> bson::DateTime { @@ -1122,10 +469,9 @@ impl MongoEngine { /// `TableBeingRestored` when `target_table_name` is a CREATING restore /// target, naming its source backup; `Ok(())` otherwise. Unfenced: the - /// target insert is the fence. Checked before the restore takes a slot - /// and again under the claim, so a restore into such a name is refused as - /// the name (the service's TableInUseException) at once, whatever state - /// the backup is in, as on the other backends. + /// target insert is the fence. Checked first, so a restore into such a + /// name is refused as the name (the service's TableInUseException) at + /// once, whatever state the backup is in, as on the other backends. async fn refuse_a_name_being_restored( &self, account_id: &str, @@ -1151,345 +497,308 @@ impl MongoEngine { Ok(()) } - /// Phase one of a restore: install `restore_in_progress = operation_id` - /// atomically with the AVAILABLE and account check, excluding - /// DeleteBackup's claim, then check that no CREATING target is restoring - /// this backup already. Returns the backup document. + /// Register a restore: one session transaction that locks the backup + /// document (`$inc lock_seq` while AVAILABLE), checks that no CREATING + /// target is restoring the backup, and inserts the CREATING target with + /// its provenance and its RUNNING job record. The transaction commits + /// right after the insert, before the target's collections and indexes + /// are created; a transaction that lost a write conflict (a DeleteBackup + /// or another registration of the backup) is re-run and sees the other's + /// result. Returns the target's description and the backup's data + /// collection id. /// - /// Only the claim holder inserts a target, and it releases the claim only - /// after that insert, so the claim followed by the target check is what - /// makes "one restore of a backup at a time" hold against a simultaneous - /// request: the second either finds the claim held or finds the first's - /// target. Both are `BackupInUse`, as the service answers while a restore - /// from the backup is still CREATING. The guarantee holds for as long as - /// the holder's heartbeat does: a holder stalled past the stale window - /// between claim and insert loses the claim to the sweep and a second - /// restore can be admitted (see the module doc). A claim held by - /// DeleteBackup is `BackupInUse` too. Missing, another account's, or not - /// AVAILABLE: `Backup not found`. A claim that was released between the - /// claim attempt and the re-read (a restore that failed before its - /// insert) is retried for up to [`RESTORE_CLAIM_RETRY_FOR`]. - async fn claim_backup_for_restore( + /// Refusals: a missing, another account's, DELETED or DELETING backup is + /// `Backup not found`; a CREATING target of the backup is `BackupInUse` + /// naming it, or `TableBeingRestored` when that target is the name asked + /// for; a taken name is what `create_table_for_restore` reports. + async fn register_restore( &self, account_id: &str, - backup_arn: &str, - target_table_name: &str, - operation_id: &str, - ) -> Result { + request: &RestoreRequest, + owner_generation: u64, + ) -> Result<(TableDescription, String), StorageError> { + let RestoreRequest { + backup_arn, + source_table_arn, + restore_date_time, + input, + } = request; + let target_table_name = input.table_name.as_str(); let backups_coll = self.catalog_db.collection::("backups"); - // The name again, unfenced, under the claim: the state may have - // changed while this restore waited for a slot. The target insert - // stays the fence. - self.refuse_a_name_being_restored(account_id, target_table_name) - .await?; - let started = Instant::now(); - let mut delay = Duration::from_millis(10); - let doc = loop { - if let Some(doc) = backups_coll - .find_one_and_update( - restore_claim_filter(account_id, backup_arn), - claim_update(RESTORE_CLAIM, RESTORE_CLAIM_HEARTBEAT, operation_id), - ) - .await - .map_err(internal)? - { - break doc; - } - let current = backups_coll - .find_one(doc! { "_id": backup_arn, "account_id": account_id }) + let mut session = self.client.start_session().await.map_err(internal)?; + let tx_options = self.write_transaction_options(); + for attempt in 0..TRANSIENT_RETRY_ATTEMPTS { + session + .start_transaction() + .with_options(tx_options.clone()) .await .map_err(internal)?; - match restore_claim_miss(current.as_ref()) { - RestoreClaimMiss::NotFound => { - return Err(StorageError::Validation(format!( + let body: Result, TxErr> = async { + let locked = backups_coll + .find_one_and_update( + doc! { + "_id": backup_arn, + "account_id": account_id, + "backup_status": "AVAILABLE", + }, + doc! { "$inc": { LOCK_SEQ: 1_i64 } }, + ) + .session(&mut session) + .await + .map_err(TxErr::from)?; + let Some(backup_doc) = locked else { + // Missing, another account's, DELETED, or DELETING: the + // backup is not there to restore from (a DELETING one has + // its data dropped by the DeleteBackup that won). + return Ok(Err(StorageError::Validation(format!( "Backup not found: {backup_arn}" - ))); - } - RestoreClaimMiss::DeleteClaimed => { - return Err(StorageError::BackupInUse(format!( - "Backup is being deleted: {backup_arn}" - ))); + )))); + }; + let backup_id = + backup_doc + .get_str("backup_id") + .map(str::to_owned) + .map_err(|_| { + TxErr::Fatal(StorageError::Internal(format!( + "backup metadata for {backup_arn} has no backup_id" + ))) + })?; + // Under the lock: a CREATING target of this backup belongs to a + // restore that is still running (or one abandoned and not yet + // removed, which blocks DeleteBackup the same way). When that + // target is the name this restore asked for, the name is what + // is refused (TableBeingRestored, the service's + // TableInUseException), as on the other backends. + if let Some(restoring) = self + .creating_restore_target(account_id, backup_arn, Some(&mut session)) + .await + .map_err(TxErr::from)? + { + return Ok(Err(if restoring == target_table_name { + StorageError::TableBeingRestored { + table_name: target_table_name.to_owned(), + backup_arn: backup_arn.clone(), + } + } else { + backup_in_use(&restoring, backup_arn) + })); } - RestoreClaimMiss::RestoreClaimed => { - return Err(StorageError::BackupInUse(format!( - "Backup is being used to restore another table: {backup_arn}" - ))); + let mut registration = RestoreRegistration { + backup_arn, + source_table_arn, + restore_date_time: epoch_secs_datetime(*restore_date_time), + owner_generation, + first_heartbeat: self.local_now(), + session: Some(&mut session), + }; + match self + .create_table_for_restore(account_id, input.clone(), &mut registration) + .await + { + Ok(desc) => Ok(Ok((desc, backup_id))), + // The insert's transaction lost a write conflict and was + // aborted before anything outside it was created. + Err(StorageError::Transient(_)) => Err(TxErr::Transient), + Err(e) => { + // A creation that failed after its insert committed + // (a collection or index could not be created) left a + // registered target under this generation; a name + // refusal left nothing, and the removal matches + // nothing. + if let Err(cleanup) = self + .remove_target_of_generation( + account_id, + target_table_name, + owner_generation, + ) + .await + { + tracing::error!( + "restore of {backup_arn} into {target_table_name} failed during \ + target creation ({e}), and cleanup failed: {cleanup}" + ); + } + Ok(Err(e)) + } } - // Free: the other claim was released between the two reads. - RestoreClaimMiss::Free => {} - } - if started.elapsed() + delay > RESTORE_CLAIM_RETRY_FOR { - return Err(StorageError::BackupInUse(format!( - "Backup is being used by another restore that is starting; retry: \ - {backup_arn}" - ))); } - // Jitter, so restores that collided do not retry in step. - let jitter_ms = u64::try_from(uuid::Uuid::new_v4().as_u128() % 10).unwrap_or(0); - tokio::time::sleep(delay + Duration::from_millis(jitter_ms)).await; - delay = (delay * 2).min(Duration::from_millis(200)); - }; - // Under the claim: a CREATING target of this backup belongs to a - // restore that is still running (or one abandoned and not yet swept, - // which blocks DeleteBackup the same way). When that target is the - // name this restore asked for, the name is what is refused - // (TableBeingRestored, the service's TableInUseException), as on the - // other backends; a CREATING restore target of another backup under - // the name is caught by the insert. - let restoring = self - .catalog_db - .collection::("tables") - .find_one(restoring_target_filter(account_id, backup_arn)) .await; - match restoring { - Ok(None) => Ok(doc), - Ok(Some(target)) => { - let target_name = target - .get_document("_id") - .and_then(|id| id.get_str("table_name")) - .unwrap_or("?"); - let mut claim = BackupMetadataClaim::new( - backups_coll, - account_id, - backup_arn, - RESTORE_CLAIM, - operation_id.to_owned(), - ); - claim.release().await?; - if target_name == target_table_name { - return Err(StorageError::TableBeingRestored { - table_name: target_table_name.to_owned(), - backup_arn: backup_arn.to_owned(), + match body { + // The insert committed its transaction before the target's + // collections were created; nothing is left to commit. + Ok(Ok(registered)) => { + let (mut desc, backup_id) = registered; + desc.restore_summary = Some(extenddb_core::types::RestoreSummary { + source_backup_arn: Some(backup_arn.clone()), + source_table_arn: Some(source_table_arn.clone()), + restore_date_time: *restore_date_time, + restore_in_progress: true, }); + return Ok((desc, backup_id)); + } + Ok(Err(refusal)) => { + let _ = session.abort_transaction().await; + return Err(refusal); + } + Err(TxErr::Transient) => { + let _ = session.abort_transaction().await; + backoff_sleep(attempt).await; + } + Err(TxErr::Fatal(e)) => { + let _ = session.abort_transaction().await; + return Err(e); } - Err(StorageError::BackupInUse(format!( - "Backup is being used to restore table {target_name}: {backup_arn}" - ))) - } - Err(e) => { - let mut claim = BackupMetadataClaim::new( - backups_coll, - account_id, - backup_arn, - RESTORE_CLAIM, - operation_id.to_owned(), - ); - claim.release().await?; - Err(internal(e)) } } + Err(StorageError::Internal(format!( + "the registration of the restore of {backup_arn} into {target_table_name} did not \ + commit after {TRANSIENT_RETRY_ATTEMPTS} conflicting attempts" + ))) } - /// Phase two of a restore, once the target document exists: replace the - /// target's local-clock heartbeat with the server's, then release the - /// backup claim. `Err` means the claim had been swept and the backup is - /// gone or being deleted, so the caller must abort the target. - async fn hand_claim_to_target( + /// Remove the CREATING restore target `target_table_name` registered + /// under `owner_generation`, with its collections; `Ok(false)` when there + /// is none. + async fn remove_target_of_generation( &self, - restore_claim: &mut BackupMetadataClaim, account_id: &str, - backup_arn: &str, target_table_name: &str, - operation_id: &str, - ) -> Result<(), StorageError> { - // A process whose clock is behind the server's must not look stale to - // another process's sweep before its first periodic refresh. - for error in refresh_heartbeats( - &self.catalog_db, - operation_id, - account_id, - backup_arn, - Some(target_table_name), - ) - .await - { - tracing::warn!( - "restore of {backup_arn} into {target_table_name}: first heartbeat refresh \ - failed: {error}" - ); - } - match restore_claim.release().await? { - true => Ok(()), - // A sweep removed the claim because its heartbeat lapsed (this - // process stalled for longer than the stale window). A DeleteBackup - // may have claimed and dropped the backup in the gap. The target - // now exists, so any later DeleteBackup sees it; one that started - // earlier is visible as a delete claim or a DELETED status. - false => match self - .catalog_db - .collection::("backups") - .find_one(doc! { - "_id": backup_arn, - "account_id": account_id, - "backup_status": "AVAILABLE", - DELETE_CLAIM: { "$exists": false }, - }) - .await - .map_err(internal)? - { - Some(_) => Ok(()), - None => Err(StorageError::Validation(format!( - "Backup not found: {backup_arn}" - ))), - }, - } - } - - /// Phase one and two of a restore: claim the backup, insert the CREATING - /// target with provenance, release the claim. Returns the target's - /// description and the backup's data collection id. - async fn register_restore( - &self, - account_id: &str, - request: RestoreRequest, - operation_id: &str, - ) -> Result<(TableDescription, String), StorageError> { - let RestoreRequest { - backup_arn, - source_table_arn, - restore_date_time, - input, - } = request; - let target_table_name = input.table_name.clone(); - let backups_coll = self.catalog_db.collection::("backups"); - let backup_doc = self - .claim_backup_for_restore(account_id, &backup_arn, &target_table_name, operation_id) - .await?; - let mut restore_claim = BackupMetadataClaim::new( - backups_coll.clone(), - account_id, - &backup_arn, - RESTORE_CLAIM, - operation_id.to_owned(), - ); - let Ok(backup_id) = backup_doc.get_str("backup_id").map(str::to_owned) else { - let error = StorageError::Internal(format!( - "backup metadata for {backup_arn} has no backup_id" - )); - if let Err(cleanup) = restore_claim.release().await { - tracing::error!( - "restore of {backup_arn} failed ({error}) and its metadata claim could not \ - be cleared: {cleanup}" - ); - } - return Err(error); + owner_generation: u64, + ) -> Result { + let removed = self + .catalog_db + .collection::("tables") + .find_one_and_delete(doc! { + "_id": { "account_id": account_id, "table_name": target_table_name }, + "table_status": "CREATING", + RESTORE_JOB_GENERATION: generation_field(owner_generation), + }) + .await + .map_err(internal)?; + let Some(removed) = removed else { + return Ok(false); }; + let table_id = removed + .get_str("table_id") + .map_err(|_| StorageError::Internal("restore target missing table_id".to_owned()))?; + self.remove_target_collections(table_id).await?; + Ok(true) + } - // Provenance is part of the initial CREATING document. The claim is - // kept until that document exists, so protection overlaps and - // DeleteBackup can observe neither an unclaimed backup nor an - // ownerless target; it is released as soon as the document is - // inserted, while the target's collections and indexes are still - // being created. - // - // The creation and the hand-over are polled together until both have - // finished. An earlier shape selected the hand-over and only then - // awaited the rest of the creation; that left the creation's - // in-flight command (its `createCollection`, issued right after the - // insert) holding a pooled connection with its reply unread while - // the hand-over waited for connections of its own. Enough restores in - // that state at once held every pooled connection, and the driver's - // checkout never times out: the process wedged (20 concurrent - // restores of distinct backups on the default pool of 10). - let inserted = tokio::sync::Notify::new(); - let provenance = RestoreProvenance { - backup_arn: &backup_arn, - source_table_arn: &source_table_arn, - restore_date_time: epoch_secs_datetime(restore_date_time), - operation_id, - first_heartbeat: self.local_now(), - inserted: Some(&inserted), + /// The heartbeat of a running restore: refresh the job record's + /// `heartbeat_at` with the server's clock at once and then every + /// [`BackupLiveness::heartbeat_interval`], for as long as the record is + /// RUNNING under `owner_generation`. Returns when a refresh matches other + /// than exactly one document or fails: the claim is lost and the caller + /// stops the copy. + async fn restore_heartbeat_loop( + &self, + account_id: &str, + target_table_name: &str, + table_id: &str, + owner_generation: u64, + ) -> StorageError { + let tables_coll = self.catalog_db.collection::("tables"); + let filter = doc! { + "_id": { "account_id": account_id, "table_name": target_table_name }, + "table_id": table_id, + RESTORE_JOB_STATUS: JobStatus::Running.as_str(), + RESTORE_JOB_GENERATION: generation_field(owner_generation), }; - let (created, handed) = { - let create = self.create_table_for_restore(account_id, input, &provenance); - let hand = async { - inserted.notified().await; - self.hand_claim_to_target( - &mut restore_claim, - account_id, - &backup_arn, - &target_table_name, - operation_id, - ) - .await - }; - tokio::pin!(create); - tokio::pin!(hand); - let mut created: Option> = None; - let mut handed: Option> = None; - // A creation that failed may have failed before its insert, in - // which case the hand-over never starts; a successful one always - // notified, so its hand-over is awaited to the end. - while created.is_none() - || (handed.is_none() && created.as_ref().is_some_and(Result::is_ok)) + loop { + #[cfg(test)] + if let Some(cell) = &self.backup_liveness.heartbeat_fault + && cell + .compare_exchange( + HeartbeatFault::ARMED, + HeartbeatFault::INJECTED, + std::sync::atomic::Ordering::SeqCst, + std::sync::atomic::Ordering::SeqCst, + ) + .is_ok() { - tokio::select! { - result = &mut create, if created.is_none() => created = Some(result), - result = &mut hand, if handed.is_none() => handed = Some(result), - } - } - (created.expect("creation polled to completion"), handed) - }; - let mut desc = match created { - Ok(desc) => desc, - Err(error) => { - if let Err(cleanup) = self - .abort_backup_restore(account_id, &target_table_name, &backup_arn, operation_id) - .await - { - tracing::error!( - "restore of {backup_arn} into {target_table_name} failed during target \ - creation ({error}), and cleanup failed: {cleanup}" - ); - } - if handed.is_none() - && let Err(cleanup) = restore_claim.release().await - { - tracing::error!( - "restore of {backup_arn} failed during target creation ({error}), and \ - its metadata claim could not be cleared: {cleanup}" - ); - } - return Err(error); + return StorageError::Internal( + "restore heartbeat: injected connection error".to_owned(), + ); } - }; - let still_ours = handed.expect("a created target was notified and handed over"); - if let Err(error) = still_ours { - if let Err(cleanup) = self - .abort_backup_restore(account_id, &target_table_name, &backup_arn, operation_id) + match tables_coll + .update_one( + filter.clone(), + vec![doc! { "$set": { RESTORE_JOB_HEARTBEAT: "$$NOW" } }], + ) .await { - tracing::error!( - "restore of {backup_arn} lost its metadata claim ({error}) and target cleanup \ - failed: {cleanup}" - ); + Ok(result) if result.matched_count == 1 => {} + Ok(result) => { + return StorageError::Internal(format!( + "the restore job record no longer carries this claim ({} document(s) \ + matched)", + result.matched_count + )); + } + Err(e) => return StorageError::Internal(format!("restore heartbeat: {e}")), } - return Err(error); + tokio::time::sleep(self.backup_liveness.heartbeat_interval).await; } + } - desc.restore_summary = Some(extenddb_core::types::RestoreSummary { - source_backup_arn: Some(backup_arn), - source_table_arn: Some(source_table_arn), - restore_date_time, - restore_in_progress: true, - }); - Ok((desc, backup_id)) + /// RUNNING to FAILED on the job record of the target, conditioned on + /// `owner_generation`: the write a failing job makes before it removes + /// its target. `Ok(false)` when the record is not RUNNING under that + /// generation. + async fn fail_restore_job_as( + &self, + account_id: &str, + target_table_name: &str, + table_id: &str, + owner_generation: u64, + reason: &str, + ) -> Result { + let result = self + .catalog_db + .collection::("tables") + .update_one( + doc! { + "_id": { "account_id": account_id, "table_name": target_table_name }, + "table_id": table_id, + RESTORE_JOB_STATUS: JobStatus::Running.as_str(), + RESTORE_JOB_GENERATION: generation_field(owner_generation), + }, + vec![doc! { "$set": { + RESTORE_JOB_STATUS: { "$literal": JobStatus::Failed.as_str() }, + RESTORE_JOB_FAILURE_REASON: { "$literal": reason }, + RESTORE_JOB_COMPLETED_AT: "$$NOW", + } }], + ) + .await + .map_err(internal)?; + Ok(result.matched_count == 1) } - /// Phase three, in the detached task: copy the backup's items into the - /// target with `$out`, then hand the target to the backfill worker. On - /// failure the target is removed. Logs the outcome either way. + /// The detached copy of a restore: copy the backup's items into the + /// target with `$out` while the job record is heartbeated, then hand the + /// target to the backfill worker, marking the record DONE in the same + /// write. On failure the record is marked FAILED and the target removed, + /// each conditioned on the generation. A lost heartbeat takes that path + /// at once, dropping the copy future; the `$out` already sent is a write + /// stage, which the server does not interrupt when its client + /// disconnects, so it runs to its end there and then fails: the removal + /// dropped the target's data collection, which the registration created + /// with its indexes, and `$out` refuses to replace a target whose indexes + /// changed while it ran, so it leaves no collection behind (MongoDB 7.0, + /// error 125 "indexes of target collection changed during processing"). + /// Logs the outcome either way. async fn copy_restore( &self, account_id: &str, target_table_name: &str, backup_arn: &str, - operation_id: &str, + owner_generation: u64, backup_id: &str, table_id: &str, ) { let started = Instant::now(); - let copied = async { + let copy = async { #[cfg(test)] if let Some(gate) = &self.backup_liveness.copy_gate { gate.acquire().await.map_err(internal)?.forget(); @@ -1517,43 +826,60 @@ impl MongoEngine { // The base copy is complete, but restored secondary-index // collections are still empty. Mark the table pending restore // completion and leave its indexes CREATING so the shared worker - // can backfill them in bounded, restartable batches. Dropping the - // operation id and heartbeat hands the target to that worker; the - // sweep no longer judges it. - let mut filter = doc! { - "_id": { "account_id": account_id, "table_name": target_table_name }, - "table_id": table_id, - "table_status": "CREATING", - }; - filter.insert(TARGET_OPERATION, operation_id); + // can backfill them in bounded, restartable batches, and mark the + // job record DONE in the same write, conditioned on its generation + // and on RUNNING. From here the worker owns the target: it + // promotes each index and flips the table ACTIVE, conditioned on + // the flag and on CREATING, and the reconciler leaves a DONE + // record alone. let handed_over = self .catalog_db .collection::("tables") .update_one( - filter, doc! { - "$set": { - "item_count": i64::try_from(item_count).unwrap_or(i64::MAX), - "restore_backfill_pending": true, - }, - "$unset": { TARGET_OPERATION: "", TARGET_HEARTBEAT: "" }, + "_id": { "account_id": account_id, "table_name": target_table_name }, + "table_id": table_id, + "table_status": "CREATING", + RESTORE_JOB_STATUS: JobStatus::Running.as_str(), + RESTORE_JOB_GENERATION: generation_field(owner_generation), }, + vec![doc! { "$set": { + "item_count": { "$literal": i64::try_from(item_count).unwrap_or(i64::MAX) }, + "restore_backfill_pending": { "$literal": true }, + RESTORE_JOB_STATUS: { "$literal": JobStatus::Done.as_str() }, + RESTORE_JOB_COMPLETED_AT: "$$NOW", + } }], ) .await .map_err(internal)?; Ok::<_, StorageError>((item_count, handed_over.matched_count == 1)) - } - .await; + }; + let heartbeat = + self.restore_heartbeat_loop(account_id, target_table_name, table_id, owner_generation); + let outcome = tokio::select! { + copied = copy => copied, + lost = heartbeat => { + tracing::error!( + "restore of {backup_arn} into {target_table_name} (account {account_id}): the \ + job record's heartbeat failed ({lost}); stopping the copy" + ); + Err(StorageError::Internal(format!("claim lost: {lost}"))) + } + }; - match copied { + match outcome { Ok((items, true)) => tracing::info!( "restore of {backup_arn} into {target_table_name}: copied {items} item(s) in \ {:.1}s; index backfill pending", started.elapsed().as_secs_f64() ), Ok((items, false)) => { - // The target was removed while `$out` ran (a sweep judged it - // abandoned). `$out` may have recreated its data collection. + // The target was removed while `$out` ran (the reconciler + // judged it abandoned), or the record passed to another + // claim. `$out` does not recreate the dropped data collection + // (module doc); the drop here covers a removal that ran + // between `$out`'s rename and the hand-over write, after + // which the collection exists with no document naming it. let orphan = self .catalog_db .collection::("tables") @@ -1571,24 +897,188 @@ impl MongoEngine { {dropped:?})" ); } - Err(error) => match self - .abort_backup_restore(account_id, target_table_name, backup_arn, operation_id) - .await - { - Ok(true) => tracing::error!( - "restore of {backup_arn} into {target_table_name} failed and the partial \ - target was removed: {error}" - ), - Ok(false) => tracing::error!( - "restore of {backup_arn} into {target_table_name} failed after its target \ - was already removed: {error}" - ), - Err(cleanup) => tracing::error!( - "restore of {backup_arn} into {target_table_name} failed ({error}), and \ - physical cleanup also failed: {cleanup}" - ), + Err(error) => { + // The target must not stay behind: it is CREATING with no + // scheduled transition, so nothing else would move it on, and + // the name would stay taken. The job record is marked FAILED + // first, conditioned on the generation and on RUNNING; only if + // that matched is the target removed. A record another + // generation owns, or a failed write, means the claim holder + // owns the target, and nothing further is touched. + let failed = self + .fail_restore_job_as( + account_id, + target_table_name, + table_id, + owner_generation, + &error.to_string(), + ) + .await; + match failed { + Ok(true) => {} + Ok(false) => { + tracing::warn!( + "restore of {backup_arn} into {target_table_name} ended with \ + \"{error}\"; its job record was no longer RUNNING under this claim, \ + so the target ({table_id}) was left to its owner" + ); + return; + } + Err(write) => { + tracing::error!( + "restore of {backup_arn} into {target_table_name} failed ({error}), \ + and its job record could not be marked FAILED ({write}); the \ + reconciler removes the target once its heartbeat is stale" + ); + return; + } + } + let key = RestoreTargetKey { + account_id: account_id.to_owned(), + table_name: target_table_name.to_owned(), + table_id: table_id.to_owned(), + }; + match BackupCatalog::remove_restore_target(self, &key).await { + Ok(true) => tracing::error!( + "restore of {backup_arn} into {target_table_name} failed and the partial \ + target was removed: {error}" + ), + Ok(false) => tracing::error!( + "restore of {backup_arn} into {target_table_name} failed after its target \ + was already removed: {error}" + ), + Err(cleanup) => tracing::error!( + "restore of {backup_arn} into {target_table_name} failed ({error}), and \ + physical cleanup also failed; the reconciler retries it: {cleanup}" + ), + } + } + } + } + + /// The destructive half of the legacy DeleteBackup, from a document that + /// is DELETING: drop the backup's data collection, then write DELETED. + /// The drop may have run on the server even if this future is dropped + /// while waiting for its reply or the reply is an error; the document + /// stays DELETING either way, and a later DeleteBackup resumes here. + async fn finish_legacy_delete( + &self, + account_id: &str, + backup_arn: &str, + backup_doc: &Document, + ) -> Result { + // A backup_id is absent on a pre-$out backup, which has no physical + // collection to drop. + if let Ok(backup_id) = backup_doc.get_str("backup_id") { + self.drop_backup_collection(backup_id).await?; + } + self.catalog_db + .collection::("backups") + .update_one( + doc! { "_id": backup_arn, "account_id": account_id, "backup_status": "DELETING" }, + doc! { "$set": { "backup_status": "DELETED" } }, + ) + .await + .map_err(internal)?; + let desc = description_from_doc(backup_doc, backup_arn)?; + Ok(BackupDescription { + backup_details: BackupDetails { + backup_status: "DELETED".to_owned(), + ..desc.backup_details }, + ..desc + }) + } + + /// The legacy DeleteBackup's decision as one session transaction: lock + /// the backup document (`$inc lock_seq` while AVAILABLE), re-check for a + /// CREATING target, write DELETING. `Ok(Some(doc))` is the document as it + /// was, now DELETING; `Ok(None)` means a concurrent DeleteBackup has + /// written DELETING already and this one resumes its drop. + async fn start_legacy_delete( + &self, + account_id: &str, + backup_arn: &str, + ) -> Result, StorageError> { + let backups_coll = self.catalog_db.collection::("backups"); + let mut session = self.client.start_session().await.map_err(internal)?; + let tx_options = self.write_transaction_options(); + for attempt in 0..TRANSIENT_RETRY_ATTEMPTS { + session + .start_transaction() + .with_options(tx_options.clone()) + .await + .map_err(internal)?; + let body: Result, StorageError>, TxErr> = async { + let locked = backups_coll + .find_one_and_update( + doc! { + "_id": backup_arn, + "account_id": account_id, + "backup_status": "AVAILABLE", + }, + doc! { "$inc": { LOCK_SEQ: 1_i64 } }, + ) + .session(&mut session) + .await + .map_err(TxErr::from)?; + let Some(backup_doc) = locked else { + let current = backups_coll + .find_one(doc! { "_id": backup_arn, "account_id": account_id }) + .session(&mut session) + .await + .map_err(TxErr::from)?; + return Ok(match lock_miss(current.as_ref()) { + LockMiss::Deleting => Ok(None), + LockMiss::NotFound => Err(StorageError::Validation(format!( + "Backup not found: {backup_arn}" + ))), + }); + }; + // A restore can register its target between the unlocked + // first check and this lock; from here new registrations + // conflict with this transaction. + if let Some(target) = self + .creating_restore_target(account_id, backup_arn, Some(&mut session)) + .await + .map_err(TxErr::from)? + { + return Ok(Err(backup_in_use(&target, backup_arn))); + } + backups_coll + .update_one( + doc! { "_id": backup_arn, "account_id": account_id, "backup_status": "AVAILABLE" }, + doc! { "$set": { "backup_status": "DELETING" } }, + ) + .session(&mut session) + .await + .map_err(TxErr::from)?; + Ok(Ok(Some(backup_doc))) + } + .await; + match body { + Ok(Ok(outcome)) => match self.commit_write_transaction(&mut session, None).await? { + CommitOutcome::Committed => return Ok(outcome), + CommitOutcome::RerunBody => backoff_sleep(attempt).await, + }, + Ok(Err(refusal)) => { + let _ = session.abort_transaction().await; + return Err(refusal); + } + Err(TxErr::Transient) => { + let _ = session.abort_transaction().await; + backoff_sleep(attempt).await; + } + Err(TxErr::Fatal(e)) => { + let _ = session.abort_transaction().await; + return Err(e); + } + } } + Err(StorageError::Internal(format!( + "DeleteBackup of {backup_arn} did not commit after {TRANSIENT_RETRY_ATTEMPTS} \ + conflicting attempts" + ))) } } @@ -1695,17 +1185,90 @@ fn restore_source_from_doc( }) } -impl BackupEngine for MongoEngine { - fn create_backup( +/// The `DescribeBackup` description of a backup document, with the status as +/// stored. `SourceTableFeatureDetails` and the throughput are filled by the +/// engine from `restore_source` for an AVAILABLE backup. +fn description_from_doc( + backup_doc: &Document, + backup_arn: &str, +) -> Result { + let name = backup_doc + .get_str("backup_name") + .unwrap_or_default() + .to_owned(); + let status = backup_doc + .get_str("backup_status") + .unwrap_or("AVAILABLE") + .to_owned(); + let table_id = backup_doc + .get_str("table_id") + .unwrap_or_default() + .to_owned(); + let table_name = backup_doc + .get_str("table_name") + .unwrap_or_default() + .to_owned(); + let table_arn = backup_doc + .get_str("table_arn") + .unwrap_or_default() + .to_owned(); + let size = backup_doc.get_i64("backup_size_bytes").unwrap_or(0); + let count = backup_doc.get_i64("item_count").unwrap_or(0); + let billing = backup_doc + .get_str("billing_mode") + .unwrap_or("PAY_PER_REQUEST") + .to_owned(); + + let created_at = backup_created_at(backup_doc); + let table_created = backup_doc + .get_f64("table_creation_date_time") + .unwrap_or(created_at); + let key_schema: Vec = decode_required(backup_doc, "key_schema")?; + + Ok(BackupDescription { + backup_details: BackupDetails { + backup_arn: backup_arn.to_owned(), + backup_name: name, + backup_status: status, + backup_type: "USER".to_owned(), + backup_size_bytes: size, + backup_creation_date_time: created_at, + }, + source_table_details: SourceTableDetails { + table_name, + table_id, + table_arn, + key_schema, + item_count: count, + table_size_bytes: size, + billing_mode: Some(billing), + table_creation_date_time: table_created, + ..SourceTableDetails::default() + }, + source_table_feature_details: None, + }) +} + +/// The statuses of a backup document the legacy describe and list leave +/// out: a deleted backup and one whose DeleteBackup has started. +fn gone_statuses() -> Document { + doc! { "$nin": ["DELETED", "DELETING"] } +} + +impl LegacyBackupReader for MongoEngine { + fn create( &self, account_id: &str, table_name: &str, backup_name: &str, + permit: JobPermit, ) -> BoxFuture<'_, Result> { let account_id = account_id.to_string(); let table_name = table_name.to_string(); let backup_name = backup_name.to_string(); Box::pin(async move { + // The job's slot, held until the backup document is written. + let _permit = permit; let tables_coll = self.catalog_db.collection::("tables"); let table_doc = tables_coll .find_one(doc! { @@ -1935,7 +1498,7 @@ impl BackupEngine for MongoEngine { }) } - fn describe_backup( + fn describe( &self, account_id: &str, backup_arn: &str, @@ -1945,82 +1508,24 @@ impl BackupEngine for MongoEngine { Box::pin(async move { let backups_coll = self.catalog_db.collection::("backups"); // Scope the lookup to the calling account so a backup ARN cannot be - // read cross-account, and exclude DELETED backups so a deleted - // backup reads as BackupNotFoundException. Matches the postgres - // backend. + // read cross-account, and leave out a deleted backup and one being + // deleted, so both read as BackupNotFoundException. let backup_doc = backups_coll .find_one(doc! { "_id": &backup_arn, "account_id": &account_id, - "backup_status": { "$ne": "DELETED" }, + "backup_status": gone_statuses(), }) .await .map_err(internal)? .ok_or_else(|| { StorageError::Validation(format!("Backup not found: {backup_arn}")) })?; - - let name = backup_doc - .get_str("backup_name") - .unwrap_or_default() - .to_owned(); - let status = backup_doc - .get_str("backup_status") - .unwrap_or("AVAILABLE") - .to_owned(); - let table_id = backup_doc - .get_str("table_id") - .unwrap_or_default() - .to_owned(); - let table_name = backup_doc - .get_str("table_name") - .unwrap_or_default() - .to_owned(); - let table_arn = backup_doc - .get_str("table_arn") - .unwrap_or_default() - .to_owned(); - let size = backup_doc.get_i64("backup_size_bytes").unwrap_or(0); - let count = backup_doc.get_i64("item_count").unwrap_or(0); - let billing = backup_doc - .get_str("billing_mode") - .unwrap_or("PAY_PER_REQUEST") - .to_owned(); - - let created_at = backup_created_at(&backup_doc); - let table_created = backup_doc - .get_f64("table_creation_date_time") - .unwrap_or(created_at); - let key_schema: Vec = decode_required(&backup_doc, "key_schema")?; - - Ok(BackupDescription { - backup_details: BackupDetails { - backup_arn: backup_arn.clone(), - backup_name: name, - backup_status: status, - backup_type: "USER".to_owned(), - backup_size_bytes: size, - backup_creation_date_time: created_at, - }, - source_table_details: SourceTableDetails { - table_name, - table_id, - table_arn, - key_schema, - item_count: count, - table_size_bytes: size, - billing_mode: Some(billing), - table_creation_date_time: table_created, - // Filled by the engine from backup_restore_source for an - // AVAILABLE backup. - ..SourceTableDetails::default() - }, - source_table_feature_details: None, - }) + description_from_doc(&backup_doc, &backup_arn) }) } - fn list_backups( + fn list( &self, account_id: &str, table_name: Option<&str>, @@ -2032,7 +1537,7 @@ impl BackupEngine for MongoEngine { let mut filter = doc! { "account_id": &account_id, - "backup_status": { "$ne": "DELETED" }, + "backup_status": gone_statuses(), }; if let Some(tn) = &table_name { filter.insert("table_name", tn.as_str()); @@ -2066,7 +1571,7 @@ impl BackupEngine for MongoEngine { }) } - fn delete_backup( + fn delete( &self, account_id: &str, backup_arn: &str, @@ -2074,123 +1579,51 @@ impl BackupEngine for MongoEngine { let account_id = account_id.to_string(); let backup_arn = backup_arn.to_string(); Box::pin(async move { - let desc = self.describe_backup(&account_id, &backup_arn).await?; let backups_coll = self.catalog_db.collection::("backups"); - let tables_coll = self.catalog_db.collection::("tables"); - - // Fast-path the durable half of the two-phase restore claim. - if let Some(target) = tables_coll - .find_one(restoring_target_filter(&account_id, &backup_arn)) - .await - .map_err(internal)? - { - let name = target - .get_document("_id") - .ok() - .and_then(|id| id.get_str("table_name").ok()) - .unwrap_or("?"); - return Err(StorageError::BackupInUse(format!( - "Backup is being used to restore table {name}: {backup_arn}" - ))); - } - - // Atomically claim deletion only when restore has not claimed the - // metadata. Restore uses the inverse predicate, so once either - // marker is installed the other operation cannot start. The - // heartbeat runs from before the claim until this future ends - // (completes or is dropped). - let delete_operation_id = uuid::Uuid::new_v4().to_string(); - let _heartbeat = - self.start_backup_heartbeat(&delete_operation_id, &account_id, &backup_arn, None); - let meta = backups_coll - .find_one_and_update( - delete_claim_filter(&account_id, &backup_arn), - claim_update(DELETE_CLAIM, DELETE_CLAIM_HEARTBEAT, &delete_operation_id), - ) - .return_document(mongodb::options::ReturnDocument::Before) + let current = backups_coll + .find_one(doc! { "_id": &backup_arn, "account_id": &account_id }) .await .map_err(internal)?; - let Some(meta) = meta else { - let restoring = tables_coll - .find_one(restoring_target_filter(&account_id, &backup_arn)) - .await - .map_err(internal)? - .is_some(); - let reread = backups_coll - .find_one(doc! { "_id": &backup_arn, "account_id": &account_id }) - .await - .map_err(internal)?; - return Err(delete_claim_miss_error( - restoring, - restore_claim_miss(reread.as_ref()), - &backup_arn, - )); + let current = match current.as_ref().map(|d| d.get_str("backup_status")) { + None | Some(Ok("DELETED")) => { + return Err(StorageError::Validation(format!( + "Backup not found: {backup_arn}" + ))); + } + // A DeleteBackup that stopped after writing DELETING: resume + // its drop and its DELETED write. + Some(Ok("DELETING")) => { + let doc = current.expect("a status was read from a document"); + return self + .finish_legacy_delete(&account_id, &backup_arn, &doc) + .await; + } + Some(_) => current.expect("a status was read from a document"), }; - let mut delete_claim = BackupMetadataClaim::new( - backups_coll.clone(), - &account_id, - &backup_arn, - DELETE_CLAIM, - delete_operation_id, - ); - // A restore can transfer its marker to a target between the first - // target read and our metadata claim. Recheck after claiming; new - // restores are now excluded by delete_in_progress. - if let Some(target) = tables_coll - .find_one(restoring_target_filter(&account_id, &backup_arn)) + // The in-use rule before the lock, so a backup a running restore + // reads is refused without a transaction. + if let Some(target) = self + .creating_restore_target(&account_id, &backup_arn, None) .await .map_err(internal)? { - let name = target - .get_document("_id") - .ok() - .and_then(|id| id.get_str("table_name").ok()) - .unwrap_or("?") - .to_owned(); - delete_claim.release().await?; - return Err(StorageError::BackupInUse(format!( - "Backup is being used to restore table {name}: {backup_arn}" - ))); - } - - // Drop the backup collection. If backup_id is absent (for an old - // pre-$out backup), there is no physical collection to drop. - // - // From here on the claim is never released. The drop may have run - // on the server even if this future is dropped while waiting for - // its reply or the reply is an error, and releasing would leave an - // AVAILABLE backup with no data, which a restore would copy as an - // empty table. A claim left behind (an error below, or this future - // dropped) stops heartbeating when this future ends, and the - // stale-claim sweep finishes the delete. - delete_claim.disarm(); - if let Ok(backup_id) = meta.get_str("backup_id") { - self.drop_backup_collection(backup_id).await?; + return Err(backup_in_use(&target, &backup_arn)); } - backups_coll - .update_one( - delete_claim.filter(), - doc! { - "$set": { "backup_status": "DELETED" }, - "$unset": { DELETE_CLAIM: "", DELETE_CLAIM_HEARTBEAT: "" }, - }, - ) + // The decision under the lock: AVAILABLE and no CREATING target, + // then DELETING, in one transaction a restore's registration + // conflicts with. + let backup_doc = match self.start_legacy_delete(&account_id, &backup_arn).await? { + Some(doc) => doc, + None => current, + }; + self.finish_legacy_delete(&account_id, &backup_arn, &backup_doc) .await - .map_err(internal)?; - - Ok(BackupDescription { - backup_details: BackupDetails { - backup_status: "DELETED".to_owned(), - ..desc.backup_details - }, - ..desc - }) }) } - fn backup_restore_source( + fn restore_source( &self, account_id: &str, backup_arn: &str, @@ -2215,32 +1648,32 @@ impl BackupEngine for MongoEngine { }) } - fn restore_table_from_backup( + fn restore( &self, account_id: &str, request: RestoreRequest, + permit: JobPermit, ) -> BoxFuture<'_, Result> { let engine = self.clone(); let account_id = account_id.to_owned(); Box::pin(async move { - // The whole restore, admission included, runs in a task that owns - // its inputs, so the request future being dropped (a client - // timeout) stops neither the wait for a slot, nor the - // registration, nor the copy. At most RESTORE_SLOTS_PER_ACCOUNT - // restores per account and restore_server_slot_count per process - // run at once; the permits are released when the copy ends. A - // restore refused for want of a slot creates nothing. The request - // waits only until the CREATING target exists and the backup claim - // has passed to it. + // The whole restore, registration included, runs in a task that + // owns its inputs and the job permit, so the request future being + // dropped (a client timeout) stops neither the registration nor + // the copy, and the permit is released when the copy ends. A + // refused restore creates nothing. The request waits for the + // CREATING target to exist, at most `RESTORE_REGISTRATION_TIMEOUT` + // (its doc says what a request that hit the bound leaves behind). let (registered_tx, registered_rx) = tokio::sync::oneshot::channel(); tokio::spawn(async move { + let _permit = permit; let backup_arn = request.backup_arn.clone(); let target_table_name = request.input.table_name.clone(); // A refusal the request is no longer there to receive (its // client went away, or the registration bound fired) would // otherwise vanish, so it is logged. A refused restore removes // what it created; a cleanup that failed is logged on its own, - // and the heartbeat sweep finishes it. + // and the reconciler finishes it. let refuse = |registered: tokio::sync::oneshot::Sender<_>, e: StorageError| { if let Err(Err(e)) = registered.send(Err(e)) { tracing::warn!( @@ -2249,8 +1682,9 @@ impl BackupEngine for MongoEngine { ); } }; - // The name before a slot is taken, so a client retrying its - // own restore is answered at once instead of after the wait. + // The name first, unfenced, as on the other backends: a + // restore into a name that is a CREATING restore target is + // refused as the name at once, whatever state the backup is in. if let Err(error) = engine .refuse_a_name_being_restored(&account_id, &target_table_name) .await @@ -2258,31 +1692,19 @@ impl BackupEngine for MongoEngine { refuse(registered_tx, error); return; } - let _permits = match acquire_restore_permits( - engine.restore_account_slots.for_account(&account_id), - Arc::clone(&engine.restore_server_slots), - engine.restore_server_slot_count, - engine.backup_liveness.slot_wait, - ) - .await - { - Ok(permits) => permits, - Err(error) => { - refuse(registered_tx, error); - return; + #[cfg(test)] + if let Some(gate) = &engine.backup_liveness.registration_gate { + match gate.acquire().await { + Ok(permit) => permit.forget(), + Err(e) => { + refuse(registered_tx, internal(e)); + return; + } } - }; - let operation_id = uuid::Uuid::new_v4().to_string(); - // Started before the claim so the claim and the target are - // refreshed for as long as this task runs. - let _heartbeat = engine.start_backup_heartbeat( - &operation_id, - &account_id, - &backup_arn, - Some(&target_table_name), - ); + } + let owner_generation = new_generation(); let (desc, backup_id) = match engine - .register_restore(&account_id, request, &operation_id) + .register_restore(&account_id, &request, owner_generation) .await { Ok(registered) => registered, @@ -2299,7 +1721,7 @@ impl BackupEngine for MongoEngine { &account_id, &target_table_name, &backup_arn, - &operation_id, + owner_generation, &backup_id, &table_id, ) @@ -2320,6 +1742,15 @@ impl BackupEngine for MongoEngine { }) } + /// None: this backend's create commits its backup document AVAILABLE + /// after the copy and leaves no CREATING row behind. + fn abandoned_creating_backups( + &self, + _stale_after: Duration, + ) -> BoxFuture<'_, Result, StorageError>> { + Box::pin(async { Ok(Vec::new()) }) + } + fn describe_continuous_backups( &self, account_id: &str, @@ -2411,11 +1842,7 @@ impl BackupEngine for MongoEngine { #[cfg(test)] mod tests { - use super::{ - RESTORE_SLOTS_PER_ACCOUNT, RestoreAccountSlots, delete_claim_filter, - insert_non_empty_array, restore_claim_filter, restore_gsi_throughput, - restore_provisioned_throughput, restoring_target_filter, - }; + use super::{insert_non_empty_array, restore_gsi_throughput, restore_provisioned_throughput}; use extenddb_core::types::{GsiInput, ProvisionedThroughput}; use mongodb::bson::Document; @@ -2489,91 +1916,19 @@ mod tests { None ); } - - #[test] - fn restore_and_delete_claims_exclude_each_other() { - let restore = restore_claim_filter("123456789012", "backup"); - let delete = delete_claim_filter("123456789012", "backup"); - for filter in [&restore, &delete] { - assert_eq!(filter.get_str("backup_status"), Ok("AVAILABLE")); - assert_eq!( - filter - .get_document("restore_in_progress") - .and_then(|predicate| predicate.get_bool("$exists")), - Ok(false) - ); - assert_eq!( - filter - .get_document("delete_in_progress") - .and_then(|predicate| predicate.get_bool("$exists")), - Ok(false) - ); - } - } - - #[test] - fn durable_restore_target_predicate_is_account_scoped_and_creating() { - let filter = restoring_target_filter("123456789012", "backup"); - assert_eq!(filter.get_str("_id.account_id"), Ok("123456789012")); - assert_eq!(filter.get_str("restore_source_backup_arn"), Ok("backup")); - assert_eq!(filter.get_str("table_status"), Ok("CREATING")); - } - - /// One engine's account slots are shared by its clones and by nothing - /// else: a second engine in the same process (another test's fixture) - /// has its own full allowance for the same account. - #[test] - fn account_restore_slots_belong_to_one_engine() { - let slots = RestoreAccountSlots::default(); - let held = slots - .for_account("123456789012") - .try_acquire_owned() - .expect("a free slot"); - assert_eq!( - slots - .clone() - .for_account("123456789012") - .available_permits(), - RESTORE_SLOTS_PER_ACCOUNT - 1, - "a clone sees the same account map" - ); - assert_eq!( - RestoreAccountSlots::default() - .for_account("123456789012") - .available_permits(), - RESTORE_SLOTS_PER_ACCOUNT, - "another engine has its own" - ); - drop(held); - // Nothing in flight for the account: its entry is pruned on the next - // lookup of any account, and a fresh one has the full allowance. - slots.for_account("other"); - assert_eq!( - slots.0.lock().unwrap().len(), - 1, - "the idle account's entry is gone" - ); - assert_eq!( - slots.for_account("123456789012").available_permits(), - RESTORE_SLOTS_PER_ACCOUNT - ); - } } #[cfg(test)] mod definition_tests { use super::{ - BackupMetadataClaim, BackupSweep, DELETE_CLAIM, DELETE_CLAIM_HEARTBEAT, RESTORE_CLAIM, - RESTORE_CLAIM_HEARTBEAT, RestoreClaimMiss, backup_definition_from_doc, claim_update, - delete_claim_miss_error, heartbeat_updates, restore_claim_miss, restore_source_from_doc, - stale_claim_filter, stale_target_filter, try_each, + LockMiss, RestoreRegistration, backup_definition_from_doc, description_from_doc, + gone_statuses, lock_miss, restore_source_from_doc, restoring_target_filter, }; use extenddb_core::types::{ OnDemandThroughput, ProvisionedThroughput, StreamViewType, TimeToLiveStatus, }; use extenddb_storage::error::StorageError; use mongodb::bson::{self, doc}; - use std::time::Duration; const ARN: &str = "arn:aws:dynamodb:us-east-1:123456789012:table/src/backup/01791976431438-0000abcd"; @@ -2620,59 +1975,6 @@ mod definition_tests { } } - #[test] - fn constant_keys_in_doc_macros_are_the_constant_values() { - // The update documents use `CONST: value` keys; this pins that the - // macro takes the constant's value, not its identifier. - let update = BackupMetadataClaim::unset_update(DELETE_CLAIM); - let unset = update.get_document("$unset").expect("$unset"); - assert!(unset.contains_key("delete_in_progress")); - assert!(unset.contains_key("delete_in_progress_heartbeat")); - let inline = doc! { DELETE_CLAIM_HEARTBEAT: "" }; - assert!(inline.contains_key("delete_in_progress_heartbeat")); - let claim = claim_update(RESTORE_CLAIM, RESTORE_CLAIM_HEARTBEAT, "op"); - let set = claim[0].get_document("$set").expect("$set"); - assert_eq!( - set.get_document("restore_in_progress") - .and_then(|d| d.get_str("$literal")), - Ok("op") - ); - assert_eq!(set.get_str("restore_in_progress_heartbeat"), Ok("$$NOW")); - } - - #[test] - fn stale_filters_compare_the_heartbeat_with_the_server_clock() { - let filter = stale_claim_filter( - RESTORE_CLAIM, - RESTORE_CLAIM_HEARTBEAT, - Duration::from_secs(60), - ); - assert_eq!( - filter - .get_document("restore_in_progress") - .and_then(|d| d.get_bool("$exists")), - Ok(true) - ); - let expr = filter.get_document("$expr").expect("$expr").to_string(); - assert!(expr.contains("$restore_in_progress_heartbeat"), "{expr}"); - assert!(expr.contains("$$NOW") && expr.contains("60000"), "{expr}"); - let target = stale_target_filter(Duration::from_secs(60)); - assert_eq!(target.get_str("table_status"), Ok("CREATING")); - assert_eq!( - target - .get_document("restore_backfill_pending") - .and_then(|d| d.get_bool("$exists")), - Ok(false) - ); - assert!( - target - .get_document("$expr") - .expect("$expr") - .to_string() - .contains("$restore_heartbeat_at") - ); - } - #[test] fn the_definition_is_read_from_the_backup_document() { let def = backup_definition_from_doc(&backup_doc(), "PROVISIONED", ARN).expect("def"); @@ -2798,129 +2100,102 @@ mod definition_tests { ); } - /// A failed heartbeat update does not stop the others. - #[tokio::test] - async fn try_each_runs_every_item_after_a_failure() { - let mut attempted = Vec::new(); - let errors = try_each(vec![1, 2, 3], |i| { - attempted.push(i); - async move { - if i == 1 { - Err(StorageError::Internal(format!("update {i}"))) - } else { - Ok(()) - } - } - }) - .await; - assert_eq!(attempted, [1, 2, 3]); - assert_eq!(errors.len(), 1); - assert!(matches!(&errors[0], StorageError::Internal(m) if m == "update 1")); - } - + /// The description reports the document's status as stored, so a + /// resumed DeleteBackup can answer DELETED from the document it read. #[test] - fn heartbeat_updates_cover_both_claims_and_the_target() { - let updates = heartbeat_updates("op", "123456789012", ARN, Some("dst")); - let names: Vec<_> = updates.iter().map(|u| (u.0, u.1)).collect(); + fn the_description_reports_the_documents_status_and_source_table() { + let desc = description_from_doc(&backup_doc(), ARN).expect("description"); + assert_eq!(desc.backup_details.backup_arn, ARN); + assert_eq!(desc.backup_details.backup_status, "AVAILABLE"); + assert_eq!(desc.backup_details.backup_size_bytes, 77); + assert!((desc.backup_details.backup_creation_date_time - 1_791_976_431.438).abs() < 1e-9); + assert_eq!(desc.source_table_details.table_name, "src"); + assert_eq!(desc.source_table_details.table_id, "tid"); + assert_eq!(desc.source_table_details.item_count, 3); + assert_eq!( + desc.source_table_details.billing_mode.as_deref(), + Some("PROVISIONED") + ); + assert!(desc.source_table_feature_details.is_none()); + let mut deleting = backup_doc(); + deleting.insert("backup_status", "DELETING"); assert_eq!( - names, - [ - ("backups", "restore_in_progress"), - ("backups", "delete_in_progress"), - ("tables", "restore_heartbeat_at"), - ] + description_from_doc(&deleting, ARN) + .expect("description") + .backup_details + .backup_status, + "DELETING" ); - assert_eq!(updates[0].2.get_str("restore_in_progress"), Ok("op")); - assert_eq!(updates[2].2.get_str("restore_operation_id"), Ok("op")); - assert_eq!(heartbeat_updates("op", "123456789012", ARN, None).len(), 2); } - /// Why a restore claim missed decides between retrying, BackupInUse, - /// and Backup not found. + /// The legacy describe and list leave out a deleted backup and one whose + /// DeleteBackup has started; a restore's registration and a DeleteBackup + /// tell the two apart only to decide whether a delete is resumed. #[test] - fn a_restore_claim_miss_is_classified_from_the_reread() { - let available = doc! { "_id": ARN, "backup_status": "AVAILABLE" }; - assert_eq!(restore_claim_miss(None), RestoreClaimMiss::NotFound); - let mut d = available.clone(); - d.insert("backup_status", "DELETED"); - assert_eq!(restore_claim_miss(Some(&d)), RestoreClaimMiss::NotFound); - assert_eq!( - restore_claim_miss(Some(&doc! { "_id": ARN })), - RestoreClaimMiss::NotFound - ); - assert_eq!(restore_claim_miss(Some(&available)), RestoreClaimMiss::Free); - let mut d = available.clone(); - d.insert(RESTORE_CLAIM, "other"); + fn a_deleting_or_deleted_document_is_gone_and_only_deleting_resumes_a_delete() { + let gone = gone_statuses(); + let excluded = gone.get_array("$nin").expect("$nin"); assert_eq!( - restore_claim_miss(Some(&d)), - RestoreClaimMiss::RestoreClaimed - ); - d.insert(DELETE_CLAIM, "delete"); - assert_eq!( - restore_claim_miss(Some(&d)), - RestoreClaimMiss::DeleteClaimed + excluded, + &[bson::Bson::from("DELETED"), bson::Bson::from("DELETING")] ); + assert_eq!(lock_miss(None), LockMiss::NotFound); + let mut d = doc! { "_id": ARN, "backup_status": "AVAILABLE" }; + assert_eq!(lock_miss(Some(&d)), LockMiss::NotFound); + d.insert("backup_status", "DELETED"); + assert_eq!(lock_miss(Some(&d)), LockMiss::NotFound); + d.insert("backup_status", "DELETING"); + assert_eq!(lock_miss(Some(&d)), LockMiss::Deleting); } - /// R4B-1: a DeleteBackup whose claim misses because another DeleteBackup - /// holds the backup answers BackupInUse, not Backup not found. #[test] - fn a_delete_claim_miss_on_a_delete_claimed_backup_is_in_use() { - let in_use = |e: StorageError| match e { - StorageError::BackupInUse(m) => m, - other => panic!("expected BackupInUse, got {other:?}"), - }; - for restoring in [false, true] { - let m = in_use(delete_claim_miss_error( - restoring, - RestoreClaimMiss::DeleteClaimed, - ARN, - )); - assert!(m.starts_with("Backup is being deleted"), "{m}"); - let m = in_use(delete_claim_miss_error( - restoring, - RestoreClaimMiss::RestoreClaimed, - ARN, - )); - assert!( - m.starts_with("Backup is being used to restore another table"), - "{m}" - ); - } - for miss in [RestoreClaimMiss::NotFound, RestoreClaimMiss::Free] { - let m = in_use(delete_claim_miss_error(true, miss, ARN)); - assert!( - m.starts_with("Backup is being used to restore another table"), - "{m}" - ); - assert!(matches!( - delete_claim_miss_error(false, miss, ARN), - StorageError::Validation(m) if m.starts_with("Backup not found") - )); - } + fn the_in_use_filter_is_account_scoped_and_reads_the_targets_status() { + let filter = restoring_target_filter("123456789012", "backup"); + assert_eq!(filter.get_str("_id.account_id"), Ok("123456789012")); + assert_eq!(filter.get_str("restore_source_backup_arn"), Ok("backup")); + assert_eq!(filter.get_str("table_status"), Ok("CREATING")); } + /// The registration writes the provenance DescribeTable reports and the + /// job record RUNNING, as a legacy restore, under its generation. #[test] - fn the_sweep_summary_names_what_it_removed() { - assert!(BackupSweep::default().is_empty()); - assert_eq!(BackupSweep::default().to_string(), ""); - let sweep = BackupSweep { - restore_claims_cleared: vec!["a".to_owned()], - restore_targets_removed: vec!["acct/t1".to_owned(), "acct/t2".to_owned()], - delete_claims_finished: vec![], - delete_claims_waiting: vec!["b".to_owned()], + fn a_registration_writes_the_provenance_and_a_running_legacy_job_record() { + let registration = RestoreRegistration { + backup_arn: ARN, + source_table_arn: "arn:aws:dynamodb:us-east-1:123456789012:table/src", + restore_date_time: bson::DateTime::from_millis(1_791_976_431_438), + owner_generation: 7, + first_heartbeat: bson::DateTime::from_millis(1_791_976_500_000), + session: None, }; - assert!(!sweep.is_empty()); + let mut table_doc = doc! { "table_status": "CREATING" }; + registration.insert_into(&mut table_doc); + assert_eq!(table_doc.get_str("restore_source_backup_arn"), Ok(ARN)); + assert_eq!( + table_doc.get_str("restore_source_table_arn"), + Ok("arn:aws:dynamodb:us-east-1:123456789012:table/src") + ); + assert_eq!( + table_doc + .get_datetime("restore_date_time") + .map(|d| d.timestamp_millis()), + Ok(1_791_976_431_438) + ); + let job = table_doc.get_document("restore_job").expect("job record"); + assert_eq!(job.get_str("status"), Ok("RUNNING")); + assert_eq!(job.get_str("kind"), Ok("legacy")); + assert_eq!(job.get_i64("owner_generation"), Ok(7)); assert_eq!( - sweep.to_string(), - "cleared abandoned restore claim(s) on [a]; removed abandoned restore target(s) \ - [acct/t1, acct/t2]; left interrupted DeleteBackup(s) to a later sweep (a \ - restore target exists) of [b]" + job.get_datetime("heartbeat_at") + .map(|d| d.timestamp_millis()), + Ok(1_791_976_500_000) ); + assert!(!table_doc.contains_key("restore_operation_id")); + assert!(!table_doc.contains_key("restore_heartbeat_at")); } } -/// The claim protocol model; see the module documentation. +/// The protocol model; see the module documentation. #[cfg(test)] #[path = "backup_protocol_model.rs"] mod protocol_model; diff --git a/crates/storage-mongodb/src/backup_live_tests.rs b/crates/storage-mongodb/src/backup_live_tests.rs index 75ed23761..b34afe988 100644 --- a/crates/storage-mongodb/src/backup_live_tests.rs +++ b/crates/storage-mongodb/src/backup_live_tests.rs @@ -1,8 +1,10 @@ // Copyright 2026 ExtendDB contributors // SPDX-License-Identifier: Apache-2.0 -//! Live MongoDB tests for backup and restore: the detached copy, the -//! heartbeat sweep, and the DeleteBackup/DeleteTable refusals during a copy. +//! Live MongoDB tests for backup and restore: the catalog index and its +//! claim, the detached copy and its job record, the reconciler's duties over +//! an abandoned restore, and the DeleteBackup/DeleteTable refusals during a +//! copy. //! //! They need a replica set (the data path uses transactions) and read its URI //! from `EXTENDDB_TEST_MONGODB_URI`, for example @@ -11,35 +13,46 @@ //! `cargo test -p extenddb-storage-mongodb --lib -- --ignored live_`. //! A run that sets `--ignored` without the variable fails rather than skips. //! -//! Each test uses its own catalog and data databases (random names) and drops -//! them at the end, so tests run in parallel and leave nothing on success. Each -//! runs its own GSI backfill worker, as `serve` does, and every engine uses -//! shortened heartbeat timing (200 ms refresh, 1.5 s stale), which the tests -//! read from the engine rather than assume. +//! Each test uses its own catalog and data databases (random names), runs the +//! catalog migration step on them, and drops them at the end, so tests run in +//! parallel and leave nothing on success. Each runs its own GSI backfill +//! worker, as `serve` does, and every engine uses a shortened heartbeat +//! interval (200 ms), which the tests read from the engine rather than +//! assume; the stale window is the one the reconciler would pass. use std::sync::Arc; use std::time::{Duration, Instant}; +use extenddb_core::backup_format::BackupArn; use extenddb_core::types::{ AttributeDefinition, AttributeValue, BillingMode, CreateTableInput, DeleteTableInput, DescribeTableInput, GsiInput, Item, KeySchemaElement, KeyType, LsiInput, Projection, ProjectionType, ScalarAttributeType, TableStatus, }; +use extenddb_storage::backup_catalog::{ + BackupRow, BackupStatus, DeleteStart, Heartbeat, JobStatus, RestoreKind, RestoreTargetKey, + StatusFields, StoreIdentity, StoreKind, TimeBounds, UNOWNED_GENERATION, WriteOutcome, +}; use extenddb_storage::error::StorageError; -use extenddb_storage::{BackupEngine, RestoreRequest, RestoreTableOverrides, TableEngine}; -use futures::FutureExt; +use extenddb_storage::{ + BackupCatalog, JobPermit, LegacyBackupReader, RestoreRequest, RestoreTableOverrides, + TableEngine, +}; +use futures::{FutureExt, TryStreamExt}; use mongodb::bson::{self, Document, doc}; -use super::{ - AfterBackupDrop, BackupLiveness, DELETE_CLAIM, DELETE_CLAIM_HEARTBEAT, RESTORE_CLAIM, - RESTORE_CLAIM_HEARTBEAT, RestoreProvenance, TARGET_HEARTBEAT, claim_update, - restore_claim_filter, -}; +use super::{AfterBackupDrop, BackupLiveness, HeartbeatFault}; use crate::MongoEngine; +use crate::backup_catalog::{BACKUP_INDEX, RESTORE_JOB, RESTORE_JOB_HEARTBEAT, RESTORE_JOB_STATUS}; +use crate::bootstrapper::{CATALOG_VERSION, apply_backup_catalog_index, require_served_catalog}; use crate::data::{data_collection_name, item_to_document}; const ACCOUNT: &str = "123456789012"; +const OTHER_ACCOUNT: &str = "210987654321"; +const REGION: &str = "us-east-1"; const ITEMS: usize = 3_000; +/// The stale window the reconciler would pass (`stale_job_after_secs`). +const STALE_AFTER: Duration = Duration::from_secs(300); fn uri() -> String { std::env::var("EXTENDDB_TEST_MONGODB_URI").expect( @@ -51,11 +64,12 @@ fn uri() -> String { fn test_liveness() -> BackupLiveness { BackupLiveness { heartbeat_interval: Duration::from_millis(200), - stale_after: Duration::from_millis(1_500), - slot_wait: Duration::from_secs(2), + registration_gate: None, copy_gate: None, after_backup_drop: None, + target_drop_fails: false, local_clock_behind: Duration::ZERO, + heartbeat_fault: None, } } @@ -78,6 +92,13 @@ impl Fixture { .insert_one(doc! { "_id": "control_plane_delay_seconds", "value": "0" }) .await .expect("settings"); + // The catalog index step alone (`002_backup_catalog_index`, as + // `init` and `migrate` run it): the collections and indexes of the + // initial step are not created here, so this catalog is not `init`'s; + // the tests of this file need only what the step creates. + apply_backup_catalog_index(&engine.catalog_db) + .await + .expect("catalog index step"); let worker = spawn_worker(&engine); Self { engine, @@ -102,20 +123,14 @@ impl Fixture { async fn engine_on(catalog: &str, data: &str, liveness: BackupLiveness) -> MongoEngine { let client = mongodb::Client::with_uri_str(uri()).await.expect("client"); - // The driver default pool (10) and the slot count production derives - // from it, so the tests see the same admission bound as `serve`. - let slots = super::restore_server_slot_count(10); MongoEngine { client: client.clone(), catalog_db: client.database(catalog), data_db: client.database(data), - region: "us-east-1".to_owned(), + region: REGION.to_owned(), gsi_cache: Arc::default(), tx_read_concern: mongodb::options::ReadConcern::snapshot(), backup_liveness: liveness, - restore_server_slot_count: slots, - restore_server_slots: Arc::new(tokio::sync::Semaphore::new(slots)), - restore_account_slots: super::RestoreAccountSlots::default(), } } @@ -226,7 +241,7 @@ async fn restore_request_for( target: &str, ) -> RestoreRequest { let source = engine - .backup_restore_source(account, backup_arn) + .restore_source(account, backup_arn) .await .expect("restore source"); source.ensure_restorable().expect("restorable"); @@ -239,13 +254,26 @@ async fn restore_request_for( } } +/// The `tables` collection read at `majority`: the state a registration's or +/// a DeleteBackup's transaction will see. The engine's transactions read at +/// its `tx_read_concern` (`snapshot` here), which is majority-committed data; +/// a read at the collection's default (`local`) sees the backfill worker's +/// ACTIVE flip as soon as it is applied on the primary, while the majority +/// commit point still waits for the journal and for any concurrent batch +/// write holding an earlier oplog slot (another test's 3,000-item seed), so +/// a test that observed ACTIVE at `local` and then issued a request could be +/// answered `BackupInUse` by a transaction that still saw CREATING. +fn tables_majority(engine: &MongoEngine) -> mongodb::Collection { + engine.catalog_db.collection_with_options( + "tables", + mongodb::options::CollectionOptions::builder() + .read_concern(mongodb::options::ReadConcern::majority()) + .build(), + ) +} + async fn table_doc(engine: &MongoEngine, name: &str) -> Option { - engine - .catalog_db - .collection::("tables") - .find_one(doc! { "_id": { "account_id": ACCOUNT, "table_name": name } }) - .await - .expect("read table") + table_doc_for(engine, ACCOUNT, name).await } async fn wait_for_status(engine: &MongoEngine, name: &str, status: &str) -> Document { @@ -339,10 +367,15 @@ async fn assert_fully_restored_with(engine: &MongoEngine, name: &str, items: usi "index {index_name}" ); } - assert!(!doc.contains_key(TARGET_HEARTBEAT)); - assert!(!doc.contains_key("restore_operation_id")); + // The job record went DONE with the hand-over and stayed with the table. + let job = doc.get_document(RESTORE_JOB).expect("job record"); + assert_eq!(job.get_str("status"), Ok("DONE")); + assert_eq!(job.get_str("kind"), Ok("legacy")); + assert!(job.get_datetime("completed_at").is_ok()); + assert!(!doc.contains_key("restore_backfill_pending")); } +/// An engine whose restores wait for one permit each before their copy. fn gated(engine: &MongoEngine) -> (MongoEngine, Arc) { let gate = Arc::new(tokio::sync::Semaphore::new(0)); let mut gated = engine.clone(); @@ -350,628 +383,1463 @@ fn gated(engine: &MongoEngine) -> (MongoEngine, Arc) { (gated, gate) } -/// (a) The response is CREATING while the copy has not run; the target then -/// reaches ACTIVE with every item and index entry; the summary carries the -/// source table ARN and the backup's creation time. -#[tokio::test(flavor = "multi_thread")] -#[ignore = "needs a MongoDB replica set in EXTENDDB_TEST_MONGODB_URI"] -async fn live_restore_returns_creating_and_the_detached_copy_finishes() { - let fx = Fixture::new().await; - let source_arn = source_table(&fx.engine, "src").await; - let backup = fx - .engine - .create_backup(ACCOUNT, "src", "b") - .await - .expect("backup"); - let request = restore_request(&fx.engine, &backup.backup_arn, "dst").await; - let (gated, gate) = gated(&fx.engine); - - let desc = tokio::time::timeout( - Duration::from_secs(20), - gated.restore_table_from_backup(ACCOUNT, request), - ) - .await - .expect("the restore must return while its copy is held") - .expect("restore"); - assert_eq!(desc.table_status, TableStatus::Creating); - let doc = table_doc(&fx.engine, "dst").await.expect("target"); - assert_eq!(doc.get_str("table_status"), Ok("CREATING")); - assert_eq!( - count(&fx.engine, &data_collection_name(&desc.table_id)).await, - 0, - "no item may be copied before the response" - ); - let summary = fx - .engine - .describe_table( - ACCOUNT, - DescribeTableInput { - table_name: "dst".to_owned(), - }, - ) - .await - .expect("describe") - .restore_summary - .expect("restore summary while CREATING"); - assert_eq!( - summary.source_backup_arn.as_deref(), - Some(backup.backup_arn.as_str()) - ); - assert_eq!( - summary.source_table_arn.as_deref(), - Some(source_arn.as_str()) - ); - assert_eq!(summary.restore_date_time, backup.backup_creation_date_time); - assert!(summary.restore_in_progress); - // The backup time is the shared id's instant, kept to the millisecond. - assert_eq!( - Some(backup.backup_creation_date_time), - extenddb_storage::util::backup_creation_date_time(&backup.backup_arn) - ); +/// An engine whose restores wait for one permit each before they register +/// their target, and one more before their copy. +fn gated_at_registration( + engine: &MongoEngine, +) -> ( + MongoEngine, + Arc, + Arc, +) { + let (mut gated, copy_gate) = gated(engine); + let registration_gate = Arc::new(tokio::sync::Semaphore::new(0)); + gated.backup_liveness.registration_gate = Some(registration_gate.clone()); + (gated, registration_gate, copy_gate) +} - gate.add_permits(1); - assert_fully_restored(&fx.engine, "dst").await; - fx.finish().await; +/// The names among `targets` that exist in the catalog: the catalog's account +/// of which restores registered, whatever their requests answered. +async fn registered_targets(engine: &MongoEngine, targets: &[String]) -> Vec { + let mut registered = Vec::new(); + for target in targets { + if table_doc(engine, target).await.is_some() { + registered.push(target.clone()); + } + } + registered } -/// (b) A request future dropped after its first poll still produces a -/// complete restore, whether the poll parked in the registration or in the -/// wait for an account slot. -#[tokio::test(flavor = "multi_thread")] -#[ignore = "needs a MongoDB replica set in EXTENDDB_TEST_MONGODB_URI"] -async fn live_a_dropped_restore_request_still_reaches_active() { - let fx = Fixture::new().await; - source_table(&fx.engine, "src").await; - let backup = fx - .engine - .create_backup(ACCOUNT, "src", "b") +/// The restore job record of `name`'s table document. +async fn job_record(engine: &MongoEngine, name: &str) -> Document { + table_doc(engine, name) .await - .expect("backup"); - let request = restore_request(&fx.engine, &backup.backup_arn, "dst").await; - let mut future = fx.engine.restore_table_from_backup(ACCOUNT, request); - assert!( - (&mut future).now_or_never().is_none(), - "the first poll must not complete the restore" - ); - drop(future); - assert_fully_restored(&fx.engine, "dst").await; + .expect("target") + .get_document(RESTORE_JOB) + .expect("job record") + .clone() +} - // Every account slot is held, so the request's first poll parks in the - // admission wait; the slots are freed only after the future is gone. - let account_slots = fx.engine.restore_account_slots.for_account(ACCOUNT); - let held = Arc::clone(&account_slots) - .acquire_many_owned(u32::try_from(account_slots.available_permits()).expect("u32")) - .await - .expect("hold every account slot"); - let backup = fx - .engine - .create_backup(ACCOUNT, "src", "b2") - .await - .expect("backup"); - let request = restore_request(&fx.engine, &backup.backup_arn, "dst2").await; - let mut future = fx.engine.restore_table_from_backup(ACCOUNT, request); - assert!( - (&mut future).now_or_never().is_none(), - "the first poll must not complete the restore" - ); - drop(future); - assert!( - table_doc(&fx.engine, "dst2").await.is_none(), - "nothing is created while the restore waits for a slot" - ); - drop(held); - assert_fully_restored(&fx.engine, "dst2").await; - fx.finish().await; +fn target_key(doc: &Document) -> RestoreTargetKey { + let id = doc.get_document("_id").expect("_id"); + RestoreTargetKey { + account_id: id.get_str("account_id").expect("account").to_owned(), + table_name: id.get_str("table_name").expect("name").to_owned(), + table_id: doc.get_str("table_id").expect("table_id").to_owned(), + } } -/// Leave the state a process killed mid-restore leaves: the backup's restore -/// claim and a CREATING target with provenance, both with a heartbeat far in -/// the past, and no task. -async fn simulate_crashed_restore(engine: &MongoEngine, backup_arn: &str, target: &str) -> String { - let dead = "dead-operation"; - let backups = engine.catalog_db.collection::("backups"); - let claimed = backups - .find_one_and_update( - restore_claim_filter(ACCOUNT, backup_arn), - claim_update(RESTORE_CLAIM, RESTORE_CLAIM_HEARTBEAT, dead), - ) - .await - .expect("claim") - .expect("claimable"); - let request = restore_request(engine, backup_arn, target).await; - let desc = engine - .create_table_for_restore( - ACCOUNT, - request.input, - &RestoreProvenance { - backup_arn, - source_table_arn: &request.source_table_arn, - restore_date_time: bson::DateTime::now(), - operation_id: dead, - first_heartbeat: bson::DateTime::now(), - inserted: None, - }, - ) - .await - .expect("target"); - drop(claimed); - let long_ago = bson::DateTime::from_millis(bson::DateTime::now().timestamp_millis() - 600_000); - let mut set = Document::new(); - set.insert(RESTORE_CLAIM_HEARTBEAT, long_ago); - backups - .update_one(doc! { "_id": backup_arn }, doc! { "$set": set }) - .await - .expect("age claim"); - let mut set = Document::new(); - set.insert(TARGET_HEARTBEAT, long_ago); +/// Leave the state a process killed mid-copy leaves: the target CREATING, its +/// job record RUNNING with a heartbeat `age` old, and no task. +async fn age_running_record(engine: &MongoEngine, name: &str, age: Duration) { + let heartbeat = bson::DateTime::from_millis( + bson::DateTime::now().timestamp_millis() - i64::try_from(age.as_millis()).expect("i64"), + ); engine .catalog_db .collection::("tables") .update_one( - doc! { "_id": { "account_id": ACCOUNT, "table_name": target } }, - doc! { "$set": set }, + doc! { "_id": { "account_id": ACCOUNT, "table_name": name } }, + doc! { "$set": { + "table_status": "CREATING", + "status_transition_at": bson::Bson::Null, + RESTORE_JOB_STATUS: "RUNNING", + RESTORE_JOB_HEARTBEAT: heartbeat, + }, "$unset": { "restore_job.completed_at": "", "restore_backfill_pending": "" } }, ) .await - .expect("age target"); - desc.table_id + .expect("age the record"); +} + +async fn backup_meta(engine: &MongoEngine, backup_arn: &str) -> Document { + engine + .catalog_db + .collection::("backups") + .find_one(doc! { "_id": backup_arn }) + .await + .expect("read backup") + .expect("backup document") +} + +fn with_drop_fault(engine: &MongoEngine, fault: AfterBackupDrop) -> MongoEngine { + let mut faulty = engine.clone(); + faulty.backup_liveness.after_backup_drop = Some(fault); + faulty +} + +fn store() -> StoreIdentity { + StoreIdentity { + store_kind: StoreKind::Filesystem, + store_root: "/var/lib/extenddb/backups".to_owned(), + store_id: "store-1".to_owned(), + } +} + +/// A `backup_index` row for a backup of `table` in `account` minted now, +/// with the manifest fixture's source table. +fn row(account: &str, table: &str, name: &str) -> BackupRow { + let backup_id = extenddb_storage::util::new_backup_id(); + let backup_arn = extenddb_storage::util::backup_arn(REGION, account, table, &backup_id); + let manifest: serde_json::Value = serde_json::from_str(include_str!( + "../../core/testdata/extenddb-manifest/v1.json" + )) + .expect("fixture"); + BackupRow { + backup_id: backup_id.clone(), + account_id: account.to_owned(), + table_name: table.to_owned(), + backup_arn, + backup_name: name.to_owned(), + status: BackupStatus::Creating, + store: store(), + size_bytes: 0, + item_count: 0, + created_at: extenddb_storage::util::backup_creation_date_time(&backup_id).unwrap(), + completed_at: None, + failure_reason: None, + source_table_details_json: manifest["source_table"].clone(), + owner_generation: 0, + heartbeat_at: 0.0, + } +} + +fn arn_of(row: &BackupRow) -> BackupArn { + BackupArn::parse(&row.backup_arn).unwrap() } -/// (c) A crashed restore is swept completely; DeleteBackup of its source then -/// succeeds and the target name can be restored into. +/// The claim: a second claim of the same backup is refused while the row +/// exists, the heartbeat confirms the claim and refreshes the row with the +/// server's clock, a status write is conditioned on the generation and the +/// `from` status, and the row is read back by account, backup id, table and +/// ARN. The catalog has an id and no maintenance claim until one is written. #[tokio::test(flavor = "multi_thread")] #[ignore = "needs a MongoDB replica set in EXTENDDB_TEST_MONGODB_URI"] -async fn live_an_abandoned_restore_is_swept_and_frees_the_backup_and_the_name() { +async fn live_claim_backup_excludes_a_second_owner_and_fences_its_writes() { let fx = Fixture::new().await; - source_table(&fx.engine, "src").await; - let crashed = fx + assert!( + uuid::Uuid::parse_str(&fx.engine.catalog_id().await.expect("id")).is_ok(), + "the catalog has an id from the migration step" + ); + assert!(!fx.engine.check_maintenance_claim().await.expect("check")); + let row = row(ACCOUNT, "Music", "nightly"); + let claim = fx.engine.claim_backup(&row).await.expect("claim"); + assert_ne!(claim.generation(), UNOWNED_GENERATION); + let second = fx.engine.claim_backup(&row).await.err(); + assert!( + matches!(second, Some(StorageError::AlreadyExists(_))), + "{second:?}" + ); + let stored = fx .engine - .create_backup(ACCOUNT, "src", "b1") + .read_backup_row(&arn_of(&row)) .await - .expect("backup 1"); - let spare = fx + .expect("read") + .expect("the row exists"); + assert_eq!(stored.status, BackupStatus::Creating); + assert_eq!(stored.owner_generation, claim.generation()); + assert_eq!(stored.store, store()); + assert_eq!(stored.created_at, row.created_at); + assert_eq!( + stored.source_table_details_json, + row.source_table_details_json + ); + assert!( + stored.heartbeat_at > 0.0, + "the heartbeat is the server's now" + ); + let before = stored.heartbeat_at; + tokio::time::sleep(Duration::from_millis(20)).await; + assert_eq!( + claim.heartbeat().await.expect("heartbeat"), + Heartbeat::Confirmed + ); + let after = fx .engine - .create_backup(ACCOUNT, "src", "b2") + .read_backup_row(&arn_of(&row)) .await - .expect("backup 2"); - // The periodic sweep in the fixture's worker would race this test's own - // sweep; stop it so the assertions see exactly one sweep's result. - fx.worker.abort(); - let table_id = simulate_crashed_restore(&fx.engine, &crashed.backup_arn, "dst").await; - let indexes = index_ids(&fx.engine, &table_id).await; - assert_eq!(indexes.len(), 2); - assert!(collection_exists(&fx.engine, &data_collection_name(&table_id)).await); + .unwrap() + .unwrap() + .heartbeat_at; + assert!(after > before, "{after} > {before}"); - let before = fx.engine.delete_backup(ACCOUNT, &crashed.backup_arn).await; + // The same id under another account, table or region finds nothing. + let other_account = + BackupArn::new("aws", REGION, OTHER_ACCOUNT, "Music", &row.backup_id).unwrap(); + assert!( + fx.engine + .read_backup_row(&other_account) + .await + .unwrap() + .is_none() + ); + let other_table = BackupArn::new("aws", REGION, ACCOUNT, "Other", &row.backup_id).unwrap(); + assert!( + fx.engine + .read_backup_row(&other_table) + .await + .unwrap() + .is_none() + ); + let other_region = + BackupArn::new("aws", "eu-west-1", ACCOUNT, "Music", &row.backup_id).unwrap(); assert!( - matches!(before, Err(StorageError::BackupInUse(_))), - "the crashed restore blocks DeleteBackup before the sweep: {before:?}" + fx.engine + .read_backup_row(&other_region) + .await + .unwrap() + .is_none() ); - let sweep = fx.engine.sweep_stale_backup_claims().await.expect("sweep"); + // A write from the wrong state matches nothing; the right one matches + // once, and the row then carries what it set. + let fields = StatusFields { + size_bytes: Some(1234), + item_count: Some(5), + failure_reason: None, + completed: true, + }; assert_eq!( - sweep.restore_claims_cleared, - std::slice::from_ref(&crashed.backup_arn) + claim + .set_status(JobStatus::Running, JobStatus::Available, &fields) + .await + .unwrap(), + WriteOutcome::NotMatched ); assert_eq!( - sweep.restore_targets_removed, - [format!("{ACCOUNT}/dst")], - "the abandoned target must be removed" + claim + .set_status(JobStatus::Creating, JobStatus::Available, &fields) + .await + .unwrap(), + WriteOutcome::Matched ); - assert!(table_doc(&fx.engine, "dst").await.is_none()); - assert!(!collection_exists(&fx.engine, &data_collection_name(&table_id)).await); - for (_, index_id, _) in &indexes { - assert!(!collection_exists(&fx.engine, &data_collection_name(index_id)).await); - } - assert!(index_ids(&fx.engine, &table_id).await.is_empty()); - let meta = fx + assert_eq!( + claim + .set_status(JobStatus::Creating, JobStatus::Available, &fields) + .await + .unwrap(), + WriteOutcome::NotMatched + ); + let read = claim.read().await.expect("read"); + assert_eq!(read.status, JobStatus::Available); + assert_eq!(read.owner_generation, claim.generation()); + let stored = fx .engine - .catalog_db - .collection::("backups") - .find_one(doc! { "_id": &crashed.backup_arn }) + .read_backup_row(&arn_of(&row)) .await - .expect("read") - .expect("backup"); - assert!(!meta.contains_key(RESTORE_CLAIM) && !meta.contains_key(RESTORE_CLAIM_HEARTBEAT)); + .unwrap() + .unwrap(); + assert_eq!(stored.status, BackupStatus::Available); + assert_eq!(stored.size_bytes, 1234); + assert_eq!(stored.item_count, 5); + assert!(stored.completed_at.is_some()); - let deleted = fx - .engine - .delete_backup(ACCOUNT, &crashed.backup_arn) + // A row another generation took over confirms nothing for this claim. + fx.engine + .catalog_db + .collection::(BACKUP_INDEX) + .update_one( + doc! { "backup_id": &row.backup_id }, + doc! { "$inc": { "owner_generation": 1_i64 } }, + ) .await - .expect("DeleteBackup after the sweep"); - assert_eq!(deleted.backup_details.backup_status, "DELETED"); + .expect("take over"); + assert_eq!(claim.heartbeat().await.expect("heartbeat"), Heartbeat::Lost); + assert_eq!( + claim + .set_status( + JobStatus::Available, + JobStatus::Failed, + &StatusFields::default() + ) + .await + .unwrap(), + WriteOutcome::NotMatched + ); + drop(claim); + // The row is the claim: a successor can take it only through its + // insert, which the existing row refuses. + let third = fx.engine.claim_backup(&row).await.err(); + assert!( + matches!(third, Some(StorageError::AlreadyExists(_))), + "{third:?}" + ); - let worker = spawn_worker(&fx.engine); - let request = restore_request(&fx.engine, &spare.backup_arn, "dst").await; + // A maintenance record, once written, is seen by every job. fx.engine - .restore_table_from_backup(ACCOUNT, request) + .catalog_db + .collection::(crate::backup_catalog::BACKUP_MAINTENANCE) + .insert_one(doc! { "holder": "backup purge", "owner_generation": 1_i64 }) .await - .expect("restore into the freed name"); - assert_fully_restored(&fx.engine, "dst").await; - worker.abort(); + .expect("maintenance record"); + assert!(fx.engine.check_maintenance_claim().await.expect("check")); fx.finish().await; } -/// (d) Claims and restore targets of another, live, instance on the same -/// catalog survive this instance's sweep for many stale windows, because their -/// heartbeats keep moving; once the owner stops, the sweep removes them. +/// DeleteBackup's first step and the tombstone writes: only an AVAILABLE +/// row moves to DELETING; a CREATING row answers its status; a backup a +/// CREATING restore target reads answers the target's name; DELETING moves +/// to DELETED once; a DELETED row is removed once; a missing row is "Backup +/// not found". #[tokio::test(flavor = "multi_thread")] #[ignore = "needs a MongoDB replica set in EXTENDDB_TEST_MONGODB_URI"] -async fn live_a_live_claim_of_another_instance_survives_this_instances_sweep() { +async fn live_start_backup_delete_checks_the_status_and_the_in_use_rule_under_the_lock() { let fx = Fixture::new().await; - fx.worker.abort(); - source_table(&fx.engine, "src").await; - let b1 = fx - .engine - .create_backup(ACCOUNT, "src", "b1") - .await - .expect("backup 1"); - let b2 = fx - .engine - .create_backup(ACCOUNT, "src", "b2") - .await - .expect("backup 2"); - let other = fx.second_instance(test_liveness()).await; - let stale_after = fx.engine.backup_liveness.stale_after; - assert!(other.backup_liveness.heartbeat_interval * 3 < stale_after); - - // A live delete claim on b1, held the way delete_backup holds it. - let op = "live-delete"; - let heartbeat = other.start_backup_heartbeat(op, ACCOUNT, &b1.backup_arn, None); - other - .catalog_db - .collection::("backups") + let row = row(ACCOUNT, "Music", "nightly"); + let key = row.key(); + assert!(matches!( + fx.engine.start_backup_delete(&key).await, + Err(StorageError::Validation(m)) if m.contains("Backup not found") + )); + let claim = fx.engine.claim_backup(&row).await.expect("claim"); + assert_eq!( + fx.engine.start_backup_delete(&key).await.unwrap(), + DeleteStart::Status(BackupStatus::Creating) + ); + assert_eq!( + claim + .set_status( + JobStatus::Creating, + JobStatus::Available, + &StatusFields::default() + ) + .await + .unwrap(), + WriteOutcome::Matched + ); + assert_eq!(fx.engine.backup_in_use(&key).await.unwrap(), None); + + // A CREATING restore target registered from the backup: in use, naming + // the target; an ACTIVE one is not. + source_table_with(&fx.engine, "del_target", 1).await; + let tables = fx.engine.catalog_db.collection::("tables"); + let target_filter = doc! { "_id": { "account_id": ACCOUNT, "table_name": "del_target" } }; + tables .update_one( - doc! { "_id": &b1.backup_arn }, - claim_update(DELETE_CLAIM, DELETE_CLAIM_HEARTBEAT, op), + target_filter.clone(), + doc! { "$set": { + "table_status": "CREATING", + "status_transition_at": bson::Bson::Null, + "restore_source_backup_arn": &row.backup_arn, + "restore_source_table_arn": "arn:aws:dynamodb:us-east-1:123456789012:table/Music", + "restore_date_time": bson::DateTime::now(), + RESTORE_JOB: { + "status": "RUNNING", + "kind": "manifest", + "owner_generation": 7_i64, + "heartbeat_at": bson::DateTime::now(), + }, + } }, ) .await - .expect("claim"); - // A live restore target restoring b2, held before its copy. - let (gated_other, gate) = gated(&other); - let request = restore_request(&other, &b2.backup_arn, "dst").await; - gated_other - .restore_table_from_backup(ACCOUNT, request) - .await - .expect("restore"); - - let backups = fx.engine.catalog_db.collection::("backups"); - let claim_heartbeat = || async { - backups - .find_one(doc! { "_id": &b1.backup_arn }) - .await - .expect("read") - .and_then(|d| d.get_datetime(DELETE_CLAIM_HEARTBEAT).ok().copied()) - }; - let target_heartbeat = || async { - table_doc(&fx.engine, "dst") + .expect("register a target"); + assert_eq!( + fx.engine.backup_in_use(&key).await.unwrap().as_deref(), + Some("del_target") + ); + assert_eq!( + fx.engine.start_backup_delete(&key).await.unwrap(), + DeleteStart::InUse("del_target".to_owned()) + ); + assert_eq!( + fx.engine + .read_backup_row(&arn_of(&row)) .await - .and_then(|d| d.get_datetime(TARGET_HEARTBEAT).ok().copied()) - }; - let first_claim = claim_heartbeat().await.expect("claim heartbeat"); - let first_target = target_heartbeat().await.expect("target heartbeat"); - - // Sweep repeatedly across three stale windows. - let until = Instant::now() + stale_after * 3; - let mut sweeps = 0; - while Instant::now() < until { - let sweep = fx.engine.sweep_stale_backup_claims().await.expect("sweep"); - assert!(sweep.is_empty(), "a live owner's state was swept: {sweep}"); - sweeps += 1; - tokio::time::sleep(Duration::from_millis(250)).await; - } - assert!(sweeps >= 10); - let meta = backups - .find_one(doc! { "_id": &b1.backup_arn }) + .unwrap() + .unwrap() + .status, + BackupStatus::Available, + "a refused delete leaves the row AVAILABLE" + ); + tables + .update_one(target_filter, doc! { "$set": { "table_status": "ACTIVE" } }) .await - .expect("read") - .expect("backup"); + .expect("make it ACTIVE"); + assert_eq!(fx.engine.backup_in_use(&key).await.unwrap(), None); + assert_eq!( - meta.get_str(DELETE_CLAIM), - Ok(op), - "the live claim must still be held" + fx.engine.start_backup_delete(&key).await.unwrap(), + DeleteStart::Deleting ); - assert!( - table_doc(&fx.engine, "dst").await.is_some(), - "the live target must remain" + assert_eq!( + fx.engine.start_backup_delete(&key).await.unwrap(), + DeleteStart::Status(BackupStatus::Deleting), + "a retry finds the row DELETING" ); - // Liveness: both heartbeats moved. - assert!(claim_heartbeat().await.expect("claim heartbeat") > first_claim); - assert!(target_heartbeat().await.expect("target heartbeat") > first_target); - let in_use = fx.engine.delete_backup(ACCOUNT, &b2.backup_arn).await; assert!( - matches!(in_use, Err(StorageError::BackupInUse(_))), - "{in_use:?}" + !fx.engine.remove_backup_row(&key).await.unwrap(), + "DELETING is not a tombstone" ); - - // The owner of the delete claim stops: the sweep takes the claim over and - // finishes the delete once the heartbeat is stale. - drop(heartbeat); - tokio::time::sleep(stale_after + Duration::from_millis(500)).await; - let sweep = fx.engine.sweep_stale_backup_claims().await.expect("sweep"); - assert_eq!( - sweep.delete_claims_finished, - std::slice::from_ref(&b1.backup_arn) + assert!(fx.engine.set_backup_deleted(&key).await.unwrap()); + assert!(!fx.engine.set_backup_deleted(&key).await.unwrap()); + let stored = fx + .engine + .read_backup_row(&arn_of(&row)) + .await + .unwrap() + .unwrap(); + assert_eq!(stored.status, BackupStatus::Deleted); + assert!(stored.completed_at.is_some()); + assert!(fx.engine.remove_backup_row(&key).await.unwrap()); + assert!(!fx.engine.remove_backup_row(&key).await.unwrap()); + assert!( + fx.engine + .read_backup_row(&arn_of(&row)) + .await + .unwrap() + .is_none() ); - assert!(sweep.restore_targets_removed.is_empty(), "{sweep}"); - - // The restore finishes normally. - let worker = spawn_worker(&fx.engine); - gate.add_permits(1); - assert_fully_restored(&fx.engine, "dst").await; - worker.abort(); + drop(claim); fx.finish().await; } -/// (e) While the copy runs, DeleteBackup of the source is BackupInUse and -/// DeleteTable of the target is refused; both succeed once it is ACTIVE. +/// The listings: by account with an optional table and the time bounds +/// (lower inclusive, upper exclusive) in (created_at, ARN) order, and by +/// store and status. #[tokio::test(flavor = "multi_thread")] #[ignore = "needs a MongoDB replica set in EXTENDDB_TEST_MONGODB_URI"] -async fn live_backup_and_target_are_protected_while_the_copy_runs() { +async fn live_the_listings_are_scoped_and_ordered() { let fx = Fixture::new().await; - source_table(&fx.engine, "src").await; - let backup = fx + let mut rows = Vec::new(); + for (account, table, name) in [ + (ACCOUNT, "Music", "a"), + (ACCOUNT, "Music", "b"), + (ACCOUNT, "Other", "c"), + (OTHER_ACCOUNT, "Music", "d"), + ] { + tokio::time::sleep(Duration::from_millis(5)).await; + let row = row(account, table, name); + let claim = fx.engine.claim_backup(&row).await.expect("claim"); + if name == "b" { + assert_eq!( + claim + .set_status( + JobStatus::Creating, + JobStatus::Failed, + &StatusFields::default() + ) + .await + .unwrap(), + WriteOutcome::Matched + ); + } + rows.push((row, claim)); + } + let names = |listed: Vec| -> Vec { + listed.into_iter().map(|r| r.backup_name).collect() + }; + let all: Vec = fx .engine - .create_backup(ACCOUNT, "src", "b") + .list_account_backup_rows(ACCOUNT, None, &TimeBounds::default()) + .try_collect() + .await + .unwrap(); + assert_eq!( + names(all.clone()), + ["a", "b", "c"], + "every status, in creation order" + ); + assert_eq!(all[1].status, BackupStatus::Failed); + let music: Vec = fx + .engine + .list_account_backup_rows(ACCOUNT, Some("Music"), &TimeBounds::default()) + .try_collect() + .await + .unwrap(); + assert_eq!(names(music), ["a", "b"]); + let bounds = TimeBounds { + lower: Some(rows[1].0.created_at), + upper: Some(rows[2].0.created_at), + }; + let bounded: Vec = fx + .engine + .list_account_backup_rows(ACCOUNT, None, &bounds) + .try_collect() + .await + .unwrap(); + assert_eq!(names(bounded), ["b"], "lower inclusive, upper exclusive"); + let other: Vec = fx + .engine + .list_account_backup_rows(OTHER_ACCOUNT, None, &TimeBounds::default()) + .try_collect() + .await + .unwrap(); + assert_eq!(names(other), ["d"]); + + let by_store: Vec = fx + .engine + .list_backup_rows(None, &store()) + .try_collect() + .await + .unwrap(); + assert_eq!(names(by_store), ["a", "b", "c", "d"]); + let failed: Vec = fx + .engine + .list_backup_rows(Some(BackupStatus::Failed), &store()) + .try_collect() + .await + .unwrap(); + assert_eq!(names(failed), ["b"]); + let elsewhere = StoreIdentity { + store_id: "store-2".to_owned(), + ..store() + }; + let none: Vec = fx + .engine + .list_backup_rows(None, &elsewhere) + .try_collect() + .await + .unwrap(); + assert!(none.is_empty(), "another store's rows are not listed"); + drop(rows); + fx.finish().await; +} + +/// A catalog at the shape before the catalog index step, holding a finished +/// restore, a restore handed to the GSI backfill worker at the stop (its +/// copy complete, the target CREATING with `restore_backfill_pending`) and a +/// restore the stop interrupted (its target CREATING, no flag), migrates with +/// the registrations backfilled: the finished and the handed-over ones DONE, +/// the interrupted one RUNNING with `heartbeat_at` at the epoch and the +/// unowned generation, so the reconciler's first pass fails it and removes +/// its target and leaves the handed-over one to the worker. The step writes +/// the version and keeps the catalog id; a second run changes nothing. +#[tokio::test(flavor = "multi_thread")] +#[ignore = "needs a MongoDB replica set in EXTENDDB_TEST_MONGODB_URI"] +async fn live_the_migration_step_backfills_the_registrations_and_the_reconciler_removes_the_interrupted_one() + { + const FEW: usize = 300; + let fx = Fixture::new().await; + source_table_with(&fx.engine, "src", FEW).await; + let backup = fx + .engine + .create(ACCOUNT, "src", "b", JobPermit::detached()) .await .expect("backup"); - let request = restore_request(&fx.engine, &backup.backup_arn, "dst").await; - let (gated, gate) = gated(&fx.engine); - gated - .restore_table_from_backup(ACCOUNT, request) + let request = restore_request(&fx.engine, &backup.backup_arn, "bf_done").await; + fx.engine + .restore(ACCOUNT, request, JobPermit::detached()) + .await + .expect("restore"); + assert_fully_restored_with(&fx.engine, "bf_done", FEW).await; + // A restore the earlier release had handed to the GSI backfill worker at + // the stop: the copy complete, the table CREATING with + // `restore_backfill_pending`, one index still CREATING (so the worker + // waits rather than flipping it while the step runs). Its data is whole, + // so the step backfills it DONE, not RUNNING: marked RUNNING with the + // epoch heartbeat, the first reconciler pass would fail it and remove a + // target the worker is about to finish. From a backup of its own: a + // CREATING target keeps its backup in use, which would refuse the + // interrupted restore below. + let handed_backup = fx + .engine + .create(ACCOUNT, "src", "b2", JobPermit::detached()) + .await + .expect("second backup"); + let request = restore_request(&fx.engine, &handed_backup.backup_arn, "bf_handed").await; + let handed = fx + .engine + .restore(ACCOUNT, request, JobPermit::detached()) + .await + .expect("restore"); + assert_fully_restored_with(&fx.engine, "bf_handed", FEW).await; + let tables = fx.engine.catalog_db.collection::("tables"); + let index_docs = fx.engine.catalog_db.collection::("indexes"); + index_docs + .update_one( + doc! { "_id.table_id": &handed.table_id, "_id.index_name": "g1" }, + doc! { "$set": { "index_status": "CREATING" } }, + ) + .await + .expect("an index still being backfilled"); + tables + .update_one( + doc! { "_id": { "account_id": ACCOUNT, "table_name": "bf_handed" } }, + doc! { "$set": { + "table_status": "CREATING", + "status_transition_at": bson::Bson::Null, + "restore_backfill_pending": true, + } }, + ) + .await + .expect("handed over to the worker"); + let (gated, _gate) = gated(&fx.engine); + let request = restore_request(&fx.engine, &backup.backup_arn, "bf_cut").await; + let interrupted = gated + .restore(ACCOUNT, request, JobPermit::detached()) .await .expect("restore"); + let catalog_id = fx.engine.catalog_id().await.expect("id"); + + // Back to the shape before the step: the registrations as the earlier + // release wrote them, the step forgotten, the version before it. The + // interrupted restore's copy task loses its claim with the record and + // stops. + tables + .update_many( + doc! { "restore_source_backup_arn": { "$exists": true } }, + doc! { + "$unset": { RESTORE_JOB: "" }, + "$set": { + "restore_operation_id": "old-operation", + "restore_heartbeat_at": bson::DateTime::now(), + }, + }, + ) + .await + .expect("old shape"); + let settings = fx.engine.catalog_db.collection::("settings"); + settings + .update_one( + doc! { "_id": "catalog_version" }, + doc! { "$set": { "value": "0.0.2" } }, + ) + .await + .expect("old version"); + fx.engine + .catalog_db + .collection::("schema_history") + .delete_one(doc! { "_id": "002_backup_catalog_index" }) + .await + .expect("forget the step"); - let in_use = fx.engine.delete_backup(ACCOUNT, &backup.backup_arn).await; + let report = apply_backup_catalog_index(&fx.engine.catalog_db) + .await + .expect("migrate again"); + assert_eq!(report.registrations_backfilled, 3); + assert!(!report.catalog_id_written, "the catalog id is kept"); + assert_eq!(fx.engine.catalog_id().await.expect("id"), catalog_id); + assert_eq!( + settings + .find_one(doc! { "_id": "catalog_version" }) + .await + .expect("read") + .and_then(|d| d.get_str("value").ok().map(str::to_owned)) + .as_deref(), + Some(CATALOG_VERSION) + ); assert!( - matches!(in_use, Err(StorageError::BackupInUse(_))), - "the source backup is in use while the copy runs: {in_use:?}" + fx.engine + .catalog_db + .collection::("schema_history") + .find_one(doc! { "_id": "002_backup_catalog_index" }) + .await + .expect("read") + .is_some() ); - let refused = fx + + let done = table_doc(&fx.engine, "bf_done").await.expect("bf_done"); + let job = done.get_document(RESTORE_JOB).expect("job record"); + assert_eq!(job.get_str("status"), Ok("DONE")); + assert_eq!(job.get_str("kind"), Ok("legacy")); + assert_eq!(job.get_i64("owner_generation"), Ok(0)); + assert_eq!( + job.get_datetime("heartbeat_at") + .map(|d| d.timestamp_millis()), + Ok(0), + "the epoch, not absent" + ); + assert_eq!( + job.get_datetime("completed_at"), + done.get_datetime("creation_date_time"), + "completed when the table was created" + ); + assert!(!done.contains_key("restore_operation_id")); + assert!(!done.contains_key("restore_heartbeat_at")); + let handed_doc = table_doc(&fx.engine, "bf_handed").await.expect("bf_handed"); + assert_eq!(handed_doc.get_str("table_status"), Ok("CREATING")); + assert_eq!(handed_doc.get_bool("restore_backfill_pending"), Ok(true)); + let job = handed_doc.get_document(RESTORE_JOB).expect("job record"); + assert_eq!( + job.get_str("status"), + Ok("DONE"), + "a target handed to the worker has its copy complete: DONE, not RUNNING" + ); + assert_eq!(job.get_str("kind"), Ok("legacy")); + assert_eq!(job.get_i64("owner_generation"), Ok(0)); + assert_eq!( + job.get_datetime("completed_at"), + handed_doc.get_datetime("creation_date_time") + ); + let cut = table_doc(&fx.engine, "bf_cut").await.expect("bf_cut"); + assert_eq!(cut.get_str("table_status"), Ok("CREATING")); + let job = cut.get_document(RESTORE_JOB).expect("job record"); + assert_eq!(job.get_str("status"), Ok("RUNNING")); + assert_eq!(job.get_str("kind"), Ok("legacy")); + assert_eq!(job.get_i64("owner_generation"), Ok(0)); + assert_eq!( + job.get_datetime("heartbeat_at") + .map(|d| d.timestamp_millis()), + Ok(0) + ); + assert!(!job.contains_key("completed_at")); + + // A second run changes nothing. + let again = apply_backup_catalog_index(&fx.engine.catalog_db) + .await + .expect("a second run"); + assert_eq!(again.registrations_backfilled, 0); + assert!(!again.catalog_id_written); + + // The first reconciler pass: the interrupted restore is dead. + let dead = fx .engine - .delete_table( - ACCOUNT, - DeleteTableInput { - table_name: "dst".to_owned(), - }, - ) - .await; + .list_dead_restore_jobs(STALE_AFTER) + .await + .expect("list"); + assert_eq!( + dead.len(), + 1, + "the interrupted restore alone; the handed-over one is DONE: {dead:?}" + ); + assert_eq!( + dead[0].target, + RestoreTargetKey { + account_id: ACCOUNT.to_owned(), + table_name: "bf_cut".to_owned(), + table_id: interrupted.table_id.clone(), + } + ); + assert_eq!(dead[0].owner_generation, UNOWNED_GENERATION); + assert_eq!(dead[0].kind, RestoreKind::Legacy); + let indexes = index_ids(&fx.engine, &interrupted.table_id).await; + assert_eq!(indexes.len(), 2); assert!( - matches!(&refused, Err(StorageError::IndexesInUse(m)) if m.contains("being restored")), - "DeleteTable of a target being restored is refused: {refused:?}" + fx.engine + .fail_restore_job(&dead[0].target, "owner dead") + .await + .unwrap() + ); + assert!( + fx.engine + .remove_restore_target(&dead[0].target) + .await + .unwrap() + ); + assert!(table_doc(&fx.engine, "bf_cut").await.is_none()); + assert!(!collection_exists(&fx.engine, &data_collection_name(&interrupted.table_id)).await); + for (_, index_id, _) in &indexes { + assert!(!collection_exists(&fx.engine, &data_collection_name(index_id)).await); + } + assert!( + index_ids(&fx.engine, &interrupted.table_id) + .await + .is_empty() ); assert_eq!( - table_doc(&fx.engine, "dst") + table_doc(&fx.engine, "bf_done") .await - .expect("target") + .expect("bf_done") .get_str("table_status"), - Ok("CREATING"), - "both calls ran inside the copy window" + Ok("ACTIVE") ); + // The worker finishes the handed-over target once its index is done. + index_docs + .update_one( + doc! { "_id.table_id": &handed.table_id, "_id.index_name": "g1" }, + doc! { "$set": { "index_status": "ACTIVE" } }, + ) + .await + .expect("the index backfill done"); + let finished = wait_for_status(&fx.engine, "bf_handed", "ACTIVE").await; + assert!(!finished.contains_key("restore_backfill_pending")); + assert_eq!( + finished + .get_document(RESTORE_JOB) + .expect("job") + .get_str("status"), + Ok("DONE") + ); + fx.finish().await; +} - gate.add_permits(1); - assert_fully_restored(&fx.engine, "dst").await; - fx.engine - .delete_table( - ACCOUNT, - DeleteTableInput { - table_name: "dst".to_owned(), - }, +/// `serve` refuses a catalog whose `catalog_version` is not this binary's, +/// naming `extenddb migrate --yes`; the step that brings it forward makes the +/// same catalog acceptable again. +#[tokio::test(flavor = "multi_thread")] +#[ignore = "needs a MongoDB replica set in EXTENDDB_TEST_MONGODB_URI"] +async fn live_serve_refuses_a_catalog_at_an_older_version() { + let fx = Fixture::new().await; + require_served_catalog(&fx.engine.catalog_db) + .await + .expect("a migrated catalog is served"); + + let settings = fx.engine.catalog_db.collection::("settings"); + settings + .update_one( + doc! { "_id": "catalog_version" }, + doc! { "$set": { "value": "0.0.2" } }, ) .await - .expect("DeleteTable after ACTIVE"); - let deleted = fx + .expect("old version"); + let refusal = require_served_catalog(&fx.engine.catalog_db) + .await + .expect_err("a 0.0.2 catalog is refused") + .to_string(); + assert!(refusal.contains("extenddb migrate --yes"), "{refusal}"); + assert!(refusal.contains("at version 0.0.2"), "{refusal}"); + assert!(refusal.contains(CATALOG_VERSION), "{refusal}"); + + settings + .delete_one(doc! { "_id": "catalog_version" }) + .await + .expect("no version row"); + assert!( + matches!( + require_served_catalog(&fx.engine.catalog_db).await, + Err(extenddb_storage::migrations::MigrationError::Status( + extenddb_storage::migrations::CatalogStatus::Uninitialized + )) + ), + "a catalog without a version row is not a catalog" + ); + + apply_backup_catalog_index(&fx.engine.catalog_db) + .await + .expect("migrate"); + require_served_catalog(&fx.engine.catalog_db) + .await + .expect("served again once migrated"); + fx.finish().await; +} + +/// The legacy restore registers through the job record: RUNNING under the +/// claim's generation while the copy runs, with a heartbeat on the server's +/// clock, and DONE with the hand-over. +#[tokio::test(flavor = "multi_thread")] +#[ignore = "needs a MongoDB replica set in EXTENDDB_TEST_MONGODB_URI"] +async fn live_a_legacy_restore_registers_a_running_job_record_and_finishes_it_done() { + const FEW: usize = 300; + let fx = Fixture::new().await; + source_table_with(&fx.engine, "src", FEW).await; + let backup = fx .engine - .delete_backup(ACCOUNT, &backup.backup_arn) + .create(ACCOUNT, "src", "b", JobPermit::detached()) .await - .expect("DeleteBackup after ACTIVE"); - assert_eq!(deleted.backup_details.backup_status, "DELETED"); + .expect("backup"); + let (gated, gate) = gated(&fx.engine); + let request = restore_request(&fx.engine, &backup.backup_arn, "dst").await; + gated + .restore(ACCOUNT, request, JobPermit::detached()) + .await + .expect("registers"); + let job = job_record(&fx.engine, "dst").await; + assert_eq!(job.get_str("status"), Ok("RUNNING")); + assert_eq!(job.get_str("kind"), Ok("legacy")); + let generation = job.get_i64("owner_generation").expect("generation"); + assert_ne!(generation, i64::try_from(UNOWNED_GENERATION).unwrap()); + let first = *job.get_datetime("heartbeat_at").expect("heartbeat"); + let age_ms = bson::DateTime::now().timestamp_millis() - first.timestamp_millis(); + assert!(age_ms.abs() < 60_000, "a fresh heartbeat: {age_ms} ms old"); + assert!( + fx.engine + .list_dead_restore_jobs(STALE_AFTER) + .await + .unwrap() + .is_empty(), + "a running restore is not dead" + ); + // The heartbeat moves while the copy is held. + tokio::time::sleep(fx.engine.backup_liveness.heartbeat_interval * 3).await; + let later = *job_record(&fx.engine, "dst") + .await + .get_datetime("heartbeat_at") + .expect("heartbeat"); + assert!(later > first, "the heartbeat moved: {later:?} > {first:?}"); + + gate.add_permits(1); + assert_fully_restored_with(&fx.engine, "dst", FEW).await; + let job = job_record(&fx.engine, "dst").await; + assert_eq!(job.get_str("status"), Ok("DONE")); + assert_eq!(job.get_i64("owner_generation"), Ok(generation)); + assert!(job.get_datetime("completed_at").is_ok()); fx.finish().await; } -/// The definition captured at CreateBackup carries TTL and the stream -/// specification, and ListBackups reports the source table id. +/// A restore whose heartbeat fails while its `$out` runs on the server (a +/// fault on the heartbeat's connection, injected at the next tick once the +/// test has seen the `$out` in `currentOp`): the copy stops without touching +/// the target (the record FAILED and the target removed, each conditioned), +/// and the `$out` the server runs to its end leaves no orphan +/// `_data_`: the removal dropped the target collection the +/// registration created, and `$out` refuses to replace a target whose +/// indexes changed while it ran. The name is free and the backup no longer +/// in use afterwards. The copy is large enough that `$out` outlasts the +/// probe and several heartbeat periods, and the fault cell records that the +/// heartbeat failed as requested. #[tokio::test(flavor = "multi_thread")] #[ignore = "needs a MongoDB replica set in EXTENDDB_TEST_MONGODB_URI"] -async fn live_create_backup_captures_ttl_stream_and_table_id() { - use extenddb_storage::MetadataEngine; +async fn live_a_heartbeat_lost_while_out_runs_leaves_no_orphan_collection() { + const MANY: usize = 150_000; let fx = Fixture::new().await; - source_table(&fx.engine, "src").await; - fx.engine - .update_ttl(ACCOUNT, "src", "expires", true) + source_table_with(&fx.engine, "src", MANY).await; + let backup = fx + .engine + .create(ACCOUNT, "src", "b", JobPermit::detached()) .await - .expect("ttl"); - let table = table_doc(&fx.engine, "src").await.expect("src"); - let table_id = table.get_str("table_id").expect("id").to_owned(); - let mut backups = Vec::new(); - for i in 0..3 { - backups.push( - fx.engine - .create_backup(ACCOUNT, "src", &format!("b{i}")) - .await - .expect("backup"), - ); - } - for b in &backups { - let source = fx - .engine - .backup_restore_source(ACCOUNT, &b.backup_arn) + .expect("backup"); + let backup_key = extenddb_storage::backup_catalog::BackupKey::from_arn( + &BackupArn::parse(&backup.backup_arn).expect("arn"), + ); + let fault = Arc::new(std::sync::atomic::AtomicU8::new(HeartbeatFault::IDLE)); + let mut faulty = fx.engine.clone(); + faulty.backup_liveness.heartbeat_interval = Duration::from_millis(50); + faulty.backup_liveness.heartbeat_fault = Some(fault.clone()); + let request = restore_request(&faulty, &backup.backup_arn, "dst").await; + let desc = faulty + .restore(ACCOUNT, request, JobPermit::detached()) + .await + .expect("restore registers"); + let data = data_collection_name(&desc.table_id); + + // The `$out` shows in `currentOp` as an insert into its temporary + // collection whose command is the aggregate over the backup collection + // (named by the legacy document's own `backup_id`, not the ARN's). + let legacy_id = backup_meta(&fx.engine, &backup.backup_arn) + .await + .get_str("backup_id") + .expect("backup_id") + .to_owned(); + let out_source = super::backup_collection_name(&legacy_id); + let out_running = || async { + !fx.engine + .client + .database("admin") + .run_command(doc! { + "currentOp": 1, + "$ownOps": false, + "command.aggregate": &out_source, + "command.pipeline.$out": { "$exists": true }, + }) .await - .expect("source"); - assert_eq!( - Some(source.backup_creation_date_time), - extenddb_storage::util::backup_creation_date_time(&b.backup_arn) - ); - assert_eq!( - source.backup_creation_date_time, - b.backup_creation_date_time + .expect("currentOp") + .get_array("inprog") + .map(|ops| ops.is_empty()) + .unwrap_or(true) + }; + let deadline = Instant::now() + Duration::from_secs(60); + while !out_running().await { + assert!(Instant::now() < deadline, "the copy's $out did not start"); + tokio::time::sleep(Duration::from_millis(20)).await; + } + fault.store(HeartbeatFault::ARMED, std::sync::atomic::Ordering::SeqCst); + + // The catalog side: the target is removed once the heartbeat fails, with + // `$out` still running. + let deadline = Instant::now() + Duration::from_secs(60); + while table_doc(&fx.engine, "dst").await.is_some() { + assert!( + Instant::now() < deadline, + "the target was not removed after the heartbeat failed" ); - let def = source.definition.expect("definition"); - let ttl = def.time_to_live.expect("ttl captured"); - assert_eq!(ttl.attribute_name.as_deref(), Some("expires")); - assert!(def.stream_specification.is_none()); - assert_eq!(def.global_secondary_indexes.len(), 1); - assert_eq!(def.local_secondary_indexes.len(), 1); - let described = fx - .engine - .describe_backup(ACCOUNT, &b.backup_arn) + tokio::time::sleep(Duration::from_millis(20)).await; + } + assert_eq!( + fault.load(std::sync::atomic::Ordering::SeqCst), + HeartbeatFault::INJECTED, + "the heartbeat failed as requested" + ); + assert!( + out_running().await, + "the window was real: $out was still running when the target was removed" + ); + + // The server side: `$out` ends on its own and leaves no collection. + let deadline = Instant::now() + Duration::from_secs(60); + while out_running().await { + assert!(Instant::now() < deadline, "$out did not end"); + tokio::time::sleep(Duration::from_millis(100)).await; + } + tokio::time::sleep(Duration::from_millis(500)).await; + assert!( + !collection_exists(&fx.engine, &data).await, + "no orphan data collection after $out ended" + ); + assert!(index_ids(&fx.engine, &desc.table_id).await.is_empty()); + assert!(table_doc(&fx.engine, "dst").await.is_none()); + assert_eq!(fx.engine.backup_in_use(&backup_key).await.unwrap(), None); + + // Liveness: the name restores again (the backup is not in use and the + // name is free), through to the hand-over; the index backfill of a table + // this size is the worker's and is not waited for. + let request = restore_request(&fx.engine, &backup.backup_arn, "dst").await; + let again = fx + .engine + .restore(ACCOUNT, request, JobPermit::detached()) + .await + .expect("restore again"); + let deadline = Instant::now() + Duration::from_secs(60); + loop { + let doc = table_doc(&fx.engine, "dst") .await - .expect("describe"); - assert_eq!( - described.backup_details.backup_creation_date_time, - b.backup_creation_date_time + .expect("the second target"); + let handed = doc + .get_document(RESTORE_JOB) + .map(|job| job.get_str("status") == Ok("DONE")) + .unwrap_or(false); + if handed || doc.get_str("table_status") == Ok("ACTIVE") { + break; + } + assert!( + Instant::now() < deadline, + "the second restore did not hand over" ); - assert!(described.source_table_feature_details.is_none()); + tokio::time::sleep(Duration::from_millis(100)).await; } - let listed = fx.engine.list_backups(ACCOUNT, None).await.expect("list"); - assert_eq!(listed.len(), 3); - for summary in &listed { - assert_eq!(summary.table_id, table_id); - assert_eq!( - Some(summary.backup_creation_date_time), - extenddb_storage::util::backup_creation_date_time(&summary.backup_arn) + assert_eq!( + count(&fx.engine, &data_collection_name(&again.table_id)).await, + MANY as u64 + ); + fx.finish().await; +} + +/// A RUNNING restore record is dead by the age of its heartbeat on the +/// server's clock, whatever the table's age; a stale record is marked FAILED +/// and its target removed through `remove_restore_target`, with its data and +/// index collections, which frees the backup and the name; a fresh record, +/// and a record that is no longer RUNNING, are left alone. +#[tokio::test(flavor = "multi_thread")] +#[ignore = "needs a MongoDB replica set in EXTENDDB_TEST_MONGODB_URI"] +async fn live_an_abandoned_restore_is_removed_only_when_its_heartbeat_is_stale() { + const FEW: usize = 300; + let fx = Fixture::new().await; + source_table_with(&fx.engine, "src", FEW).await; + let backup = fx + .engine + .create(ACCOUNT, "src", "b", JobPermit::detached()) + .await + .expect("backup"); + let request = restore_request(&fx.engine, &backup.backup_arn, "dst").await; + fx.engine + .restore(ACCOUNT, request, JobPermit::detached()) + .await + .expect("restore"); + assert_fully_restored_with(&fx.engine, "dst", FEW).await; + let doc = table_doc(&fx.engine, "dst").await.expect("target"); + let key = target_key(&doc); + let indexes = index_ids(&fx.engine, &key.table_id).await; + assert_eq!(indexes.len(), 2); + let backup_key = extenddb_storage::backup_catalog::BackupKey::from_arn( + &BackupArn::parse(&backup.backup_arn).expect("arn"), + ); + + // An ACTIVE former target under the right key: no longer a CREATING + // restore target, so the removal is refused and the table, its data and + // its index collections survive (the `table_status: CREATING` condition + // is what excludes the removal from the ACTIVE flip). + assert!( + !fx.engine.remove_restore_target(&key).await.expect("remove"), + "an ACTIVE table under the key is not a CREATING restore target" + ); + let active = table_doc(&fx.engine, "dst") + .await + .expect("the ACTIVE table survives"); + assert_eq!(active.get_str("table_status"), Ok("ACTIVE")); + assert_eq!( + count(&fx.engine, &data_collection_name(&key.table_id)).await, + FEW as u64 + ); + for (index_name, index_id, _) in &indexes { + assert!( + collection_exists(&fx.engine, &data_collection_name(index_id)).await, + "index {index_name} survives" ); } - let foreign = fx + + // A fresh heartbeat: a live restore, here or on another instance. + age_running_record(&fx.engine, "dst", Duration::from_secs(1)).await; + assert!( + fx.engine + .list_dead_restore_jobs(STALE_AFTER) + .await + .expect("list") + .is_empty() + ); + assert_eq!( + fx.engine + .backup_in_use(&backup_key) + .await + .unwrap() + .as_deref(), + Some("dst"), + "a CREATING target keeps its backup in use" + ); + let in_use = fx.engine.delete(ACCOUNT, &backup.backup_arn).await; + assert!( + matches!(in_use, Err(StorageError::BackupInUse(_))), + "{in_use:?}" + ); + + // A stale heartbeat: dead. + age_running_record(&fx.engine, "dst", Duration::from_secs(600)).await; + let dead = fx + .engine + .list_dead_restore_jobs(STALE_AFTER) + .await + .expect("list"); + assert_eq!(dead.len(), 1, "{dead:?}"); + assert_eq!(dead[0].target, key); + assert_eq!(dead[0].kind, RestoreKind::Legacy); + assert_eq!(dead[0].status, JobStatus::Running); + assert_eq!(dead[0].source_backup_arn, backup.backup_arn); + assert!( + fx.engine + .fail_restore_job(&key, "owner dead") + .await + .expect("fail") + ); + let job = job_record(&fx.engine, "dst").await; + assert_eq!(job.get_str("status"), Ok("FAILED")); + assert_eq!(job.get_str("failure_reason"), Ok("owner dead")); + assert!(job.get_datetime("completed_at").is_ok()); + assert!( + !fx.engine + .fail_restore_job(&key, "owner dead") + .await + .expect("fail again"), + "a record that is no longer RUNNING is not written twice" + ); + assert!( + fx.engine + .list_dead_restore_jobs(STALE_AFTER) + .await + .unwrap() + .is_empty(), + "a FAILED record is not dead" + ); + // A key with another table id names a table re-created under the name, + // not this target. + let other_id = RestoreTargetKey { + table_id: "another-table".to_owned(), + ..key.clone() + }; + assert!(!fx.engine.remove_restore_target(&other_id).await.unwrap()); + // The removal with the reply to its collection drops lost: the document + // is gone, so the target is removed (`true`), the name and the backup + // are free and no record is left for a later cycle; the collections, + // dropped before the reply was lost, are gone too. + let mut drops_fail = fx.engine.clone(); + drops_fail.backup_liveness.target_drop_fails = true; + assert!( + drops_fail + .remove_restore_target(&key) + .await + .expect("the target is removed although the reply to its collection drops was lost") + ); + assert!(table_doc(&fx.engine, "dst").await.is_none()); + assert!(!collection_exists(&fx.engine, &data_collection_name(&key.table_id)).await); + for (_, index_id, _) in &indexes { + assert!(!collection_exists(&fx.engine, &data_collection_name(index_id)).await); + } + assert!(index_ids(&fx.engine, &key.table_id).await.is_empty()); + assert!( + !fx.engine.remove_restore_target(&key).await.unwrap(), + "a target that is gone is not removed twice" + ); + + // The backup and the name are free again. + assert_eq!(fx.engine.backup_in_use(&backup_key).await.unwrap(), None); + let request = restore_request(&fx.engine, &backup.backup_arn, "dst").await; + fx.engine + .restore(ACCOUNT, request, JobPermit::detached()) + .await + .expect("restore into the freed name"); + assert_fully_restored_with(&fx.engine, "dst", FEW).await; + let deleted = fx .engine - .backup_restore_source("999999999999", &backups[0].backup_arn) + .delete(ACCOUNT, &backup.backup_arn) + .await + .expect("DeleteBackup once the target is ACTIVE"); + assert_eq!(deleted.backup_details.backup_status, "DELETED"); + fx.finish().await; +} + +/// The FAILED-target duty's listing: a FAILED record whose table is still a +/// CREATING restore target (the FAILED write landed and the removal did not) +/// is not listed while the record is fresh (the task that failed it may +/// still be removing the target), is listed once `completed_at` is older +/// than the stale window, as FAILED, and its target is then removed with no +/// second FAILED write; a FAILED record without a `completed_at` is aged by +/// its heartbeat; a FAILED record whose table is DELETING or ACTIVE is never +/// listed. +#[tokio::test(flavor = "multi_thread")] +#[ignore = "needs a MongoDB replica set in EXTENDDB_TEST_MONGODB_URI"] +async fn live_a_failed_record_whose_target_removal_failed_is_listed_once_stale_and_removed() { + const FEW: usize = 300; + let fx = Fixture::new().await; + source_table_with(&fx.engine, "src", FEW).await; + let backup = fx + .engine + .create(ACCOUNT, "src", "b", JobPermit::detached()) + .await + .expect("backup"); + let request = restore_request(&fx.engine, &backup.backup_arn, "dst").await; + fx.engine + .restore(ACCOUNT, request, JobPermit::detached()) + .await + .expect("restore"); + assert_fully_restored_with(&fx.engine, "dst", FEW).await; + let key = target_key(&table_doc(&fx.engine, "dst").await.expect("target")); + let server_ago = |age: Duration| { + bson::DateTime::from_millis( + bson::DateTime::now().timestamp_millis() - i64::try_from(age.as_millis()).expect("i64"), + ) + }; + let set = |update: Document| { + let tables = fx.engine.catalog_db.collection::("tables"); + async move { + tables + .update_one( + doc! { "_id": { "account_id": ACCOUNT, "table_name": "dst" } }, + update, + ) + .await + .expect("set the record"); + } + }; + let listed = || async { + fx.engine + .list_dead_restore_jobs(STALE_AFTER) + .await + .expect("list") + }; + + // The state a dead restore leaves once the reconciler failed its record + // and the target's removal did not land. + age_running_record(&fx.engine, "dst", Duration::from_secs(600)).await; + let dead = listed().await; + assert_eq!(dead.len(), 1, "{dead:?}"); + assert_eq!(dead[0].status, JobStatus::Running); + assert!( + fx.engine + .fail_restore_job(&key, "owner dead") + .await + .expect("fail") + ); + let job = job_record(&fx.engine, "dst").await; + assert_eq!(job.get_str("status"), Ok("FAILED")); + assert!(job.get_datetime("completed_at").is_ok()); + assert_eq!( + table_doc(&fx.engine, "dst") + .await + .expect("target") + .get_str("table_status"), + Ok("CREATING") + ); + + // Fresh: `fail_restore_job` stamped `completed_at` with the server's + // clock a moment ago, and whoever failed it may still be removing the + // target. + assert!( + listed().await.is_empty(), + "a FAILED record within the stale window is left to its writer" + ); + // Stale: listed, as FAILED, for the FAILED-target duty. + set(doc! { "$set": { "restore_job.completed_at": server_ago(Duration::from_secs(600)) } }) .await; + let dead = listed().await; + assert_eq!(dead.len(), 1, "{dead:?}"); + assert_eq!(dead[0].target, key); + assert_eq!(dead[0].status, JobStatus::Failed); + assert_eq!(dead[0].kind, RestoreKind::Legacy); + assert_eq!(dead[0].source_backup_arn, backup.backup_arn); + // A FAILED record without `completed_at` is aged by its heartbeat: fresh, + // not listed; stale, listed. + set(doc! { + "$set": { RESTORE_JOB_HEARTBEAT: bson::DateTime::now() }, + "$unset": { "restore_job.completed_at": "" }, + }) + .await; assert!( - matches!(&foreign, Err(StorageError::Validation(m)) if m.starts_with("Backup not found")), - "{foreign:?}" + listed().await.is_empty(), + "a FAILED record with a fresh heartbeat and no completed_at is fresh" + ); + set(doc! { "$set": { RESTORE_JOB_HEARTBEAT: server_ago(Duration::from_secs(600)) } }).await; + assert_eq!(listed().await.len(), 1); + // A FAILED record whose table the removal already moved to DELETING is + // the control plane's row. + set(doc! { "$set": { "table_status": "DELETING" } }).await; + assert!( + listed().await.is_empty(), + "a DELETING table is not a target" + ); + set(doc! { "$set": { "table_status": "CREATING" } }).await; + + // The reconciler's FAILED-target duty: no second FAILED write, the target + // removed with its collections. + assert!( + !fx.engine + .fail_restore_job(&key, "owner dead") + .await + .expect("fail again"), + "a FAILED record is not RUNNING and is not written FAILED again" + ); + assert!(fx.engine.remove_restore_target(&key).await.expect("remove")); + assert!(table_doc(&fx.engine, "dst").await.is_none()); + assert!(!collection_exists(&fx.engine, &data_collection_name(&key.table_id)).await); + assert!(listed().await.is_empty()); + assert_eq!( + fx.engine + .backup_in_use(&extenddb_storage::backup_catalog::BackupKey::from_arn( + &BackupArn::parse(&backup.backup_arn).expect("arn"), + )) + .await + .unwrap(), + None, + "the backup is free once the target is gone" + ); + + // A FAILED record whose table is ACTIVE (a flip that won against a late + // FAILED write) is nobody's duty. + let request = restore_request(&fx.engine, &backup.backup_arn, "dst").await; + fx.engine + .restore(ACCOUNT, request, JobPermit::detached()) + .await + .expect("restore again"); + assert_fully_restored_with(&fx.engine, "dst", FEW).await; + set(doc! { "$set": { + RESTORE_JOB_STATUS: "FAILED", + "restore_job.completed_at": server_ago(Duration::from_secs(600)), + } }) + .await; + assert!( + listed().await.is_empty(), + "a FAILED record of an ACTIVE table is not listed" ); fx.finish().await; } -/// The GSI backfill worker's periodic sweep, with no explicit sweep call and -/// no restart, removes a crashed restore and finishes an interrupted -/// DeleteBackup. +/// A restore target of another, live, instance on the same catalog is not +/// dead for many stale windows, because its heartbeat keeps moving, while a +/// record whose owner stopped is; DeleteBackup of the live one's source is +/// refused; the live restore then finishes. #[tokio::test(flavor = "multi_thread")] #[ignore = "needs a MongoDB replica set in EXTENDDB_TEST_MONGODB_URI"] -async fn live_the_worker_sweeps_abandoned_state_without_a_restart() { +async fn live_a_restore_of_another_instance_is_not_dead_while_it_heartbeats() { + const FEW: usize = 300; let fx = Fixture::new().await; - source_table(&fx.engine, "src").await; - let crashed = fx + source_table_with(&fx.engine, "src", FEW).await; + let b1 = fx .engine - .create_backup(ACCOUNT, "src", "b1") + .create(ACCOUNT, "src", "b1", JobPermit::detached()) .await .expect("backup 1"); - let interrupted = fx + let b2 = fx .engine - .create_backup(ACCOUNT, "src", "b2") + .create(ACCOUNT, "src", "b2", JobPermit::detached()) .await .expect("backup 2"); - simulate_crashed_restore(&fx.engine, &crashed.backup_arn, "dst").await; - let mut set = doc! { DELETE_CLAIM: "dead-delete" }; - set.insert( - DELETE_CLAIM_HEARTBEAT, - bson::DateTime::from_millis(bson::DateTime::now().timestamp_millis() - 600_000), - ); + // A restore whose owner stopped: finished, then aged into a dead record. + let request = restore_request(&fx.engine, &b1.backup_arn, "stopped").await; fx.engine - .catalog_db - .collection::("backups") - .update_one( - doc! { "_id": &interrupted.backup_arn }, - doc! { "$set": set }, - ) + .restore(ACCOUNT, request, JobPermit::detached()) .await - .expect("stale delete claim"); + .expect("restore"); + assert_fully_restored_with(&fx.engine, "stopped", FEW).await; + age_running_record(&fx.engine, "stopped", Duration::from_secs(60)).await; - // Worst case: one heartbeat interval until the sweep is due, plus the - // worker's 5 s tick; the state is already older than stale_after. - let deadline = - Instant::now() + fx.engine.backup_liveness.heartbeat_interval + Duration::from_secs(15); - loop { - let target_gone = table_doc(&fx.engine, "dst").await.is_none(); - let status = fx + // A live restore of another instance, held before its copy. + let other = fx.second_instance(test_liveness()).await; + let stale_after = other.backup_liveness.heartbeat_interval * 6; + let (gated_other, gate) = gated(&other); + let request = restore_request(&other, &b2.backup_arn, "dst").await; + gated_other + .restore(ACCOUNT, request, JobPermit::detached()) + .await + .expect("restore"); + let first = *job_record(&fx.engine, "dst") + .await + .get_datetime("heartbeat_at") + .expect("heartbeat"); + + // Across three stale windows, only the stopped one is dead. + let until = Instant::now() + stale_after * 3; + let mut checks = 0; + while Instant::now() < until { + let dead = fx .engine - .catalog_db - .collection::("backups") - .find_one(doc! { "_id": &interrupted.backup_arn }) + .list_dead_restore_jobs(stale_after) .await - .expect("read") - .and_then(|d| d.get_str("backup_status").ok().map(str::to_owned)); - if target_gone && status.as_deref() == Some("DELETED") { - break; - } - assert!( - Instant::now() < deadline, - "the worker did not sweep: target gone {target_gone}, b2 status {status:?}" + .expect("list"); + assert_eq!( + dead.iter() + .map(|j| j.target.table_name.as_str()) + .collect::>(), + ["stopped"], + "only the record whose owner stopped is dead" ); - tokio::time::sleep(Duration::from_millis(200)).await; + checks += 1; + tokio::time::sleep(Duration::from_millis(250)).await; } - fx.engine - .delete_backup(ACCOUNT, &crashed.backup_arn) - .await - .expect("DeleteBackup after the worker's sweep"); - fx.finish().await; -} - -async fn backup_meta(engine: &MongoEngine, backup_arn: &str) -> Document { - engine - .catalog_db - .collection::("backups") - .find_one(doc! { "_id": backup_arn }) + assert!(checks >= 10); + // Liveness: the live heartbeat moved. + let later = *job_record(&fx.engine, "dst") .await - .expect("read backup") - .expect("backup document") -} + .get_datetime("heartbeat_at") + .expect("heartbeat"); + assert!(later > first, "{later:?} > {first:?}"); + let in_use = fx.engine.delete(ACCOUNT, &b2.backup_arn).await; + assert!( + matches!(in_use, Err(StorageError::BackupInUse(_))), + "{in_use:?}" + ); -fn with_drop_fault(engine: &MongoEngine, fault: AfterBackupDrop) -> MongoEngine { - let mut faulty = engine.clone(); - faulty.backup_liveness.after_backup_drop = Some(fault); - faulty + gate.add_permits(1); + assert_fully_restored_with(&fx.engine, "dst", FEW).await; + fx.finish().await; } -/// After a DeleteBackup or sweep stopped after its drop: the claim is still -/// held, the backup's data is gone, the backup is not restorable, and once the -/// claim is stale the sweep finishes the delete. -async fn assert_claim_held_then_swept(engine: &MongoEngine, backup_arn: &str, backup_id: &str) { - // Give a claim's Drop (a spawned release) time to run if it were armed. - tokio::time::sleep(Duration::from_millis(500)).await; +/// After a DeleteBackup stopped after its drop: the backup is DELETING, so +/// it is neither described, listed nor restorable, and a retry of +/// DeleteBackup resumes the drop and answers DELETED. +async fn assert_deleting_then_resumed(engine: &MongoEngine, backup_arn: &str, backup_id: &str) { let meta = backup_meta(engine, backup_arn).await; - assert!( - meta.contains_key(DELETE_CLAIM), - "the delete claim must not be released once the drop may have run: {meta:?}" + assert_eq!( + meta.get_str("backup_status"), + Ok("DELETING"), + "the backup must not read AVAILABLE once the drop may have run: {meta:?}" ); - assert_eq!(meta.get_str("backup_status"), Ok("AVAILABLE")); assert!(!collection_exists(engine, &format!("_backup_{backup_id}")).await); - let request = restore_request(engine, backup_arn, "dst").await; - let refused = engine.restore_table_from_backup(ACCOUNT, request).await; + let described = engine.describe(ACCOUNT, backup_arn).await; + assert!( + matches!(&described, Err(StorageError::Validation(m)) if m.starts_with("Backup not found")), + "{described:?}" + ); + assert!(engine.list(ACCOUNT, None).await.expect("list").is_empty()); + let source = engine.restore_source(ACCOUNT, backup_arn).await; assert!( - matches!(&refused, Err(StorageError::BackupInUse(_))), - "a restore must not start from a backup whose data was dropped: {refused:?}" + matches!(&source, Err(StorageError::Validation(m)) if m.starts_with("Backup not found")), + "a restore must not start from a backup whose data was dropped: {source:?}" ); assert!(table_doc(engine, "dst").await.is_none()); - let operation = meta.get_str(DELETE_CLAIM).expect("claim id").to_owned(); - let second = engine.delete_backup(ACCOUNT, backup_arn).await; + let resumed = engine + .delete(ACCOUNT, backup_arn) + .await + .expect("a retry resumes the delete"); + assert_eq!(resumed.backup_details.backup_status, "DELETED"); + assert_eq!( + backup_meta(engine, backup_arn) + .await + .get_str("backup_status"), + Ok("DELETED") + ); + let again = engine.delete(ACCOUNT, backup_arn).await; assert!( - matches!(&second, Err(StorageError::BackupInUse(m)) if m.starts_with("Backup is being deleted")), - "a DeleteBackup during a held delete claim must answer BackupInUse: {second:?}" + matches!(&again, Err(StorageError::Validation(m)) if m.starts_with("Backup not found")), + "{again:?}" ); - let meta = backup_meta(engine, backup_arn).await; - assert_eq!(meta.get_str(DELETE_CLAIM), Ok(operation.as_str())); - assert_eq!(meta.get_str("backup_status"), Ok("AVAILABLE")); - - tokio::time::sleep(engine.backup_liveness.stale_after + Duration::from_millis(500)).await; - let sweep = engine.sweep_stale_backup_claims().await.expect("sweep"); - assert_eq!(sweep.delete_claims_finished, [backup_arn.to_owned()]); - let meta = backup_meta(engine, backup_arn).await; - assert_eq!(meta.get_str("backup_status"), Ok("DELETED")); - assert!(!meta.contains_key(DELETE_CLAIM)); } /// The reply to DeleteBackup's drop is an error after the server dropped -/// the collection. The request fails, but the claim stays until the sweep -/// finishes the delete. +/// the collection. The request fails, but the backup stays DELETING until a +/// retry finishes the delete. #[tokio::test(flavor = "multi_thread")] #[ignore = "needs a MongoDB replica set in EXTENDDB_TEST_MONGODB_URI"] -async fn live_a_delete_whose_drop_reply_fails_keeps_its_claim() { +async fn live_a_delete_whose_drop_reply_fails_leaves_the_backup_deleting_until_a_retry() { let fx = Fixture::new().await; - fx.worker.abort(); - source_table(&fx.engine, "src").await; + source_table_with(&fx.engine, "src", 300).await; let backup = fx .engine - .create_backup(ACCOUNT, "src", "b") + .create(ACCOUNT, "src", "b", JobPermit::detached()) .await .expect("backup"); let backup_id = backup_meta(&fx.engine, &backup.backup_arn) @@ -980,12 +1848,12 @@ async fn live_a_delete_whose_drop_reply_fails_keeps_its_claim() { .expect("backup_id") .to_owned(); let faulty = with_drop_fault(&fx.engine, AfterBackupDrop::Fail); - let failed = faulty.delete_backup(ACCOUNT, &backup.backup_arn).await; + let failed = faulty.delete(ACCOUNT, &backup.backup_arn).await; assert!( matches!(&failed, Err(StorageError::Internal(_))), "{failed:?}" ); - assert_claim_held_then_swept(&fx.engine, &backup.backup_arn, &backup_id).await; + assert_deleting_then_resumed(&fx.engine, &backup.backup_arn, &backup_id).await; fx.finish().await; } @@ -993,13 +1861,12 @@ async fn live_a_delete_whose_drop_reply_fails_keeps_its_claim() { /// for the reply to a drop the server already ran. #[tokio::test(flavor = "multi_thread")] #[ignore = "needs a MongoDB replica set in EXTENDDB_TEST_MONGODB_URI"] -async fn live_a_delete_dropped_while_waiting_for_its_drop_keeps_its_claim() { +async fn live_a_delete_dropped_while_waiting_for_its_drop_leaves_the_backup_deleting() { let fx = Fixture::new().await; - fx.worker.abort(); - source_table(&fx.engine, "src").await; + source_table_with(&fx.engine, "src", 300).await; let backup = fx .engine - .create_backup(ACCOUNT, "src", "b") + .create(ACCOUNT, "src", "b", JobPermit::detached()) .await .expect("backup"); let backup_id = backup_meta(&fx.engine, &backup.backup_arn) @@ -1010,224 +1877,469 @@ async fn live_a_delete_dropped_while_waiting_for_its_drop_keeps_its_claim() { let faulty = with_drop_fault(&fx.engine, AfterBackupDrop::Hang); let timed_out = tokio::time::timeout( Duration::from_secs(3), - faulty.delete_backup(ACCOUNT, &backup.backup_arn), + faulty.delete(ACCOUNT, &backup.backup_arn), ) .await; assert!(timed_out.is_err(), "the delete must hang after its drop"); - assert_claim_held_then_swept(&fx.engine, &backup.backup_arn, &backup_id).await; + assert_deleting_then_resumed(&fx.engine, &backup.backup_arn, &backup_id).await; fx.finish().await; } -/// B1 addendum: the sweep's own delete takeover gets an error reply after its -/// drop ran. The claim it took over stays held, and the next sweep finishes. +/// A restore from a process whose clock is 10 minutes behind the server's +/// is not dead to another process (stale window 5 minutes): the record's +/// first heartbeat, written with the local clock at the insert, is replaced +/// by the server's clock at once, not one heartbeat interval later. #[tokio::test(flavor = "multi_thread")] #[ignore = "needs a MongoDB replica set in EXTENDDB_TEST_MONGODB_URI"] -async fn live_a_sweep_whose_drop_reply_fails_keeps_the_claim() { +async fn live_a_restore_from_a_process_with_a_slow_clock_is_not_dead() { + const FEW: usize = 300; let fx = Fixture::new().await; - fx.worker.abort(); - source_table(&fx.engine, "src").await; + source_table_with(&fx.engine, "src", FEW).await; let backup = fx .engine - .create_backup(ACCOUNT, "src", "b") + .create(ACCOUNT, "src", "b", JobPermit::detached()) .await .expect("backup"); - let backup_id = backup_meta(&fx.engine, &backup.backup_arn) + let liveness = BackupLiveness { + heartbeat_interval: Duration::from_secs(3_600), + registration_gate: None, + copy_gate: None, + after_backup_drop: None, + target_drop_fails: false, + local_clock_behind: Duration::from_secs(600), + heartbeat_fault: None, + }; + let behind = fx.second_instance(liveness.clone()).await; + let (behind, gate) = gated(&behind); + let other = fx + .second_instance(BackupLiveness { + local_clock_behind: Duration::ZERO, + ..liveness + }) + .await; + let request = restore_request(&fx.engine, &backup.backup_arn, "dst").await; + behind + .restore(ACCOUNT, request, JobPermit::detached()) .await - .get_str("backup_id") - .expect("backup_id") - .to_owned(); - let mut set = doc! { DELETE_CLAIM: "dead-delete" }; - set.insert( - DELETE_CLAIM_HEARTBEAT, - bson::DateTime::from_millis(bson::DateTime::now().timestamp_millis() - 600_000), + .expect("restore"); + let deadline = Instant::now() + Duration::from_secs(5); + loop { + let beat = *job_record(&fx.engine, "dst") + .await + .get_datetime("heartbeat_at") + .expect("heartbeat"); + let age_ms = bson::DateTime::now().timestamp_millis() - beat.timestamp_millis(); + if age_ms.abs() < 60_000 { + break; + } + assert!( + Instant::now() < deadline, + "the record's heartbeat is {age_ms} ms old: the server clock must replace the \ + local one at once" + ); + tokio::time::sleep(Duration::from_millis(50)).await; + } + assert!( + other + .list_dead_restore_jobs(STALE_AFTER) + .await + .expect("list") + .is_empty() ); - fx.engine - .catalog_db - .collection::("backups") - .update_one(doc! { "_id": &backup.backup_arn }, doc! { "$set": set }) + assert!(table_doc(&fx.engine, "dst").await.is_some()); + + gate.add_permits(1); + assert_fully_restored_with(&fx.engine, "dst", FEW).await; + fx.finish().await; +} + +/// A DeleteBackup and restores of one backup order wholly one before the +/// other: when the delete goes first, every restore is "Backup not found" +/// and no target is left; when a registration goes first, exactly one target +/// exists, the delete is BackupInUse, and so is every other restore. Even +/// rounds hold the three registrations behind a gate until the delete has +/// answered, so the delete wins; odd rounds release the registrations and +/// start the delete at the same moment, a race over the backup's lock that +/// the registrations have won on every run so far. Both orders must occur. +/// The restores are accounted by the +/// catalog (the targets that exist for the request names), not by what their +/// requests answered: a request that outlives the registration bound answers +/// an error while its target appears, and the copy of every registered +/// target is released and driven to ACTIVE before the backup is deleted. +#[tokio::test(flavor = "multi_thread")] +#[ignore = "needs a MongoDB replica set in EXTENDDB_TEST_MONGODB_URI"] +async fn live_a_delete_and_concurrent_restores_of_one_backup_serialize() { + const FEW: usize = 20; + const ROUNDS: usize = 6; + const RESTORES: usize = 3; + let fx = Fixture::new().await; + source_table_with(&fx.engine, "src", FEW).await; + // A backup that stays, so a restore request of a deleted backup can be + // built after the delete (the request is built from an AVAILABLE source). + let keep = fx + .engine + .create(ACCOUNT, "src", "keep", JobPermit::detached()) .await - .expect("stale delete claim"); + .expect("backup"); + let b_arn_for_request = keep.backup_arn.clone(); + let (mut delete_won, mut registration_won) = (0, 0); + for round in 0..ROUNDS { + let backup = fx + .engine + .create(ACCOUNT, "src", &format!("b{round}"), JobPermit::detached()) + .await + .expect("backup"); + let targets: Vec = (0..RESTORES).map(|i| format!("r{round}_{i}")).collect(); + let mut requests = Vec::new(); + for target in &targets { + requests.push(restore_request(&fx.engine, &backup.backup_arn, target).await); + } + let (gated, registration_gate, copy_gate) = gated_at_registration(&fx.engine); + let delete_first = round % 2 == 0; + let restores = futures::future::join_all( + requests + .into_iter() + .map(|request| gated.restore(ACCOUNT, request, JobPermit::detached())), + ); + let delete = async { + if delete_first { + let outcome = fx.engine.delete(ACCOUNT, &backup.backup_arn).await; + registration_gate.add_permits(RESTORES); + outcome + } else { + registration_gate.add_permits(RESTORES); + fx.engine.delete(ACCOUNT, &backup.backup_arn).await + } + }; + let (restores, delete) = tokio::join!(restores, delete); + // The catalog's account of the race, which every answer agrees with + // where the request saw one. + let registered = registered_targets(&fx.engine, &targets).await; + for (target, result) in targets.iter().zip(&restores) { + match result { + Ok(_) => assert!( + registered.contains(target), + "round {round}: {target} was accepted but has no target" + ), + Err(StorageError::BackupInUse(_)) => assert!( + !registered.contains(target), + "round {round}: {target} was refused but has a target" + ), + Err(StorageError::Validation(m)) if m.starts_with("Backup not found") => { + assert!( + !registered.contains(target), + "round {round}: {target} was refused but has a target" + ); + } + Err(StorageError::Internal(m)) if m.contains("was not registered within") => { + eprintln!( + "round {round}: {target}: the request outlived the registration bound" + ); + } + Err(other) => panic!("round {round}: {target}: {other}"), + } + } + match delete { + Ok(desc) => { + delete_won += 1; + assert_eq!(desc.backup_details.backup_status, "DELETED"); + assert!( + registered.is_empty(), + "round {round}: a target was registered after the delete won: \ + {registered:?} {restores:?}" + ); + for (target, result) in targets.iter().zip(&restores) { + assert!( + matches!(result, Err(StorageError::Validation(m)) if m.starts_with("Backup not found")), + "round {round}: {target}: {result:?}" + ); + } + } + Err(StorageError::BackupInUse(_)) => { + registration_won += 1; + assert_eq!( + registered.len(), + 1, + "round {round}: exactly one restore registers: {registered:?} {restores:?}" + ); + for (target, result) in targets.iter().zip(&restores) { + assert!( + !matches!(result, Err(StorageError::Validation(_))), + "round {round}: {target}: a registration won, so no restore saw the \ + backup gone: {result:?}" + ); + } + copy_gate.add_permits(registered.len()); + for target in ®istered { + assert_fully_restored_with(&fx.engine, target, FEW).await; + } + fx.engine + .delete(ACCOUNT, &backup.backup_arn) + .await + .expect("DeleteBackup once every target is ACTIVE"); + } + Err(other) => panic!("round {round}: the delete failed: {other}"), + } + assert_eq!( + backup_meta(&fx.engine, &backup.backup_arn) + .await + .get_str("backup_status"), + Ok("DELETED") + ); + // The registration's own check, whichever order the race took: a + // deleted backup is not there to restore from, and nothing is created. + let late = format!("r{round}_late"); + let mut request = restore_request(&fx.engine, &b_arn_for_request, &late).await; + request.backup_arn = backup.backup_arn.clone(); + let refused = fx + .engine + .restore(ACCOUNT, request, JobPermit::detached()) + .await; + assert!( + matches!(&refused, Err(StorageError::Validation(m)) if m.starts_with("Backup not found")), + "round {round}: {refused:?}" + ); + assert!(table_doc(&fx.engine, &late).await.is_none()); + } + eprintln!( + "the delete won {delete_won} and a registration won {registration_won} of {ROUNDS} rounds" + ); + assert!( + delete_won >= 1 && registration_won >= 1, + "both orders occur: the delete won {delete_won}, a registration won {registration_won}" + ); + fx.finish().await; +} - let faulty = with_drop_fault(&fx.engine, AfterBackupDrop::Fail); - let failed = faulty.sweep_stale_backup_claims().await; +/// (a) The response is CREATING while the copy has not run; the target then +/// reaches ACTIVE with every item and index entry; the summary carries the +/// source table ARN and the backup's creation time. +#[tokio::test(flavor = "multi_thread")] +#[ignore = "needs a MongoDB replica set in EXTENDDB_TEST_MONGODB_URI"] +async fn live_restore_returns_creating_and_the_detached_copy_finishes() { + let fx = Fixture::new().await; + let source_arn = source_table(&fx.engine, "src").await; + let backup = fx + .engine + .create(ACCOUNT, "src", "b", JobPermit::detached()) + .await + .expect("backup"); + let request = restore_request(&fx.engine, &backup.backup_arn, "dst").await; + let (gated, gate) = gated(&fx.engine); + + let desc = tokio::time::timeout( + Duration::from_secs(20), + gated.restore(ACCOUNT, request, JobPermit::detached()), + ) + .await + .expect("the restore must return while its copy is held") + .expect("restore"); + assert_eq!(desc.table_status, TableStatus::Creating); + let doc = table_doc(&fx.engine, "dst").await.expect("target"); + assert_eq!(doc.get_str("table_status"), Ok("CREATING")); + assert_eq!( + count(&fx.engine, &data_collection_name(&desc.table_id)).await, + 0, + "no item may be copied before the response" + ); + let summary = fx + .engine + .describe_table( + ACCOUNT, + DescribeTableInput { + table_name: "dst".to_owned(), + }, + ) + .await + .expect("describe") + .restore_summary + .expect("restore summary while CREATING"); + assert_eq!( + summary.source_backup_arn.as_deref(), + Some(backup.backup_arn.as_str()) + ); + assert_eq!( + summary.source_table_arn.as_deref(), + Some(source_arn.as_str()) + ); + assert_eq!(summary.restore_date_time, backup.backup_creation_date_time); + assert!(summary.restore_in_progress); + // The backup time is the shared id's instant, kept to the millisecond. + assert_eq!( + Some(backup.backup_creation_date_time), + extenddb_storage::util::backup_creation_date_time(&backup.backup_arn) + ); + + gate.add_permits(1); + assert_fully_restored(&fx.engine, "dst").await; + fx.finish().await; +} + +/// (b) A request future dropped after its first poll still produces a +/// complete restore. +#[tokio::test(flavor = "multi_thread")] +#[ignore = "needs a MongoDB replica set in EXTENDDB_TEST_MONGODB_URI"] +async fn live_a_dropped_restore_request_still_reaches_active() { + let fx = Fixture::new().await; + source_table(&fx.engine, "src").await; + let backup = fx + .engine + .create(ACCOUNT, "src", "b", JobPermit::detached()) + .await + .expect("backup"); + let request = restore_request(&fx.engine, &backup.backup_arn, "dst").await; + let mut future = fx.engine.restore(ACCOUNT, request, JobPermit::detached()); assert!( - matches!(&failed, Err(StorageError::Internal(_))), - "{failed:?}" - ); - let meta = backup_meta(&fx.engine, &backup.backup_arn).await; - assert_ne!( - meta.get_str(DELETE_CLAIM), - Ok("dead-delete"), - "the sweep took the claim over before dropping" + (&mut future).now_or_never().is_none(), + "the first poll must not complete the restore" ); - assert_claim_held_then_swept(&fx.engine, &backup.backup_arn, &backup_id).await; + drop(future); + assert_fully_restored(&fx.engine, "dst").await; fx.finish().await; } -/// The sweep leaves a stale delete claim alone while a restore target of the -/// backup is CREATING (the DeleteBackup died between its claim and its -/// recheck), and finishes the delete once the restore is ACTIVE. +/// (e) While the copy runs, DeleteBackup of the source is BackupInUse and +/// DeleteTable of the target is refused; both succeed once it is ACTIVE. #[tokio::test(flavor = "multi_thread")] #[ignore = "needs a MongoDB replica set in EXTENDDB_TEST_MONGODB_URI"] -async fn live_a_stale_delete_claim_waits_for_a_running_restore() { +async fn live_backup_and_target_are_protected_while_the_copy_runs() { let fx = Fixture::new().await; - fx.worker.abort(); source_table(&fx.engine, "src").await; let backup = fx .engine - .create_backup(ACCOUNT, "src", "b") + .create(ACCOUNT, "src", "b", JobPermit::detached()) .await .expect("backup"); let request = restore_request(&fx.engine, &backup.backup_arn, "dst").await; let (gated, gate) = gated(&fx.engine); gated - .restore_table_from_backup(ACCOUNT, request) + .restore(ACCOUNT, request, JobPermit::detached()) .await .expect("restore"); - let mut set = doc! { DELETE_CLAIM: "dead-delete" }; - set.insert( - DELETE_CLAIM_HEARTBEAT, - bson::DateTime::from_millis(bson::DateTime::now().timestamp_millis() - 600_000), - ); - fx.engine - .catalog_db - .collection::("backups") - .update_one(doc! { "_id": &backup.backup_arn }, doc! { "$set": set }) - .await - .expect("stale delete claim"); - for _ in 0..3 { - let sweep = fx.engine.sweep_stale_backup_claims().await.expect("sweep"); - assert!(sweep.is_empty(), "{sweep}"); - tokio::time::sleep(Duration::from_millis(300)).await; - } - let meta = backup_meta(&fx.engine, &backup.backup_arn).await; - assert_eq!( - meta.get_str(DELETE_CLAIM), - Ok("dead-delete"), - "the claim must be left as it was: {meta:?}" + let in_use = fx.engine.delete(ACCOUNT, &backup.backup_arn).await; + assert!( + matches!(in_use, Err(StorageError::BackupInUse(_))), + "the source backup is in use while the copy runs: {in_use:?}" ); - - let worker = spawn_worker(&fx.engine); - gate.add_permits(1); - assert_fully_restored(&fx.engine, "dst").await; - let sweep = fx.engine.sweep_stale_backup_claims().await.expect("sweep"); - assert_eq!( - sweep.delete_claims_finished, - std::slice::from_ref(&backup.backup_arn) + let refused = fx + .engine + .delete_table( + ACCOUNT, + DeleteTableInput { + table_name: "dst".to_owned(), + }, + ) + .await; + assert!( + matches!(&refused, Err(StorageError::IndexesInUse(m)) if m.contains("being restored")), + "DeleteTable of a target being restored is refused: {refused:?}" ); assert_eq!( - backup_meta(&fx.engine, &backup.backup_arn) + table_doc(&fx.engine, "dst") .await - .get_str("backup_status"), - Ok("DELETED") + .expect("target") + .get_str("table_status"), + Ok("CREATING"), + "both calls ran inside the copy window" ); - worker.abort(); + + gate.add_permits(1); + assert_fully_restored(&fx.engine, "dst").await; + fx.engine + .delete_table( + ACCOUNT, + DeleteTableInput { + table_name: "dst".to_owned(), + }, + ) + .await + .expect("DeleteTable after ACTIVE"); + let deleted = fx + .engine + .delete(ACCOUNT, &backup.backup_arn) + .await + .expect("DeleteBackup after ACTIVE"); + assert_eq!(deleted.backup_details.backup_status, "DELETED"); fx.finish().await; } -/// The first heartbeat refresh happens at once, not one interval later. +/// The definition captured at CreateBackup carries TTL and the stream +/// specification, and ListBackups reports the source table id. #[tokio::test(flavor = "multi_thread")] #[ignore = "needs a MongoDB replica set in EXTENDDB_TEST_MONGODB_URI"] -async fn live_the_heartbeat_refreshes_at_once() { +async fn live_create_backup_captures_ttl_stream_and_table_id() { + use extenddb_storage::MetadataEngine; let fx = Fixture::new().await; - fx.worker.abort(); source_table(&fx.engine, "src").await; - let backup = fx - .engine - .create_backup(ACCOUNT, "src", "b") - .await - .expect("backup"); - let long_ago = bson::DateTime::from_millis(bson::DateTime::now().timestamp_millis() - 600_000); - let mut set = doc! { DELETE_CLAIM: "op" }; - set.insert(DELETE_CLAIM_HEARTBEAT, long_ago); fx.engine - .catalog_db - .collection::("backups") - .update_one(doc! { "_id": &backup.backup_arn }, doc! { "$set": set }) + .update_ttl(ACCOUNT, "src", "expires", true) .await - .expect("claim"); - let mut slow = fx.engine.clone(); - slow.backup_liveness.heartbeat_interval = Duration::from_secs(3_600); - let _heartbeat = slow.start_backup_heartbeat("op", ACCOUNT, &backup.backup_arn, None); - let deadline = Instant::now() + Duration::from_secs(5); - loop { - let beat = *backup_meta(&fx.engine, &backup.backup_arn) + .expect("ttl"); + let table = table_doc(&fx.engine, "src").await.expect("src"); + let table_id = table.get_str("table_id").expect("id").to_owned(); + let mut backups = Vec::new(); + for i in 0..3 { + backups.push( + fx.engine + .create(ACCOUNT, "src", &format!("b{i}"), JobPermit::detached()) + .await + .expect("backup"), + ); + } + for b in &backups { + let source = fx + .engine + .restore_source(ACCOUNT, &b.backup_arn) .await - .get_datetime(DELETE_CLAIM_HEARTBEAT) - .expect("heartbeat"); - if beat > long_ago { - break; - } - assert!( - Instant::now() < deadline, - "the first refresh did not happen at once" + .expect("source"); + assert_eq!( + Some(source.backup_creation_date_time), + extenddb_storage::util::backup_creation_date_time(&b.backup_arn) ); - tokio::time::sleep(Duration::from_millis(50)).await; + assert_eq!( + source.backup_creation_date_time, + b.backup_creation_date_time + ); + let def = source.definition.expect("definition"); + let ttl = def.time_to_live.expect("ttl captured"); + assert_eq!(ttl.attribute_name.as_deref(), Some("expires")); + assert!(def.stream_specification.is_none()); + assert_eq!(def.global_secondary_indexes.len(), 1); + assert_eq!(def.local_secondary_indexes.len(), 1); + let described = fx + .engine + .describe(ACCOUNT, &b.backup_arn) + .await + .expect("describe"); + assert_eq!( + described.backup_details.backup_creation_date_time, + b.backup_creation_date_time + ); + assert!(described.source_table_feature_details.is_none()); } - fx.finish().await; -} - -/// A restore from a process whose clock is 10 minutes behind the server's -/// is not swept by another process (stale window 5 minutes), because the -/// target's first heartbeat is replaced with the server's clock at once. -#[tokio::test(flavor = "multi_thread")] -#[ignore = "needs a MongoDB replica set in EXTENDDB_TEST_MONGODB_URI"] -async fn live_a_restore_from_a_process_with_a_slow_clock_is_not_swept() { - let fx = Fixture::new().await; - fx.worker.abort(); - source_table(&fx.engine, "src").await; - let backup = fx + let listed = fx.engine.list(ACCOUNT, None).await.expect("list"); + assert_eq!(listed.len(), 3); + for summary in &listed { + assert_eq!(summary.table_id, table_id); + assert_eq!( + Some(summary.backup_creation_date_time), + extenddb_storage::util::backup_creation_date_time(&summary.backup_arn) + ); + } + let foreign = fx .engine - .create_backup(ACCOUNT, "src", "b") - .await - .expect("backup"); - let liveness = BackupLiveness { - heartbeat_interval: Duration::from_secs(3_600), - stale_after: Duration::from_secs(300), - slot_wait: Duration::from_secs(2), - copy_gate: None, - after_backup_drop: None, - local_clock_behind: Duration::from_secs(600), - }; - let behind = fx.second_instance(liveness.clone()).await; - let (behind, gate) = gated(&behind); - let other = fx - .second_instance(BackupLiveness { - local_clock_behind: Duration::ZERO, - ..liveness - }) + .restore_source("999999999999", &backups[0].backup_arn) .await; - let request = restore_request(&fx.engine, &backup.backup_arn, "dst").await; - behind - .restore_table_from_backup(ACCOUNT, request) - .await - .expect("restore"); - let beat = *table_doc(&fx.engine, "dst") - .await - .expect("target") - .get_datetime(TARGET_HEARTBEAT) - .expect("heartbeat"); - let age_ms = bson::DateTime::now().timestamp_millis() - beat.timestamp_millis(); assert!( - age_ms.abs() < 60_000, - "the target heartbeat is {age_ms} ms old: the server clock must replace the local one" + matches!(&foreign, Err(StorageError::Validation(m)) if m.starts_with("Backup not found")), + "{foreign:?}" ); - let sweep = other.sweep_stale_backup_claims().await.expect("sweep"); - assert!(sweep.is_empty(), "{sweep}"); - assert!(table_doc(&fx.engine, "dst").await.is_some()); - - let worker = spawn_worker(&other); - gate.add_permits(1); - assert_fully_restored(&fx.engine, "dst").await; - worker.abort(); fx.finish().await; } async fn table_doc_for(engine: &MongoEngine, account: &str, name: &str) -> Option { - engine - .catalog_db - .collection::("tables") + tables_majority(engine) .find_one(doc! { "_id": { "account_id": account, "table_name": name } }) .await .expect("read table") @@ -1258,7 +2370,8 @@ async fn wait_for_status_for(engine: &MongoEngine, account: &str, name: &str, st /// started at the same moment: exactly one is accepted and the other three /// are `BackupInUse`; a restore started while the accepted one is still /// CREATING (copy held) is `BackupInUse` as well; once the target is ACTIVE a -/// new restore of the backup is accepted. +/// new restore of the backup is accepted. Every refusal names the target the +/// backup is being restored into. #[tokio::test(flavor = "multi_thread")] #[ignore = "needs a MongoDB replica set in EXTENDDB_TEST_MONGODB_URI"] async fn live_a_second_restore_of_a_backup_while_one_is_creating_is_backup_in_use() { @@ -1267,7 +2380,7 @@ async fn live_a_second_restore_of_a_backup_while_one_is_creating_is_backup_in_us source_table_with(&fx.engine, "src", FEW).await; let backup = fx .engine - .create_backup(ACCOUNT, "src", "b") + .create(ACCOUNT, "src", "b", JobPermit::detached()) .await .expect("backup"); let (gated, gate) = gated(&fx.engine); @@ -1278,7 +2391,7 @@ async fn live_a_second_restore_of_a_backup_while_one_is_creating_is_backup_in_us let results = futures::future::join_all( requests .into_iter() - .map(|request| gated.restore_table_from_backup(ACCOUNT, request)), + .map(|request| gated.restore(ACCOUNT, request, JobPermit::detached())), ) .await; let accepted: Vec = results @@ -1295,8 +2408,7 @@ async fn live_a_second_restore_of_a_backup_while_one_is_creating_is_backup_in_us if i != accepted[0] { match result { Err(StorageError::BackupInUse(m)) => assert!( - m.starts_with("Backup is being used to restore table dst") - || m.starts_with("Backup is being used to restore another table: "), + m.starts_with("Backup is being used to restore table dst"), "restore {i}: {m}" ), other => panic!("restore {i} must be BackupInUse: {other:?}"), @@ -1314,7 +2426,11 @@ async fn live_a_second_restore_of_a_backup_while_one_is_creating_is_backup_in_us // Sequential, while the winner's copy is held: still in use, and the // refusal names the target with the wording the other backends use. let request = restore_request(&fx.engine, &backup.backup_arn, "late").await; - match fx.engine.restore_table_from_backup(ACCOUNT, request).await { + match fx + .engine + .restore(ACCOUNT, request, JobPermit::detached()) + .await + { Err(StorageError::BackupInUse(m)) => assert_eq!( m, format!( @@ -1325,16 +2441,12 @@ async fn live_a_second_restore_of_a_backup_while_one_is_creating_is_backup_in_us other => panic!("a restore during the CREATING window is BackupInUse: {other:?}"), } assert!(table_doc(&fx.engine, "late").await.is_none()); - // The refused restores left no claim behind: DeleteBackup is refused only - // because of the winner's CREATING target. - let meta = backup_meta(&fx.engine, &backup.backup_arn).await; - assert!(!meta.contains_key(RESTORE_CLAIM), "{meta:?}"); gate.add_permits(1); assert_fully_restored_with(&fx.engine, &winner, FEW).await; let request = restore_request(&fx.engine, &backup.backup_arn, "late").await; fx.engine - .restore_table_from_backup(ACCOUNT, request) + .restore(ACCOUNT, request, JobPermit::detached()) .await .expect("a restore after the first is ACTIVE is accepted"); assert_fully_restored_with(&fx.engine, "late", FEW).await; @@ -1352,22 +2464,26 @@ async fn live_a_restore_into_a_creating_restore_target_is_table_being_restored() source_table_with(&fx.engine, "src", FEW).await; let b1 = fx .engine - .create_backup(ACCOUNT, "src", "b1") + .create(ACCOUNT, "src", "b1", JobPermit::detached()) .await .expect("backup 1"); let b2 = fx .engine - .create_backup(ACCOUNT, "src", "b2") + .create(ACCOUNT, "src", "b2", JobPermit::detached()) .await .expect("backup 2"); let (gated, gate) = gated(&fx.engine); let request = restore_request(&fx.engine, &b1.backup_arn, "dst").await; gated - .restore_table_from_backup(ACCOUNT, request) + .restore(ACCOUNT, request, JobPermit::detached()) .await .expect("first restore"); let request = restore_request(&fx.engine, &b2.backup_arn, "dst").await; - match fx.engine.restore_table_from_backup(ACCOUNT, request).await { + match fx + .engine + .restore(ACCOUNT, request, JobPermit::detached()) + .await + { Err(StorageError::TableBeingRestored { table_name, backup_arn, @@ -1377,11 +2493,36 @@ async fn live_a_restore_into_a_creating_restore_target_is_table_being_restored() } other => panic!("restore into a CREATING restore target: {other:?}"), } + // What the engine's own name check reads before its job permit: the + // target describes as CREATING with its source backup in the + // RestoreSummary, which is what it answers TableInUse with. + let described = fx + .engine + .describe_table( + ACCOUNT, + DescribeTableInput { + table_name: "dst".to_owned(), + }, + ) + .await + .expect("describe the CREATING target"); + assert_eq!(described.table_status, TableStatus::Creating); + assert_eq!( + described + .restore_summary + .as_ref() + .and_then(|r| r.source_backup_arn.as_deref()), + Some(b1.backup_arn.as_str()), + "the CREATING target names its source backup" + ); // The same backup into the same name: the name is what is refused, not - // the backup (the service answers TableInUseException), and b1's claim - // is released. + // the backup (the service answers TableInUseException). let request = restore_request(&fx.engine, &b1.backup_arn, "dst").await; - match fx.engine.restore_table_from_backup(ACCOUNT, request).await { + match fx + .engine + .restore(ACCOUNT, request, JobPermit::detached()) + .await + { Err(StorageError::TableBeingRestored { table_name, backup_arn, @@ -1395,7 +2536,11 @@ async fn live_a_restore_into_a_creating_restore_target_is_table_being_restored() // restored into that name, is refused as the name too. let mut missing = restore_request(&fx.engine, &b2.backup_arn, "dst").await; missing.backup_arn = flip_last_hex(&b2.backup_arn); - match fx.engine.restore_table_from_backup(ACCOUNT, missing).await { + match fx + .engine + .restore(ACCOUNT, missing, JobPermit::detached()) + .await + { Err(StorageError::TableBeingRestored { table_name, backup_arn, @@ -1405,48 +2550,37 @@ async fn live_a_restore_into_a_creating_restore_target_is_table_being_restored() } other => panic!("a missing backup into a CREATING restore target: {other:?}"), } - let meta = backup_meta(&fx.engine, &b1.backup_arn).await; - assert!(!meta.contains_key(RESTORE_CLAIM), "{meta:?}"); - // b2 was not left claimed by the refused restore. - let meta = backup_meta(&fx.engine, &b2.backup_arn).await; - assert!(!meta.contains_key(RESTORE_CLAIM), "{meta:?}"); - // With every account slot held, a client retrying its own restore is - // still answered at once, not after the slot wait. - let account_slots = fx.engine.restore_account_slots.for_account(ACCOUNT); - let held = Arc::clone(&account_slots) - .acquire_many_owned(u32::try_from(account_slots.available_permits()).expect("u32")) - .await - .expect("hold every account slot"); - let request = restore_request(&fx.engine, &b1.backup_arn, "dst").await; - let started = Instant::now(); - let retried = fx.engine.restore_table_from_backup(ACCOUNT, request).await; - assert!( - started.elapsed() < fx.engine.backup_liveness.slot_wait, - "the retry waited for a slot: {:?}", - started.elapsed() - ); - assert!( - matches!(&retried, Err(StorageError::TableBeingRestored { table_name, .. }) if table_name == "dst"), - "{retried:?}" - ); - drop(held); gate.add_permits(1); assert_fully_restored_with(&fx.engine, "dst", FEW).await; let request = restore_request(&fx.engine, &b2.backup_arn, "dst").await; assert!( matches!( - fx.engine.restore_table_from_backup(ACCOUNT, request).await, + fx.engine + .restore(ACCOUNT, request, JobPermit::detached()) + .await, Err(StorageError::TableAlreadyExists(_)) ), "an ACTIVE table of that name is TableAlreadyExists" ); + // The former target describes as ACTIVE, so the engine's name check + // answers TableAlreadyExists for it too. + let former = fx + .engine + .describe_table( + ACCOUNT, + DescribeTableInput { + table_name: "dst".to_owned(), + }, + ) + .await + .expect("describe the former target"); + assert_eq!(former.table_status, TableStatus::Active); fx.finish().await; } -/// Twenty restores of twenty distinct backups (twenty accounts, so no -/// per-account bound and no shared claim serializes their registration) -/// against the driver's default pool of 10, with the server bound lifted, -/// all register, the engine answers a catalog read right after, and every +/// Twenty restores of twenty distinct backups (twenty accounts, so no shared +/// lock serializes their registration) against the driver's default pool of +/// 10 all register, the engine answers a catalog read right after, and every /// target reaches ACTIVE. Before the fix, the registration's hand-over left /// each creation's in-flight command holding a pooled connection while it /// waited for another, and this wedged the process for good. @@ -1456,16 +2590,13 @@ async fn live_twenty_concurrent_restores_of_distinct_backups_do_not_wedge_the_po const N: usize = 20; const FEW: usize = 20; let fx = Fixture::new().await; - let mut unbounded = fx.engine.clone(); - unbounded.restore_server_slot_count = 64; - unbounded.restore_server_slots = Arc::new(tokio::sync::Semaphore::new(64)); let accounts: Vec = (0..N).map(|i| format!("9100000000{i:02}")).collect(); let mut requests = Vec::new(); for account in &accounts { source_table_for(&fx.engine, account, "src", FEW).await; let backup = fx .engine - .create_backup(account, "src", "b") + .create(account, "src", "b", JobPermit::detached()) .await .expect("backup"); requests.push(( @@ -1475,11 +2606,11 @@ async fn live_twenty_concurrent_restores_of_distinct_backups_do_not_wedge_the_po } let started = Instant::now(); let results = futures::future::join_all(requests.into_iter().map(|(account, request)| { - let engine = unbounded.clone(); + let engine = fx.engine.clone(); async move { tokio::time::timeout( Duration::from_secs(60), - engine.restore_table_from_backup(&account, request), + engine.restore(&account, request, JobPermit::detached()), ) .await } @@ -1514,140 +2645,6 @@ async fn live_twenty_concurrent_restores_of_distinct_backups_do_not_wedge_the_po fx.finish().await; } -/// With the default bound (`min(16, pool/2)` = 5 for the default pool -/// of 10), seven restores of seven backups from two accounts (four and three, -/// under the per-account bound of four) started while copies are held: five are -/// admitted and two are `LimitExceeded` naming the server bound after the -/// slot wait, the engine still answers meanwhile, the five finish once the -/// copies are released, and their slots are then free again. -#[tokio::test(flavor = "multi_thread")] -#[ignore = "needs a MongoDB replica set in EXTENDDB_TEST_MONGODB_URI"] -async fn live_restores_beyond_the_server_bound_are_limit_exceeded_and_slots_are_freed() { - const FEW: usize = 20; - let fx = Fixture::new().await; - assert_eq!(fx.engine.restore_server_slot_count, 5); - let (gated, gate) = gated(&fx.engine); - let accounts = ["920000000001", "920000000002"]; - let mut requests = Vec::new(); - // One backup per restore: a second restore of one backup while the first - // is CREATING is BackupInUse, which is not what this test is about. - for (account, restores) in accounts.iter().zip([4, 3]) { - source_table_for(&fx.engine, account, "src", FEW).await; - for i in 0..restores { - let backup = fx - .engine - .create_backup(account, "src", &format!("b{i}")) - .await - .expect("backup"); - let target = format!("dst{i}"); - requests.push(( - (*account).to_owned(), - target.clone(), - restore_request_for(&fx.engine, account, &backup.backup_arn, &target).await, - )); - } - } - let started = Instant::now(); - let results = futures::future::join_all(requests.iter().map(|(account, _, request)| { - let engine = gated.clone(); - async move { - engine - .restore_table_from_backup(account, request.clone()) - .await - } - })) - .await; - let elapsed = started.elapsed(); - let admitted: Vec<&(String, String, RestoreRequest)> = requests - .iter() - .zip(&results) - .filter_map(|(r, result)| result.is_ok().then_some(r)) - .collect(); - assert_eq!(admitted.len(), 5, "five restores are admitted: {results:?}"); - for result in &results { - if let Err(e) = result { - match e { - StorageError::LimitExceeded(m) => { - assert!(m.contains("on this server (at most 5"), "{m}"); - } - other => panic!("a refused restore is LimitExceeded: {other:?}"), - } - } - } - assert!( - elapsed >= fx.engine.backup_liveness.slot_wait, - "the refused restores waited the slot wait: {elapsed:?}" - ); - // The two refused restores never claimed their backup or inserted a - // target: the refusal came before any catalog write. - for (i, (account, target, _)) in requests.iter().enumerate() { - let doc = table_doc_for(&fx.engine, account, target).await; - let is_admitted = admitted.iter().any(|(a, t, _)| a == account && t == target); - assert_eq!(doc.is_some(), is_admitted, "request {i}: {target}"); - } - // Liveness while five copies are held. - tokio::time::timeout( - Duration::from_secs(10), - fx.engine.describe_table( - accounts[0], - DescribeTableInput { - table_name: "src".to_owned(), - }, - ), - ) - .await - .expect("a catalog read while copies are held returns within 10 s") - .expect("describe"); - assert_eq!(fx.engine.restore_server_slots.available_permits(), 0); - - gate.add_permits(5); - for (account, target, _) in &admitted { - wait_for_status_for(&fx.engine, account, target, "ACTIVE").await; - } - // Slots are released when the copy task ends, shortly after hand-over. - let deadline = Instant::now() + Duration::from_secs(10); - while fx.engine.restore_server_slots.available_permits() < 5 { - assert!(Instant::now() < deadline, "the five slots are freed"); - tokio::time::sleep(Duration::from_millis(50)).await; - } - fx.finish().await; -} - -/// The refusal paths of the admission bound, without a server. -#[tokio::test] -async fn restore_permits_wait_then_limit_exceeded() { - let wait = Duration::from_millis(50); - let account = Arc::new(tokio::sync::Semaphore::new(4)); - let server = Arc::new(tokio::sync::Semaphore::new(1)); - let held = super::acquire_restore_permits(Arc::clone(&account), Arc::clone(&server), 1, wait) - .await - .expect("the first restore gets both slots"); - let other_account = Arc::new(tokio::sync::Semaphore::new(4)); - match super::acquire_restore_permits(Arc::clone(&other_account), Arc::clone(&server), 1, wait) - .await - { - Err(StorageError::LimitExceeded(m)) => { - assert!(m.contains("on this server (at most 1"), "{m}") - } - other => panic!("a full server bound must refuse: {:?}", other.err()), - } - assert_eq!(other_account.available_permits(), 4); - drop(held); - super::acquire_restore_permits(Arc::clone(&other_account), Arc::clone(&server), 1, wait) - .await - .expect("a freed server slot is granted"); - let full_account = Arc::new(tokio::sync::Semaphore::new(0)); - let roomy_server = Arc::new(tokio::sync::Semaphore::new(4)); - match super::acquire_restore_permits(full_account, Arc::clone(&roomy_server), 4, wait).await { - Err(StorageError::LimitExceeded(m)) => assert!(m.contains("for this account"), "{m}"), - other => panic!("a full account must refuse: {:?}", other.err()), - } - assert_eq!(roomy_server.available_permits(), 4); - assert_eq!(super::restore_server_slot_count(1), 1); - assert_eq!(super::restore_server_slot_count(10), 5); - assert_eq!(super::restore_server_slot_count(40), 16); -} - /// UpdateTable on a CREATING restore target is `TableNotActive`, for /// every kind of change (billing mode, deletion protection, a stream, a GSI), /// and leaves the target as the backup describes; once ACTIVE the same @@ -1664,13 +2661,13 @@ async fn live_update_table_on_a_creating_restore_target_is_refused() { source_table_with(&fx.engine, "src", FEW).await; let backup = fx .engine - .create_backup(ACCOUNT, "src", "b") + .create(ACCOUNT, "src", "b", JobPermit::detached()) .await .expect("backup"); let (gated, gate) = gated(&fx.engine); let request = restore_request(&fx.engine, &backup.backup_arn, "dst").await; gated - .restore_table_from_backup(ACCOUNT, request) + .restore(ACCOUNT, request, JobPermit::detached()) .await .expect("restore"); let before = table_doc(&fx.engine, "dst").await.expect("target"); @@ -1735,10 +2732,18 @@ async fn live_update_table_on_a_creating_restore_target_is_refused() { ); } let mut after = table_doc(&fx.engine, "dst").await.expect("target"); - // The restore's heartbeat refreshes this field on its own schedule. + // The restore's heartbeat refreshes the job record on its own schedule; + // the record stays RUNNING. + assert_eq!( + after + .get_document(RESTORE_JOB) + .expect("job") + .get_str("status"), + Ok("RUNNING") + ); let mut before = before; - before.remove(TARGET_HEARTBEAT); - after.remove(TARGET_HEARTBEAT); + before.remove(RESTORE_JOB); + after.remove(RESTORE_JOB); assert_eq!(after, before, "a refused UpdateTable changes nothing"); assert_eq!( index_ids(&fx.engine, before.get_str("table_id").expect("id")) diff --git a/crates/storage-mongodb/src/backup_protocol_model.rs b/crates/storage-mongodb/src/backup_protocol_model.rs index bada2d560..e7d546627 100644 --- a/crates/storage-mongodb/src/backup_protocol_model.rs +++ b/crates/storage-mongodb/src/backup_protocol_model.rs @@ -1,139 +1,94 @@ // Copyright 2026 ExtendDB contributors // SPDX-License-Identifier: Apache-2.0 -//! An exhaustive model of the restore / DeleteBackup claim protocol, with the -//! heartbeat sweep as an adversary. It is a model of the protocol, not a test -//! of the code: each step stands for one MongoDB operation in -//! `register_restore`, `copy_restore`, `delete_backup`, or -//! `sweep_stale_backup_claims`, except `RRelease`, and the model is kept in -//! step with them by hand. `RRelease` stands for up to three operations of -//! `register_restore` run as one atomic step: the claim release (an -//! `update_one`), the lost-claim recheck of the backup document (a `find_one`, -//! only when the release matched nothing), and the abort of the target -//! (`abort_backup_restore`, only when the recheck finds the backup gone or -//! delete-claimed). Interleavings between those three are not explored. +//! An exhaustive model of the legacy restore / DeleteBackup protocol, with +//! the reconciler as an adversary. It is a model of the protocol, not a test +//! of the code: each step stands for one MongoDB operation, or one +//! transaction, in `register_restore`, `copy_restore`, `delete` or the +//! reconciler's removal of a target, and the model is kept in step with them +//! by hand. //! //! Threads, each a fixed sequence of steps; every interleaving of them is //! checked: //! -//! - two restores of the same backup into different targets: claim (the second -//! restore retries once, as `register_restore` retries a claim held by another -//! restore), insert the CREATING target, release the claim (with the -//! lost-claim recheck), read the backup's data (`$out`, as one read), hand the -//! target over, and the backfill worker flipping it ACTIVE. The retry also -//! happens when the first claim found a delete claim, where the code ends the -//! restore with BackupInUse at once; that allows more interleavings than the -//! code has; -//! - DeleteBackup: target check, claim, recheck, drop the data, mark DELETED; +//! - two restores of the same backup into different targets: register (one +//! transaction: the backup must be AVAILABLE and no CREATING target of it +//! may exist, else the restore is refused), read the backup's data (`$out`, +//! as one read), hand the target over (the job record DONE), and the +//! backfill worker flipping it ACTIVE; +//! - DeleteBackup: the unlocked target check, the decision (one transaction: +//! AVAILABLE and no CREATING target, then DELETING), the drop of the data, +//! DELETED; //! - DeleteBackup stopping at any point (its request future dropped, or an -//! error returned): its claim is released only if it is still armed, and it -//! is disarmed before the drop; -//! - sweep events that act on a live owner whose heartbeat lapsed (a stalled -//! process): clearing the restore claim (twice), removing each CREATING -//! target that was not handed over; -//! - the sweep's delete takeover (take over a stale delete claim, check for a -//! restore target, drop, mark DELETED), and that takeover stopping at any -//! point. The sweep never releases a delete claim it took over. The code -//! also checks for a CREATING target before taking over and leaves the claim -//! alone when it finds one; the model has no such check and always takes -//! over, which allows more interleavings than the code has, so the model -//! does not exercise that check. +//! error returned, including an ambiguous error from the drop itself): +//! nothing is reverted; +//! - a retry of DeleteBackup, which resumes a DELETING backup: drops the data +//! again (idempotent) and writes DELETED; +//! - the reconciler removing each CREATING target that was not handed over, +//! as it does once the owner's heartbeat is stale; the step may fire at any +//! point, whether or not the owner is stalled, which over-approximates what +//! the reconciler can do. //! -//! Sweep events may fire at any point, whether or not the owner is stalled, -//! which over-approximates what the sweep can do. +//! Not modelled: `$out` as a non-atomic read, more than one restore of each +//! target, the timing of heartbeats (staleness is assumed possible at every +//! step), a failed write, and a registration's transaction retry (a write +//! conflict re-runs the registration, which the model's atomic step stands +//! for). //! -//! Not modelled: `$out` as a non-atomic read, more than one DeleteBackup, more -//! than one delete takeover, the timing of heartbeats (staleness is assumed -//! possible at every step), and a release that fails. In the model every -//! release succeeds. In the code: -//! -//! - DeleteBackup's `release()` after its recheck found a target can return an -//! error; the request then fails with the claim still armed, its `Drop` -//! spawns a second release, and if that fails too the claim stays until the -//! sweep finishes the delete, which it does only once no CREATING target of -//! the backup exists (so a DeleteBackup that returned an error can still -//! take effect later); -//! - a restore's release in `hand_claim_to_target` can return an error, and so -//! can the lost-claim recheck; the restore then aborts its target (if the -//! abort fails, the sweep removes the target once its heartbeat is stale) -//! and its still-armed claim is released by `Drop` or, if that fails, cleared -//! by the sweep; -//! - the releases on a restore's earlier failure paths (no `backup_id`, target -//! creation failed before the hand-over) log an error and leave the same -//! `Drop` / sweep backstop. -//! -//! Invariants, checked after every step: no restore claim and delete claim are -//! held together; no restore is handed over after reading dropped data; the -//! backup is never AVAILABLE with no claim after its data was dropped (a later -//! restore would copy nothing). Liveness: some interleaving completes a restore -//! and the delete, some completes both restores, and without stops and sweep -//! events no interleaving leaves a claim behind. +//! Invariants, checked after every step: no restore is handed over after +//! reading dropped data; the backup is never AVAILABLE after its data was +//! dropped (a later restore would copy nothing); a DELETING backup never +//! reads AVAILABLE again. Liveness: some interleaving completes a restore and +//! the delete, some completes both restores, and without a stop no +//! interleaving leaves the backup DELETING. //! //! The checker explores reachable states once each (depth first, memoised on //! the state and every thread's position) and counts the interleavings below //! each state, so the count is the number of interleavings covered without -//! enumerating them. A brute-force enumerator cross-checks it on smaller -//! configurations, including the 39,060 interleavings of the previous model. -//! -//! The four tests that explore the full model take about 25 s each in a debug -//! build and are `#[ignore]`d, so `cargo test` stays quick; run them with -//! `cargo test -p extenddb-storage-mongodb --lib protocol_model -- --include-ignored` -//! after any change to the claim protocol or to this model. The three quick -//! tests (the 20-ordering model, the brute-force cross-check, and the -//! no-stops liveness check) run by default. +//! enumerating them. A brute-force enumerator cross-checks it on a smaller +//! configuration. use std::collections::{HashMap, HashSet}; #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] enum Step { - // Restore `i` into target `i`. `RClaim(i, last)`: a failed claim ends the - // restore only when `last`. - RClaim(usize, bool), - RInsertTarget(usize), - RRelease(usize), + // Restore `i` into target `i`. + RRegister(usize), RCopyRead(usize), RHandOver(usize), RActivate(usize), // DeleteBackup DCheckTarget, - DClaim, - DRecheckTarget, + DDecide, DDropData, DMarkDeleted, - /// The DeleteBackup request stops: future dropped, or an error returned - /// (including an ambiguous error from the drop itself). + /// The DeleteBackup request stops: future dropped, or an error returned. DStop, - // Sweep acting on a stalled live owner - SClearRestoreClaim, + /// A retry of DeleteBackup finding the backup DELETING. + DResume, + /// The reconciler removes restore `i`'s target, judging its owner dead. SRemoveTarget(usize), - // Sweep step 3 - STakeOverDelete, - SCheckTarget, - SDropData, - SMarkDeleted, - /// The sweep's step 3 returns early (an error from any of its operations). - SStop, } #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] enum Target { None, Creating, - /// CREATING with `restore_backfill_pending`: still matched by - /// `restoring_target_filter`, no longer by the sweep. + /// CREATING with `restore_backfill_pending` and the job record DONE: + /// still in use, no longer the reconciler's. HandedOver, Active, } #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] -enum DeleteOwner { - Request, - Sweep, +enum Status { + Available, + Deleting, + Deleted, } #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] struct Restore { - claimed: bool, target: Target, copy_saw_data: Option, done: bool, @@ -141,39 +96,27 @@ struct Restore { #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] struct World { - available: bool, + status: Status, data_present: bool, - restore_claim: Option, - delete_claim: Option, restores: [Restore; 2], d_done: bool, - /// DeleteBackup's `BackupMetadataClaim` would release on drop/error. - d_armed: bool, - s_active: bool, - s_done: bool, - /// The sweep's taken-over claim would be released on an early return. - s_armed: bool, + /// Whether the DeleteBackup request has written DELETING. + d_decided: bool, } impl World { fn initial() -> Self { let restore = Restore { - claimed: false, target: Target::None, copy_saw_data: None, done: false, }; Self { - available: true, + status: Status::Available, data_present: true, - restore_claim: None, - delete_claim: None, restores: [restore; 2], d_done: false, - d_armed: false, - s_active: false, - s_done: false, - s_armed: false, + d_decided: false, } } @@ -185,49 +128,28 @@ impl World { } } -/// Earlier revisions of the code, to show the model notices each of them. +/// Earlier or weaker shapes of the protocol, to show the model notices each. #[derive(Clone, Copy, Debug, Default)] struct Variant { - /// B1 before the fix: DeleteBackup's claim (and the sweep's taken-over - /// claim) stays armed across the drop, so a stop after the drop releases it. - release_after_drop: bool, - /// The sweep before this fix: it released a taken-over delete claim when it - /// found a restore target, and its armed claim released on an early return. - sweep_releases: bool, - /// `register_restore` without its recheck after a lost claim. - no_lost_claim_recheck: bool, + /// The delete's decision without the in-use check under the lock: only + /// the unlocked first check guards a registered target. + decide_without_target_check: bool, + /// A registration that does not require the backup to be AVAILABLE. + register_without_status_check: bool, + /// A DeleteBackup that stops after its decision puts the backup back to + /// AVAILABLE (the shape of a claim released on error). + revert_on_stop: bool, } fn apply(w: &mut World, step: Step, v: Variant) -> Result<(), String> { match step { - Step::RClaim(i, last) => { - let r = &mut w.restores[i]; - if !r.done && !r.claimed { - if w.available && w.restore_claim.is_none() && w.delete_claim.is_none() { - w.restore_claim = Some(i); - r.claimed = true; - } else if last { - r.done = true; - } - } - } - Step::RInsertTarget(i) => { - if !w.restores[i].done { - w.restores[i].target = Target::Creating; - } - } - Step::RRelease(i) => { + Step::RRegister(i) => { if !w.restores[i].done { - if w.restore_claim == Some(i) { - w.restore_claim = None; - } else if !v.no_lost_claim_recheck && !(w.available && w.delete_claim.is_none()) { - // Claim lost and the backup is gone or being deleted: - // abort the target. - let r = &mut w.restores[i]; - if r.target == Target::Creating { - r.target = Target::None; - } - r.done = true; + let available = w.status == Status::Available || v.register_without_status_check; + if available && !w.restoring_target() { + w.restores[i].target = Target::Creating; + } else { + w.restores[i].done = true; } } } @@ -260,116 +182,62 @@ fn apply(w: &mut World, step: Step, v: Variant) -> Result<(), String> { w.d_done = true; } } - Step::DClaim => { + Step::DDecide => { if !w.d_done { - if w.available && w.restore_claim.is_none() && w.delete_claim.is_none() { - w.delete_claim = Some(DeleteOwner::Request); - w.d_armed = true; + let in_use = !v.decide_without_target_check && w.restoring_target(); + if w.status == Status::Available && !in_use { + w.status = Status::Deleting; + w.d_decided = true; } else { + // In use: the request ends here. Not AVAILABLE (a + // concurrent delete decided first): the code resumes that + // delete's drop, which the model gives its own step, + // `DResume`; this request's own steps end here. w.d_done = true; } } } - Step::DRecheckTarget => { - if !w.d_done && w.restoring_target() { - // release() matches the operation id. - if w.delete_claim == Some(DeleteOwner::Request) { - w.delete_claim = None; - } - w.d_armed = false; - w.d_done = true; - } - } Step::DDropData => { if !w.d_done { - if !v.release_after_drop { - w.d_armed = false; - } w.data_present = false; } } Step::DMarkDeleted => { if !w.d_done { - if w.delete_claim == Some(DeleteOwner::Request) { - w.available = false; - w.delete_claim = None; + if w.status == Status::Deleting { + w.status = Status::Deleted; } - w.d_armed = false; w.d_done = true; } } Step::DStop => { if !w.d_done { w.d_done = true; - if w.d_armed && w.delete_claim == Some(DeleteOwner::Request) { - w.delete_claim = None; - } - w.d_armed = false; - } - } - Step::SClearRestoreClaim => w.restore_claim = None, - Step::SRemoveTarget(i) => { - if w.restores[i].target == Target::Creating { - w.restores[i].target = Target::None; - } - } - Step::STakeOverDelete => { - if !w.s_done { - if w.delete_claim == Some(DeleteOwner::Request) { - w.delete_claim = Some(DeleteOwner::Sweep); - w.s_active = true; - w.s_armed = v.sweep_releases; - } else { - w.s_done = true; + if v.revert_on_stop && w.d_decided && w.status == Status::Deleting { + w.status = Status::Available; } } } - Step::SCheckTarget => { - if w.s_active && !w.s_done && w.restoring_target() { - if v.sweep_releases && w.delete_claim == Some(DeleteOwner::Sweep) { - w.delete_claim = None; - } - // Otherwise the claim is left for a later sweep. - w.s_armed = false; - w.s_done = true; - } - } - Step::SDropData => { - if w.s_active && !w.s_done { - if !v.release_after_drop { - w.s_armed = false; - } + Step::DResume => { + if w.status == Status::Deleting { w.data_present = false; + w.status = Status::Deleted; } } - Step::SMarkDeleted => { - if w.s_active && !w.s_done { - if w.delete_claim == Some(DeleteOwner::Sweep) { - w.available = false; - w.delete_claim = None; - } - w.s_armed = false; - w.s_done = true; - } - } - Step::SStop => { - if w.s_active && !w.s_done { - w.s_done = true; - if w.s_armed && w.delete_claim == Some(DeleteOwner::Sweep) { - w.delete_claim = None; - } - w.s_armed = false; + Step::SRemoveTarget(i) => { + if w.restores[i].target == Target::Creating { + w.restores[i].target = Target::None; } } } - if w.restore_claim.is_some() && w.delete_claim.is_some() { - return Err(format!("both claims held: {w:?}")); - } - if w.available && !w.data_present && w.delete_claim.is_none() { + if w.status == Status::Available && !w.data_present { return Err(format!( - "backup AVAILABLE and unclaimed after its data was dropped: {w:?}" + "backup AVAILABLE after its data was dropped: {w:?}" )); } + if w.d_decided && w.status == Status::Available { + return Err(format!("a DELETING backup read AVAILABLE again: {w:?}")); + } Ok(()) } @@ -485,67 +353,36 @@ fn multinomial(lengths: &[usize]) -> u128 { result } -/// The previous model's restore: no retry, no ACTIVE flip. -const RESTORE_0: &[Step] = &[ - Step::RClaim(0, true), - Step::RInsertTarget(0), - Step::RRelease(0), - Step::RCopyRead(0), - Step::RHandOver(0), -]; const RESTORE_A: &[Step] = &[ - Step::RClaim(0, true), - Step::RInsertTarget(0), - Step::RRelease(0), + Step::RRegister(0), Step::RCopyRead(0), Step::RHandOver(0), Step::RActivate(0), ]; const RESTORE_B: &[Step] = &[ - Step::RClaim(1, false), - Step::RClaim(1, true), - Step::RInsertTarget(1), - Step::RRelease(1), + Step::RRegister(1), Step::RCopyRead(1), Step::RHandOver(1), Step::RActivate(1), ]; const DELETE: &[Step] = &[ Step::DCheckTarget, - Step::DClaim, - Step::DRecheckTarget, + Step::DDecide, Step::DDropData, Step::DMarkDeleted, ]; -const SWEEP_DELETE: &[Step] = &[ - Step::STakeOverDelete, - Step::SCheckTarget, - Step::SDropData, - Step::SMarkDeleted, -]; -/// The previous model's sweep event sets. -const OLD_SWEEPS: &[&[Step]] = &[ - &[], - &[Step::SClearRestoreClaim], - &[Step::SRemoveTarget(0)], - &[Step::SClearRestoreClaim, Step::SRemoveTarget(0)], - &[Step::SRemoveTarget(0), Step::SClearRestoreClaim], -]; -/// Every thread: two restores, DeleteBackup and its stop, four sweep events, -/// the delete takeover and its stop. +/// Every thread: two restores, DeleteBackup, its stop, a retry, and the +/// reconciler's removal of each target. fn full_model() -> Vec<&'static [Step]> { vec![ RESTORE_A, RESTORE_B, DELETE, &[Step::DStop], - &[Step::SClearRestoreClaim], - &[Step::SClearRestoreClaim], + &[Step::DResume], &[Step::SRemoveTarget(0)], &[Step::SRemoveTarget(1)], - SWEEP_DELETE, - &[Step::SStop], ] } @@ -553,75 +390,65 @@ fn lengths(threads: &[&[Step]]) -> Vec { threads.iter().map(|t| t.len()).collect() } +/// A restore's registration and DeleteBackup's decision, as the two +/// transactions order: every interleaving of one restore and one delete, +/// without a stop or the reconciler, ends with the delete DELETED and the +/// restore refused, the restore ACTIVE and the delete refused, or the delete +/// DELETED after the restore reached ACTIVE; never a delete that proceeded +/// while the target was CREATING. #[test] -fn the_original_20_orderings_hold_without_a_sweep() { - // restore {claim, insert, release} x delete {claim, recheck, drop}. - let orders = interleavings(&[ - &[ - Step::RClaim(0, true), - Step::RInsertTarget(0), - Step::RRelease(0), - ], - &[Step::DClaim, Step::DRecheckTarget, Step::DDropData], - ]); - assert_eq!(orders.len(), 20); - for order in orders { - run(&order, Variant::default()).unwrap_or_else(|e| panic!("{order:?}: {e}")); +fn a_registration_and_a_delete_order_wholly_one_before_the_other() { + let threads: [&[Step]; 2] = [RESTORE_A, DELETE]; + let orders = interleavings(&threads); + assert_eq!(orders.len() as u128, multinomial(&[4, 4])); + let mut delete_won = 0; + let mut restore_won = 0; + let mut both = 0; + for order in &orders { + let w = run(order, Variant::default()).unwrap_or_else(|e| panic!("{order:?}: {e}")); + match (w.status, w.restores[0].target) { + (Status::Deleted, Target::None) => delete_won += 1, + (Status::Available, Target::Active) => restore_won += 1, + (Status::Deleted, Target::Active) => both += 1, + other => panic!("{order:?} ended in {other:?}"), + } } + assert!( + delete_won > 0 && restore_won > 0 && both > 0, + "{delete_won} / {restore_won} / {both}" + ); } #[test] fn the_explorer_agrees_with_brute_force_enumeration() { - // The previous model's 39,060 interleavings, checked both ways, for the - // fixed protocol and for a variant that the model must reject. for v in [ Variant::default(), Variant { - no_lost_claim_recheck: true, + decide_without_target_check: true, ..Variant::default() }, Variant { - release_after_drop: true, + revert_on_stop: true, ..Variant::default() }, ] { - let mut total = 0_u128; - for sweep in OLD_SWEEPS { - let threads = [RESTORE_0, DELETE, sweep]; - let explored = explore(&threads, v); - let orders = interleavings(&threads); - assert_eq!(orders.len() as u128, multinomial(&lengths(&threads))); - let brute_bad = orders.iter().filter(|o| run(o, v).is_err()).count(); - // The explorer stops a schedule at its first violation, so it - // counts exactly the schedules that never violate an invariant. - assert_eq!( - explored.interleavings, - (orders.len() - brute_bad) as u128, - "{v:?} {sweep:?}" - ); - assert_eq!( - explored.violations.is_empty(), - brute_bad == 0, - "{v:?} {sweep:?}" - ); - total += orders.len() as u128; - } - assert_eq!(total, 252 + 2 * 2772 + 2 * 16632); + let threads: [&[Step]; 4] = [RESTORE_A, DELETE, &[Step::DStop], &[Step::SRemoveTarget(0)]]; + let explored = explore(&threads, v); + let orders = interleavings(&threads); + assert_eq!(orders.len() as u128, multinomial(&lengths(&threads))); + let brute_bad = orders.iter().filter(|o| run(o, v).is_err()).count(); + // The explorer stops a schedule at its first violation, so it counts + // exactly the schedules that never violate an invariant. + assert_eq!( + explored.interleavings, + (orders.len() - brute_bad) as u128, + "{v:?}" + ); + assert_eq!(explored.violations.is_empty(), brute_bad == 0, "{v:?}"); } - // And with the DeleteBackup stop, where the B1 variant must fail both ways. - let threads: [&[Step]; 3] = [RESTORE_0, DELETE, &[Step::DStop]]; - let orders = interleavings(&threads); - let v = Variant { - release_after_drop: true, - ..Variant::default() - }; - assert!(orders.iter().any(|o| run(o, v).is_err())); - assert!(!explore(&threads, v).violations.is_empty()); - assert!(orders.iter().all(|o| run(o, Variant::default()).is_ok())); } #[test] -#[ignore = "explores the full model (about 25 s in a debug build); run with --include-ignored"] fn no_interleaving_of_the_full_model_breaks_the_protocol() { let threads = full_model(); let explored = explore(&threads, Variant::default()); @@ -631,14 +458,14 @@ fn no_interleaving_of_the_full_model_breaks_the_protocol() { explored.violations.len(), explored.violations[0] ); - // Every interleaving was covered: 28 steps in 10 threads. + // Every interleaving was covered: 16 steps in 7 threads. assert_eq!(explored.interleavings, multinomial(&lengths(&threads))); - assert_eq!(explored.interleavings, 29_173_282_778_255_616_000); + assert_eq!(explored.interleavings, 1_513_512_000); // Liveness: the protocol does make progress. let t = &explored.terminals; assert!( - t.iter() - .any(|w| !w.available && w.restores.iter().any(|r| r.target == Target::Active)), + t.iter().any(|w| w.status == Status::Deleted + && w.restores.iter().any(|r| r.target == Target::Active)), "no interleaving completes both a restore and the delete" ); assert!( @@ -649,86 +476,81 @@ fn no_interleaving_of_the_full_model_breaks_the_protocol() { assert!( t.iter().any(|w| w.restores[1].target == Target::Active && w.restores[0].target == Target::Active - && !w.available), + && w.status == Status::Deleted), "no interleaving completes both restores and then the delete" ); + // A DeleteBackup that stopped after writing DELETING leaves the backup + // DELETING only until a retry resumes it; with the retry in the model, a + // terminal state is DELETING only when the stop came after the retry. + assert!(t.iter().any(|w| w.status == Status::Deleting)); } #[test] -fn without_stops_or_sweeps_no_claim_is_left_behind() { - let threads: [&[Step]; 3] = [RESTORE_A, RESTORE_B, DELETE]; +fn without_a_stop_the_backup_is_never_left_deleting() { + let threads: [&[Step]; 5] = [ + RESTORE_A, + RESTORE_B, + DELETE, + &[Step::SRemoveTarget(0)], + &[Step::SRemoveTarget(1)], + ]; let explored = explore(&threads, Variant::default()); assert!(explored.violations.is_empty(), "{:?}", explored.violations); - assert_eq!(explored.interleavings, multinomial(&[6, 7, 5])); + assert_eq!(explored.interleavings, multinomial(&[4, 4, 4, 1, 1])); for w in &explored.terminals { - assert_eq!(w.restore_claim, None, "{w:?}"); - assert_eq!(w.delete_claim, None, "{w:?}"); - // Without a stall, a restore that claimed always completes. - for r in &w.restores { - assert!(!r.claimed || r.target == Target::Active, "{w:?}"); - } + assert_ne!(w.status, Status::Deleting, "{w:?}"); + // A delete that decided finished; one that did not left the backup + // AVAILABLE with its data. + assert_eq!(w.data_present, w.status == Status::Available, "{w:?}"); } - // The retry lets the second restore claim after the first released in - // some interleaving where its first attempt failed; without the retry - // step that schedule ends the restore. - assert!( - explored - .terminals - .iter() - .any(|w| w.restores.iter().all(|r| r.target == Target::Active)) - ); } #[test] -#[ignore = "explores the full model (about 25 s in a debug build); run with --include-ignored"] -fn a_delete_claim_released_after_the_drop_is_found() { - // The claim released by a stop after the drop. +fn a_delete_deciding_without_the_in_use_check_under_the_lock_is_found() { let explored = explore( &full_model(), Variant { - release_after_drop: true, + decide_without_target_check: true, ..Variant::default() }, ); assert!(!explored.violations.is_empty()); assert!( - explored.violations.iter().any(|e| e - .contains("AVAILABLE and unclaimed after its data was dropped") - && e.contains("DDropData, DStop")), + explored + .violations + .iter() + .any(|e| e.contains("dropped backup data")), "{}", explored.violations[0] ); } #[test] -#[ignore = "explores the full model (about 25 s in a debug build); run with --include-ignored"] -fn a_sweep_that_releases_a_taken_over_delete_claim_is_found() { - // The sweep before this fix: the stalled DeleteBackup it took over from - // can still drop the data after the sweep released the claim. +fn a_registration_without_the_status_check_is_found() { let explored = explore( &full_model(), Variant { - sweep_releases: true, + register_without_status_check: true, ..Variant::default() }, ); assert!(!explored.violations.is_empty()); assert!( - explored.violations.iter().all(|e| e - .contains("AVAILABLE and unclaimed after its data was dropped") - || e.contains("dropped backup data")), + explored + .violations + .iter() + .any(|e| e.contains("dropped backup data")), "{}", explored.violations[0] ); } #[test] -#[ignore = "explores the full model (about 25 s in a debug build); run with --include-ignored"] -fn the_lost_claim_recheck_is_what_keeps_a_stalled_restore_safe() { +fn a_delete_that_reverts_on_a_stop_is_found() { let explored = explore( &full_model(), Variant { - no_lost_claim_recheck: true, + revert_on_stop: true, ..Variant::default() }, ); @@ -737,21 +559,9 @@ fn the_lost_claim_recheck_is_what_keeps_a_stalled_restore_safe() { explored .violations .iter() - .any(|e| e.contains("dropped backup data")), + .any(|e| e.contains("AVAILABLE after its data was dropped") + && e.contains("DDropData, DStop")), "{}", explored.violations[0] ); - // Without a sweep (no lapse) the recheck never matters. - let threads: [&[Step]; 3] = [RESTORE_A, RESTORE_B, DELETE]; - assert!( - explore( - &threads, - Variant { - no_lost_claim_recheck: true, - ..Variant::default() - } - ) - .violations - .is_empty() - ); } diff --git a/crates/storage-mongodb/src/bootstrapper.rs b/crates/storage-mongodb/src/bootstrapper.rs index c37c3f36f..ac9839544 100644 --- a/crates/storage-mongodb/src/bootstrapper.rs +++ b/crates/storage-mongodb/src/bootstrapper.rs @@ -2,6 +2,12 @@ // SPDX-License-Identifier: Apache-2.0 //! Bootstrapper implementation for `MongoDB`. +//! +//! The catalog migrates through this bootstrapper, not the shared runner: +//! [`Bootstrapper::run_catalog_migrations`] runs its steps in order, each +//! idempotent and recorded in `schema_history`, and the last step writes +//! [`CATALOG_VERSION`], the one value `extenddb migrate` compares the stored +//! `catalog_version` with. use async_trait::async_trait; use bson::{Document, doc}; @@ -12,6 +18,196 @@ use extenddb_storage::bootstrapper::{AdminBootstrapResult, Bootstrapper}; use extenddb_storage::error::StorageError; use extenddb_storage::management_store::{OpError, OpResult}; +use crate::backup_catalog::{ + BACKUP_INDEX, BACKUP_MAINTENANCE, CATALOG_ID, RESTORE_JOB, RESTORE_JOB_HEARTBEAT, + RESTORE_JOB_STATUS, +}; + +/// The catalog version this build expects. The last step of +/// [`Bootstrapper::run_catalog_migrations`] writes it, `extenddb migrate` +/// compares the stored version with it, and `extenddb version` prints it. +pub(crate) const CATALOG_VERSION: &str = "0.0.5"; + +/// The version the first step seeds on a fresh catalog; the steps after it +/// bring the catalog to [`CATALOG_VERSION`]. +const INITIAL_CATALOG_VERSION: &str = "0.0.2"; + +/// `serve`'s catalog guard: the stored `catalog_version` must be +/// [`CATALOG_VERSION`], else the server refuses to start with the command +/// that brings the catalog forward (`extenddb migrate --yes`). The rule is +/// [`extenddb_storage::util::require_served_catalog_version`], the one the +/// Cassandra factory applies too; the PostgreSQL and SQLite factories read +/// their ledgers for the same refusal. A server on an older catalog would +/// serve table and item operations and fail every backup operation on the +/// collections and indexes the missing step creates (a `claim_backup` upsert +/// without the unique index would not refuse a second claim). +/// +/// # Errors +/// +/// A read failure of `settings` as `Store`; a version other than this +/// binary's as the refusal `serve` prints. +pub(crate) async fn require_served_catalog( + catalog_db: &mongodb::Database, +) -> Result<(), extenddb_storage::migrations::MigrationError> { + use extenddb_storage::migrations::MigrationError; + let stored = catalog_db + .collection::("settings") + .find_one(doc! { "_id": "catalog_version" }) + .await + .map_err(|e| MigrationError::Store(format!("read catalog_version: {e}")))? + .and_then(|d| d.get_str("value").ok().map(str::to_owned)); + extenddb_storage::util::require_served_catalog_version( + stored.as_deref(), + CATALOG_VERSION, + "MongoDB", + ) +} + +/// What the backup catalog index step did, for the log and the tests. +#[derive(Debug, Default, Clone, PartialEq, Eq)] +pub(crate) struct BackupCatalogIndexStep { + /// Restore registrations that gained a job record. + pub(crate) registrations_backfilled: u64, + /// Whether the catalog id was written by this run (false when it existed). + pub(crate) catalog_id_written: bool, +} + +/// The catalog migration step that adds the backup catalog index: the +/// `backup_index` collection and its indexes, the job record on the restore +/// registrations that exist, the `backup_maintenance` collection, the +/// `catalog_id` settings document, and the version. Every statement is +/// idempotent and the backfill touches only registrations without a job +/// record, so a second run changes nothing. +/// +/// The backfill: every registration that exists was written by a release +/// without the job record, so each is a legacy restore. One whose table is +/// not CREATING, or whose copy had finished and was handed to the backfill +/// worker (`restore_backfill_pending`), is DONE, completed when the table was +/// created. One whose table is CREATING with no hand-over is a restore the +/// upgrade's stop interrupted: RUNNING, with a heartbeat at the epoch (not +/// absent, so the staleness test selects it) and an owner generation no job +/// holds (0), so the first reconciler pass fails it and removes its target. +/// The owner and heartbeat fields of the earlier registration shape are +/// removed; the job record replaces them. +pub(crate) async fn apply_backup_catalog_index( + db: &mongodb::Database, +) -> OpResult { + fn index_err(what: &'static str) -> impl Fn(mongodb::error::Error) -> OpError { + move |e| OpError::Internal(format!("{what}: {e}")) + } + let unique = || IndexOptions::builder().unique(true).build(); + + for coll_name in [BACKUP_INDEX, BACKUP_MAINTENANCE] { + // Idempotent on the versions this backend supports. + let _ = db.create_collection(coll_name).await; + } + let backup_index = db.collection::(BACKUP_INDEX); + backup_index + .create_index( + IndexModel::builder() + .keys(doc! { "account_id": 1, "backup_id": 1 }) + .options(unique()) + .build(), + ) + .await + .map_err(index_err("backup_index (account_id, backup_id) index"))?; + backup_index + .create_index( + IndexModel::builder() + .keys(doc! { "backup_id": 1 }) + .options(unique()) + .build(), + ) + .await + .map_err(index_err("backup_index backup_id index"))?; + backup_index + .create_index( + IndexModel::builder() + .keys(doc! { "account_id": 1, "table_name": 1, "created_at": 1 }) + .build(), + ) + .await + .map_err(index_err("backup_index account and table index"))?; + backup_index + .create_index( + IndexModel::builder() + .keys(doc! { "store_kind": 1, "store_root": 1, "store_id": 1, "status": 1 }) + .build(), + ) + .await + .map_err(index_err("backup_index store and status index"))?; + + let tables = db.collection::("tables"); + tables + .create_index( + IndexModel::builder() + .keys(doc! { RESTORE_JOB_STATUS: 1, RESTORE_JOB_HEARTBEAT: 1 }) + .build(), + ) + .await + .map_err(index_err("tables restore job index"))?; + + let interrupted = doc! { "$and": [ + { "$eq": ["$table_status", "CREATING"] }, + { "$ne": ["$restore_backfill_pending", true] }, + ] }; + let backfilled = tables + .update_many( + doc! { + "restore_source_backup_arn": { "$exists": true }, + RESTORE_JOB: { "$exists": false }, + }, + vec![ + doc! { "$set": { RESTORE_JOB: { + "status": { "$cond": [interrupted.clone(), "RUNNING", "DONE"] }, + "kind": "legacy", + "owner_generation": 0_i64, + "heartbeat_at": bson::DateTime::from_millis(0), + "completed_at": { "$cond": [interrupted, "$$REMOVE", "$creation_date_time"] }, + } } }, + doc! { "$unset": ["restore_operation_id", "restore_heartbeat_at"] }, + ], + ) + .await + .map_err(index_err("backfill the restore job records"))? + .modified_count; + + let settings = db.collection::("settings"); + let catalog_id_written = settings + .update_one( + doc! { "_id": CATALOG_ID }, + doc! { "$setOnInsert": { "value": uuid::Uuid::new_v4().to_string() } }, + ) + .upsert(true) + .await + .map_err(index_err("write catalog_id"))? + .upserted_id + .is_some(); + + settings + .update_one( + doc! { "_id": "catalog_version" }, + doc! { "$set": { "value": CATALOG_VERSION } }, + ) + .upsert(true) + .await + .map_err(index_err("write catalog_version"))?; + + // Ignore E11000 (already recorded). + let _ = db + .collection::("schema_history") + .insert_one(doc! { + "_id": "002_backup_catalog_index", + "applied_at": bson::DateTime::now(), + }) + .await; + + Ok(BackupCatalogIndexStep { + registrations_backfilled: backfilled, + catalog_id_written, + }) +} + /// `MongoDB` bootstrapper for init/destroy/migrate operations. pub struct MongoBootstrapper { client: mongodb::Client, @@ -342,12 +538,12 @@ impl Bootstrapper for MongoBootstrapper { .await .map_err(|e| OpError::Internal(format!("backups index: {e}")))?; - // Seed catalog_version in settings + // Seed catalog_version in settings; the later steps move it on. let settings = db.collection::("settings"); let _ = settings .update_one( doc! { "_id": "catalog_version" }, - doc! { "$setOnInsert": { "value": "0.0.2" } }, + doc! { "$setOnInsert": { "value": INITIAL_CATALOG_VERSION } }, ) .upsert(true) .await; @@ -373,6 +569,14 @@ impl Bootstrapper for MongoBootstrapper { }) .await; // Ignore E11000 (already applied) + let step = apply_backup_catalog_index(&db).await?; + if step.registrations_backfilled > 0 { + println!( + " Backfilled the job record on {} restore registration(s)", + step.registrations_backfilled + ); + } + Ok(()) } @@ -614,7 +818,7 @@ impl Bootstrapper for MongoBootstrapper { } fn expected_catalog_version(&self) -> String { - "0.0.2".to_string() + CATALOG_VERSION.to_string() } fn catalog_database_name(&self) -> String { diff --git a/crates/storage-mongodb/src/data_engine.rs b/crates/storage-mongodb/src/data_engine.rs index dd8f4cc9f..b20ccfd59 100644 --- a/crates/storage-mongodb/src/data_engine.rs +++ b/crates/storage-mongodb/src/data_engine.rs @@ -379,7 +379,7 @@ impl MongoEngine { transaction_options(self.transaction_read_concern(), TransactionKind::ReadOnly) } - fn write_transaction_options(&self) -> mongodb::options::TransactionOptions { + pub(crate) fn write_transaction_options(&self) -> mongodb::options::TransactionOptions { transaction_options(self.transaction_read_concern(), TransactionKind::Write) } @@ -3384,7 +3384,7 @@ impl MongoEngine { /// `gate_table` names the table for the unknown-commit test gate and /// is only passed by UpdateItem, the operation the gate is specified /// for. It is ignored in builds without the `test-hooks` feature. - async fn commit_write_transaction( + pub(crate) async fn commit_write_transaction( &self, session: &mut mongodb::ClientSession, gate_table: Option<&str>, @@ -3497,7 +3497,7 @@ pub(crate) struct GsiBackfillProgress { /// sustained hot-key contention; large enough to absorb ordinary /// snapshot-isolation aborts. Matches the OCC retry ceiling elsewhere /// in this file. -const TRANSIENT_RETRY_ATTEMPTS: u32 = 50; +pub(crate) const TRANSIENT_RETRY_ATTEMPTS: u32 = 50; /// Return type of `try_build_native_update`. Distinguishes an /// operator-document update (`{$set, $unset, $inc}`) from an @@ -3522,7 +3522,7 @@ enum NativeUpdate { /// Error signal used inside per-attempt transaction bodies. Lets the /// body use `?` for control flow while distinguishing "retry this /// whole transaction" from "return this error to the caller." -enum TxErr { +pub(crate) enum TxErr { Transient, Fatal(StorageError), } @@ -3557,7 +3557,7 @@ impl From for TxErr { /// UnknownTransactionCommitResult label, under which the commit may /// already have applied. Commit errors are classified by /// `classify_commit_error` instead. -fn is_transient_write_conflict(e: &mongodb::error::Error) -> bool { +pub(crate) fn is_transient_write_conflict(e: &mongodb::error::Error) -> bool { if e.contains_label(mongodb::error::TRANSIENT_TRANSACTION_ERROR) { return true; } @@ -3603,7 +3603,7 @@ fn classify_commit_error(e: &mongodb::error::Error) -> CommitErrorClass { } /// Result of `commit_write_transaction` for the caller's retry loop. -enum CommitOutcome { +pub(crate) enum CommitOutcome { /// The transaction committed. Return the attempt's success value. Committed, /// The transaction aborted without committing. The caller may re-run @@ -3615,7 +3615,7 @@ enum CommitOutcome { /// conditional-insert sites — a duplicate is the manifestation of a /// conditional-put race, so it must be surfaced as /// `ConditionalCheckFailedException` rather than a 500. -fn is_duplicate_key(e: &mongodb::error::Error) -> bool { +pub(crate) fn is_duplicate_key(e: &mongodb::error::Error) -> bool { matches!( *e.kind, mongodb::error::ErrorKind::Write(mongodb::error::WriteFailure::WriteError( @@ -3647,7 +3647,7 @@ fn synthetic_error_with_labels(message: &str, labels: &[&str]) -> mongodb::error /// Exponential-backoff sleep with random jitter. Used inside the OCC /// / WriteConflict retry loops so competing writers don't lock-step /// re-retry into the same conflict window. -async fn backoff_sleep(attempt: u32) { +pub(crate) async fn backoff_sleep(attempt: u32) { let base_us = 50u64.saturating_mul(1u64 << attempt.min(8)); let jitter = rand::random_range(0..=base_us); tokio::time::sleep(std::time::Duration::from_micros(jitter)).await; @@ -4181,9 +4181,6 @@ mod tests { gsi_cache: Default::default(), tx_read_concern: mongodb::options::ReadConcern::snapshot(), backup_liveness: Default::default(), - restore_server_slot_count: 1, - restore_server_slots: std::sync::Arc::new(tokio::sync::Semaphore::new(1)), - restore_account_slots: Default::default(), }; let base_coll = engine.data_db.collection::("base"); let key_schema = vec![KeySchemaElement { diff --git a/crates/storage-mongodb/src/lib.rs b/crates/storage-mongodb/src/lib.rs index 3d0c619c5..755dc0cf2 100644 --- a/crates/storage-mongodb/src/lib.rs +++ b/crates/storage-mongodb/src/lib.rs @@ -8,8 +8,8 @@ mod admin_store; mod authorization_store; +mod backup_catalog; mod backup_engine; -pub use backup_engine::BackupSweep; mod bootstrapper; mod catalog_store; pub mod condition; @@ -126,17 +126,6 @@ fn server_components_factory( details: e.to_string(), })?; - // A crash can leave a backup's metadata claimed by a restore or a - // delete that will never finish, or a restore target nothing will - // finish. The sweep removes only state whose heartbeat has stopped, so - // it is safe while another process shares this catalog; the GSI - // backfill worker repeats it periodically. - match engine.sweep_stale_backup_claims().await { - Ok(sweep) if sweep.is_empty() => {} - Ok(sweep) => tracing::warn!("startup backup sweep: {sweep}"), - Err(e) => tracing::error!("Failed to sweep stale backup claims: {e}"), - } - let engine = Arc::new(engine); // Create one shared catalog/management client. MongoDB Client clones @@ -155,6 +144,11 @@ fn server_components_factory( // panic in aes_gcm (`Key::from_slice` requires 32 bytes). Refuse to // start instead, matching the postgres backend. let catalog_db = catalog_client.database("extenddb_catalog"); + // The catalog's version is this binary's, or `serve` does not start: + // the refusal names `extenddb migrate --yes`, as on the ledger backends. + bootstrapper::require_served_catalog(&catalog_db) + .await + .map_err(BackendError::Catalog)?; let settings_coll = catalog_db.collection::("settings"); let enc_key = settings_coll .find_one(mongodb::bson::doc! { "_id": "encryption_key" }) @@ -357,17 +351,8 @@ pub struct MongoEngine { /// [`crate::config::MongoStorageConfig::transaction_read_concern`] for /// MongoDB-wire-compatible backends that don't support snapshot reads. tx_read_concern: mongodb::options::ReadConcern, - /// Heartbeat timing for backup claims and restore targets - /// (see [`backup_engine`]). + /// Heartbeat timing of a restore's job record (see [`backup_engine`]). pub(crate) backup_liveness: backup_engine::BackupLiveness, - /// Restores this process may run at once across accounts, and the - /// semaphore that enforces it; see - /// [`backup_engine::restore_server_slot_count`]. - pub(crate) restore_server_slot_count: usize, - pub(crate) restore_server_slots: Arc, - /// Restores each account may run at once in this process; see - /// [`backup_engine::RestoreAccountSlots`]. - pub(crate) restore_account_slots: backup_engine::RestoreAccountSlots, } /// Build a MongoDB client from a connection string, applying the shared @@ -435,22 +420,6 @@ async fn guarded_client_options( Ok(options) } -/// The connection pool size the driver will use for `options`: the explicit -/// `maxPoolSize`, else the driver's default of 10 (`mongodb` 3.x). The driver -/// does not enforce `waitQueueTimeoutMS` (its `ClientOptions::wait_queue_timeout` -/// documents it as not supported), so a checkout waits for as long as every -/// connection stays checked out; the work this backend admits at once is -/// therefore bounded from the pool size instead (see -/// [`backup_engine::restore_server_slot_count`]). -pub(crate) fn effective_pool_size(options: &mongodb::options::ClientOptions) -> u32 { - options - .max_pool_size - .unwrap_or(DRIVER_DEFAULT_MAX_POOL_SIZE) -} - -/// `mongodb::cmap::DEFAULT_MAX_POOL_SIZE`, which the driver does not export. -const DRIVER_DEFAULT_MAX_POOL_SIZE: u32 = 10; - async fn client_options_with_pool_size( connection_string: &str, max_pool_size: Option, @@ -471,9 +440,10 @@ fn apply_max_pool_size(options: &mut mongodb::options::ClientOptions, max_pool_s /// Refuse a connection-string write concern weaker than `majority`. /// -/// The backup claim protocol (DeleteBackup, restores and the stale-claim -/// sweep, see [`backup_engine`]) treats an acknowledged claim or CREATING -/// target as permanent. A write acknowledged under `w=1` (or any node count or +/// The backup protocol (DeleteBackup's DELETING write, a restore's CREATING +/// target and its job record, the catalog index's claims, see +/// [`backup_engine`] and [`backup_catalog`]) treats an acknowledged write as +/// permanent. A write acknowledged under `w=1` (or any node count or /// tag, whose relation to a majority this cannot see) can be rolled back when /// the primary fails over, and `journal=false` lets a majority acknowledge a /// write none of them has journaled. Accepted: no write concern (the server @@ -501,7 +471,7 @@ fn check_write_concern(wc: Option<&mongodb::options::WriteConcern>) -> Result<() None => Ok(()), Some(found) => Err(StorageError::Connection(format!( "MongoDB connection string must leave the write concern at the server \ - default or set w=majority (found {found}). Backup claims and restore \ + default or set w=majority (found {found}). Backup rows and restore \ targets assume an acknowledged write survives a primary failover, \ which a weaker write concern does not guarantee." ))), @@ -528,10 +498,11 @@ pub(crate) fn majority_write_concern( /// whatever the client's options or the server default say (inside a /// transaction the transaction's own options apply). /// -/// The backup claim protocol depends on this: DeleteBackup, a restore, and -/// the stale-claim sweep each check the other's marker or CREATING target -/// with a `find_one` after writing their own, and that check must see every -/// write the primary has acknowledged. A secondary can lag, and a `majority` +/// The backup protocol depends on this: DeleteBackup and a restore's +/// registration each check for the other's CREATING target or status write +/// (under their transactions, and DeleteBackup's unlocked first check +/// before), and the reconciler reads the job records' heartbeats; every such +/// read must see what the primary has acknowledged. A secondary can lag, and a `majority` /// or `snapshot` read can miss a write acknowledged with `w: 1`; a write /// acknowledged with `w: 1` can also be rolled back on failover, after the /// other side has acted on it. @@ -564,14 +535,12 @@ impl MongoEngine { tx_read_concern: mongodb::options::ReadConcern, ) -> Result { let options = guarded_client_options(connection_string, max_connections, true).await?; - let pool_size = effective_pool_size(&options); let client = mongodb::Client::with_options(options) .map_err(|e| StorageError::Connection(e.to_string()))?; let catalog_db = catalog_database(&client); let data_db = client.database("extenddb_data"); - let restore_server_slot_count = backup_engine::restore_server_slot_count(pool_size); Ok(Self { client, catalog_db, @@ -580,9 +549,6 @@ impl MongoEngine { gsi_cache: Arc::default(), tx_read_concern, backup_liveness: backup_engine::BackupLiveness::default(), - restore_server_slot_count, - restore_server_slots: Arc::new(tokio::sync::Semaphore::new(restore_server_slot_count)), - restore_account_slots: backup_engine::RestoreAccountSlots::default(), }) } @@ -647,28 +613,8 @@ impl MongoEngine { mod tests { use super::GsiCache; use super::client_options_with_pool_size; - use super::effective_pool_size; use std::time::Duration; - /// The restore bound is sized from the pool the driver will really use: - /// the URI's maxPoolSize, the configured override, or the driver default. - #[tokio::test] - async fn effective_pool_size_follows_uri_override_and_default() { - let uri = client_options_with_pool_size("mongodb://localhost:27017/?maxPoolSize=30", None) - .await - .expect("parse"); - assert_eq!(effective_pool_size(&uri), 30); - let configured = - client_options_with_pool_size("mongodb://localhost:27017/?maxPoolSize=30", Some(8)) - .await - .expect("parse"); - assert_eq!(effective_pool_size(&configured), 8); - let default = client_options_with_pool_size("mongodb://localhost:27017", None) - .await - .expect("parse"); - assert_eq!(effective_pool_size(&default), 10); - } - #[tokio::test] async fn absent_pool_override_preserves_uri_max_pool_size() { let options = diff --git a/crates/storage-mongodb/src/operations.rs b/crates/storage-mongodb/src/operations.rs index 4257fdbdf..ae899cd2e 100644 --- a/crates/storage-mongodb/src/operations.rs +++ b/crates/storage-mongodb/src/operations.rs @@ -109,9 +109,7 @@ impl OperationsEngine for MongoOperationsEngine { } fn catalog_version(&self) -> String { - // Mongo catalog version — matches the constant enforced by - // MongoBootstrapper::expected_catalog_version. - "0.0.2".to_owned() + crate::bootstrapper::CATALOG_VERSION.to_owned() } fn setting_default(&self, key: &str) -> Option { diff --git a/crates/storage-mongodb/src/snapshot_engine.rs b/crates/storage-mongodb/src/snapshot_engine.rs index c28c55397..75b744aad 100644 --- a/crates/storage-mongodb/src/snapshot_engine.rs +++ b/crates/storage-mongodb/src/snapshot_engine.rs @@ -470,7 +470,6 @@ mod tests { .insert_one(doc! { "_id": "control_plane_delay_seconds", "value": "0" }) .await .expect("setting"); - let slots = crate::backup_engine::restore_server_slot_count(10); let engine = MongoEngine { client: client.clone(), catalog_db, @@ -479,9 +478,6 @@ mod tests { gsi_cache: Arc::default(), tx_read_concern: ReadConcern::majority(), backup_liveness: crate::backup_engine::BackupLiveness::default(), - restore_server_slot_count: slots, - restore_server_slots: Arc::new(tokio::sync::Semaphore::new(slots)), - restore_account_slots: crate::backup_engine::RestoreAccountSlots::default(), }; let worker = tokio::spawn(crate::ttl_worker::gsi_backfill_worker(Arc::new( engine.clone(), diff --git a/crates/storage-mongodb/src/stream_records_live_tests.rs b/crates/storage-mongodb/src/stream_records_live_tests.rs index bf769836a..ec5956c7a 100644 --- a/crates/storage-mongodb/src/stream_records_live_tests.rs +++ b/crates/storage-mongodb/src/stream_records_live_tests.rs @@ -42,7 +42,6 @@ impl Fixture { async fn new() -> Self { let tag = uuid::Uuid::new_v4().simple().to_string(); let client = mongodb::Client::with_uri_str(uri()).await.expect("client"); - let slots = crate::backup_engine::restore_server_slot_count(10); let engine = MongoEngine { client: client.clone(), catalog_db: client.database(&format!("s{}_cat", &tag[..12])), @@ -51,9 +50,6 @@ impl Fixture { gsi_cache: Arc::default(), tx_read_concern: mongodb::options::ReadConcern::snapshot(), backup_liveness: crate::backup_engine::BackupLiveness::default(), - restore_server_slot_count: slots, - restore_server_slots: Arc::new(tokio::sync::Semaphore::new(slots)), - restore_account_slots: crate::backup_engine::RestoreAccountSlots::default(), }; Self { engine } } diff --git a/crates/storage-mongodb/src/table_engine.rs b/crates/storage-mongodb/src/table_engine.rs index 77706c3c7..e744e4b70 100644 --- a/crates/storage-mongodb/src/table_engine.rs +++ b/crates/storage-mongodb/src/table_engine.rs @@ -129,15 +129,21 @@ impl MongoEngine { .await } - /// Create a restore target with provenance present in its first catalog - /// document, so backup deletion cannot observe an ownerless target. + /// Create a restore target with its provenance and its RUNNING job + /// record in its first catalog document, so nothing ever observes an + /// unregistered or unowned target. When the registration carries a + /// session, the document is inserted in that session's transaction and + /// the transaction is committed right after the insert, before the + /// target's collections and indexes are created; a transaction the + /// server aborted for a write conflict is reported as `Transient`, with + /// nothing created, so the caller re-runs it. pub(crate) async fn create_table_for_restore( &self, account_id: &str, input: CreateTableInput, - provenance: &crate::backup_engine::RestoreProvenance<'_>, + registration: &mut crate::backup_engine::RestoreRegistration<'_>, ) -> Result { - self.create_table_impl_inner(account_id, input, true, Some(provenance)) + self.create_table_impl_inner(account_id, input, true, Some(registration)) .await } @@ -146,7 +152,7 @@ impl MongoEngine { account_id: &str, input: CreateTableInput, defer_active: bool, - restore: Option<&crate::backup_engine::RestoreProvenance<'_>>, + restore: Option<&mut crate::backup_engine::RestoreRegistration<'_>>, ) -> Result { Self::validate_account_id(account_id)?; @@ -255,28 +261,46 @@ impl MongoEngine { "sse_specification": sse_spec_bson, "on_demand_throughput": on_demand_bson, }; - if let Some(provenance) = restore { - provenance.insert_into(&mut table_doc); + if let Some(registration) = restore.as_deref() { + registration.insert_into(&mut table_doc); } let tables_coll = self.catalog_db.collection::("tables"); - if let Err(e) = tables_coll.insert_one(table_doc).await { - if !e.to_string().contains("E11000") { - return Err(StorageError::Internal(e.to_string())); + let for_restore = restore.is_some(); + let mut session = restore.and_then(|registration| registration.session.as_deref_mut()); + let inserted = match session.as_deref_mut() { + Some(session) => tables_coll.insert_one(table_doc).session(session).await, + None => tables_coll.insert_one(table_doc).await, + }; + if let Err(e) = inserted { + if let Some(session) = session { + let _ = session.abort_transaction().await; } - return Err(self - .name_taken_error( - &tables_coll, - account_id, - &input.table_name, - restore.is_some(), - ) - .await); + if crate::data_engine::is_duplicate_key(&e) { + return Err(self + .name_taken_error(&tables_coll, account_id, &input.table_name, for_restore) + .await); + } + if crate::data_engine::is_transient_write_conflict(&e) { + return Err(StorageError::Transient(format!( + "the registration of {} lost a write conflict: {e}", + input.table_name + ))); + } + return Err(StorageError::Internal(e.to_string())); } - // A restore releases its backup claim as soon as the target document, - // which carries the provenance, exists. - if let Some(inserted) = restore.and_then(|provenance| provenance.inserted) { - inserted.notify_one(); + // The registration's transaction ends here, before any collection is + // created, so an aborted one leaves nothing behind. + if let Some(session) = session { + match self.commit_write_transaction(session, None).await? { + crate::data_engine::CommitOutcome::Committed => {} + crate::data_engine::CommitOutcome::RerunBody => { + return Err(StorageError::Transient(format!( + "the registration of {} was aborted at commit", + input.table_name + ))); + } + } } // Create the data collection with appropriate indexes diff --git a/crates/storage-mongodb/src/ttl_worker.rs b/crates/storage-mongodb/src/ttl_worker.rs index e9cd41941..c8e54ef61 100644 --- a/crates/storage-mongodb/src/ttl_worker.rs +++ b/crates/storage-mongodb/src/ttl_worker.rs @@ -76,26 +76,13 @@ pub(crate) async fn stream_record_cleanup_worker(storage: Arc) { /// says the table has no indexes; an invalidated in-flight observation /// cannot publish that stale no-index result. /// -/// Every `heartbeat_interval` the loop also runs the backup sweep -/// ([`MongoEngine::sweep_stale_backup_claims`]), so a claim or restore target -/// left by a process that died is removed within about -/// `stale_after + heartbeat_interval + GSI_BACKFILL_INTERVAL` without a restart -/// of this one. The sweep removes only state whose heartbeat stopped, so it is -/// safe while other processes share the catalog. +/// A restore target whose copy stopped before its hand-over is not this +/// worker's: its job record keeps a stale heartbeat, and the backup +/// reconciler fails the record and removes the target. pub(crate) async fn gsi_backfill_worker(storage: Arc) { - let mut last_backup_sweep = std::time::Instant::now(); loop { tokio::time::sleep(GSI_BACKFILL_INTERVAL).await; - if last_backup_sweep.elapsed() >= storage.backup_liveness.heartbeat_interval { - last_backup_sweep = std::time::Instant::now(); - match storage.sweep_stale_backup_claims().await { - Ok(sweep) if sweep.is_empty() => {} - Ok(sweep) => tracing::warn!("periodic backup sweep: {sweep}"), - Err(e) => tracing::warn!("periodic backup sweep failed: {e}"), - } - } - let indexes_coll = storage.catalog_db.collection::("indexes"); let cursor = match indexes_coll .find(doc! { diff --git a/crates/storage-postgres/migrations/0005_backup_catalog_index.sql b/crates/storage-postgres/migrations/0005_backup_catalog_index.sql new file mode 100644 index 000000000..1f87febd4 --- /dev/null +++ b/crates/storage-postgres/migrations/0005_backup_catalog_index.sql @@ -0,0 +1,98 @@ +-- Copyright 2026 ExtendDB contributors +-- SPDX-License-Identifier: Apache-2.0 +-- +-- Migration 5, backup_catalog_index, PostgreSQL catalog database: the catalog +-- index of backups kept in a backup store, the restore job fields on the +-- restore registrations, the maintenance claim record and the catalog id. +-- Every statement is guarded, and the backfill touches only rows the new +-- columns are still empty on, so a second run changes nothing. + +-- One row per backup in a backup store: the fast index for DescribeBackup +-- and ListBackups, and the state machine record of its job. Keyed by +-- (account_id, backup_id); the backup id is unique on its own. The backups +-- earlier releases wrote into the catalog stay in `backups`, which the +-- legacy reader alone consults. `backup_arn` is `COLLATE "C"` so the +-- listings' `ORDER BY created_at, backup_arn` is byte order, the order the +-- engine merges and pages by, whatever the database's collation. +CREATE TABLE IF NOT EXISTS backup_index ( + account_id TEXT NOT NULL, + backup_id TEXT NOT NULL, + table_name TEXT NOT NULL, + backup_arn TEXT COLLATE "C" NOT NULL, + backup_name TEXT NOT NULL, + status TEXT NOT NULL, + store_kind TEXT NOT NULL, + store_root TEXT NOT NULL, + store_id TEXT NOT NULL, + size_bytes BIGINT NOT NULL DEFAULT 0, + item_count BIGINT NOT NULL DEFAULT 0, + created_at TIMESTAMPTZ NOT NULL, + completed_at TIMESTAMPTZ, + failure_reason TEXT, + source_table_details_json JSONB NOT NULL, + owner_generation BIGINT NOT NULL, + heartbeat_at TIMESTAMPTZ NOT NULL, + PRIMARY KEY (account_id, backup_id) +); + +CREATE UNIQUE INDEX IF NOT EXISTS idx_backup_index_backup_id ON backup_index (backup_id); +CREATE INDEX IF NOT EXISTS idx_backup_index_account_table ON backup_index (account_id, table_name); +CREATE INDEX IF NOT EXISTS idx_backup_index_store_status + ON backup_index (store_kind, store_root, store_id, status); + +-- The restore registration becomes the restore job record: its status +-- (RUNNING, FAILED, DONE), how it reads its backup (legacy or manifest), the +-- store a manifest restore reads, the job claim (owner_generation, +-- heartbeat_at), the spool host, per-batch progress, and how it ended. +ALTER TABLE table_restores ADD COLUMN IF NOT EXISTS status TEXT; +ALTER TABLE table_restores ADD COLUMN IF NOT EXISTS kind TEXT; +ALTER TABLE table_restores ADD COLUMN IF NOT EXISTS store_kind TEXT; +ALTER TABLE table_restores ADD COLUMN IF NOT EXISTS store_root TEXT; +ALTER TABLE table_restores ADD COLUMN IF NOT EXISTS store_id TEXT; +ALTER TABLE table_restores ADD COLUMN IF NOT EXISTS owner_generation BIGINT; +ALTER TABLE table_restores ADD COLUMN IF NOT EXISTS heartbeat_at TIMESTAMPTZ; +ALTER TABLE table_restores ADD COLUMN IF NOT EXISTS spool_host TEXT; +ALTER TABLE table_restores ADD COLUMN IF NOT EXISTS file_index BIGINT; +ALTER TABLE table_restores ADD COLUMN IF NOT EXISTS items_applied BIGINT; +ALTER TABLE table_restores ADD COLUMN IF NOT EXISTS failure_reason TEXT; +ALTER TABLE table_restores ADD COLUMN IF NOT EXISTS completed_at TIMESTAMPTZ; + +-- The registrations that exist were written by a release without the job +-- record: every one is a legacy restore. One whose table is not CREATING +-- finished (DONE). One whose table is CREATING is a restore the upgrade's +-- stop interrupted: RUNNING, with a heartbeat at the epoch (not NULL, so the +-- staleness test selects it) and an owner generation no job holds, so the +-- first reconciler pass fails it and removes its target. +UPDATE table_restores r +SET kind = 'legacy', + owner_generation = 0, + heartbeat_at = '1970-01-01T00:00:00Z', + status = CASE WHEN t.table_status = 'CREATING' THEN 'RUNNING' ELSE 'DONE' END, + completed_at = CASE WHEN t.table_status = 'CREATING' THEN NULL ELSE t.creation_date_time END +FROM tables t +WHERE t.table_id = r.table_id AND r.status IS NULL; + +ALTER TABLE table_restores ALTER COLUMN status SET NOT NULL; +ALTER TABLE table_restores ALTER COLUMN kind SET NOT NULL; +ALTER TABLE table_restores ALTER COLUMN owner_generation SET NOT NULL; +ALTER TABLE table_restores ALTER COLUMN heartbeat_at SET NOT NULL; + +CREATE INDEX IF NOT EXISTS idx_table_restores_status ON table_restores (status, heartbeat_at); + +-- The catalog-level maintenance claim: at most one row, held by the +-- maintenance command that changes rows or bytes jobs may be using. Empty +-- until a command takes it. +CREATE TABLE IF NOT EXISTS backup_maintenance ( + singleton BOOLEAN PRIMARY KEY DEFAULT TRUE CHECK (singleton), + holder TEXT NOT NULL, + owner_generation BIGINT NOT NULL, + claimed_at TIMESTAMPTZ NOT NULL DEFAULT now(), + heartbeat_at TIMESTAMPTZ NOT NULL DEFAULT now() +); + +-- The catalog's random id, which a backup store's identity file names so a +-- store can tell which catalog owns it. Written once, here, for a new and +-- an existing catalog alike. +INSERT INTO settings (key, value) +SELECT 'catalog_id', gen_random_uuid()::text +WHERE NOT EXISTS (SELECT 1 FROM settings WHERE key = 'catalog_id'); diff --git a/crates/storage-postgres/src/backup_catalog.rs b/crates/storage-postgres/src/backup_catalog.rs new file mode 100644 index 000000000..4477c823b --- /dev/null +++ b/crates/storage-postgres/src/backup_catalog.rs @@ -0,0 +1,698 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! The catalog index on `PostgreSQL`: `backup_index`, the restore job fields +//! of `table_restores`, and `backup_maintenance`. +//! +//! The claim of a backup job is `pg_try_advisory_lock(hashtextextended( +//! 'backup:' || backup_id, RESTORE_LOCK_SEED))` on a dedicated, non-pooled +//! connection to the catalog database (a [`ClaimSession`]), taken before the +//! row is inserted and held for the job's life; the backup job's heartbeats +//! and status writes go through that connection and carry the claim's +//! `owner_generation`, so a session the server dropped surfaces as a failed +//! heartbeat and a row a successor took over matches no write (the legacy +//! restore heartbeats through its session too, and makes its DONE and FAILED +//! writes on pooled connections, each conditioned on RUNNING and the +//! generation). The lock on a row for DeleteBackup and a restore's +//! registration is `FOR UPDATE` on the `backup_index` row. Staleness is +//! measured on the catalog's clock. +//! +//! A conditional write whose error is of the connection class (an I/O or TLS +//! error, a pool or worker failure, SQLSTATE class 08, or 57P01 to 57P03) +//! has an unknown outcome (`WriteOutcome::Unknown`: the statement may have +//! reached the server); a statement error, a `statement_timeout` included, +//! is `Err`. The heartbeat period is +//! [`extenddb_storage::backup_catalog::JOB_HEARTBEAT_INTERVAL`] (15 s), the +//! constant every backend runs at until `[backup] heartbeat_interval_secs` +//! reaches the jobs in a later release. + +use std::time::Duration; + +use extenddb_core::backup_format::BackupArn; +use extenddb_storage::backup_catalog::{ + BackupCatalog, BackupKey, BackupRow, BackupStatus, DeleteStart, Heartbeat, JobClaim, JobRow, + JobStatus, RestoreJobRow, RestoreKind, RestoreTargetKey, StatusFields, StoreIdentity, + StoreKind, TimeBounds, WriteOutcome, +}; +use extenddb_storage::error::StorageError; +use futures::future::BoxFuture; +use futures::stream::BoxStream; +use sqlx::PgExecutor; +use tokio::sync::Mutex; + +use crate::PostgresEngine; +use crate::backup_engine::{ClaimSession, backup_lock_key, generation_column, try_claim}; + +/// The name of a CREATING restore target registered from `backup_arn`, if +/// any: the one in-use rule DeleteBackup, a restore's registration and +/// `backup_in_use` share. Reads the target's status, not the job's. +pub(crate) async fn creating_restore_target<'e, E: PgExecutor<'e>>( + executor: E, + backup_arn: &str, +) -> Result, StorageError> { + sqlx::query_scalar( + "SELECT t.table_name FROM table_restores r JOIN tables t ON t.table_id = r.table_id \ + WHERE r.source_backup_arn = $1 AND t.table_status = 'CREATING' LIMIT 1", + ) + .bind(backup_arn) + .fetch_optional(executor) + .await + .map_err(db_err) +} + +/// The error for a backup a CREATING restore target still reads: the shared +/// text, [`extenddb_storage::backup_catalog::backup_in_use`]. +pub(crate) fn backup_in_use(table: &str, backup_arn: &str) -> StorageError { + extenddb_storage::backup_catalog::backup_in_use(table, backup_arn) +} + +fn db_err(e: sqlx::Error) -> StorageError { + StorageError::Internal(format!("Database error: {e}")) +} + +fn not_found(backup_arn: &str) -> StorageError { + extenddb_storage::backup_catalog::backup_not_found(backup_arn) +} + +/// The columns of one `backup_index` row, in the order every read selects +/// them. +const ROW_COLUMNS: &str = "backup_id, account_id, table_name, backup_arn, backup_name, status, \ + store_kind, store_root, store_id, size_bytes, item_count, \ + EXTRACT(EPOCH FROM created_at)::FLOAT8 AS created_at, \ + EXTRACT(EPOCH FROM completed_at)::FLOAT8 AS completed_at, \ + failure_reason, source_table_details_json, owner_generation, \ + EXTRACT(EPOCH FROM heartbeat_at)::FLOAT8 AS heartbeat_at"; + +/// One `backup_index` row as the driver reads it. +#[derive(sqlx::FromRow)] +struct RowTuple { + backup_id: String, + account_id: String, + table_name: String, + backup_arn: String, + backup_name: String, + status: String, + store_kind: String, + store_root: String, + store_id: String, + size_bytes: i64, + item_count: i64, + created_at: f64, + completed_at: Option, + failure_reason: Option, + source_table_details_json: serde_json::Value, + owner_generation: i64, + heartbeat_at: f64, +} + +fn row_from_tuple(t: RowTuple) -> Result { + let RowTuple { + backup_id, + account_id, + table_name, + backup_arn, + backup_name, + status, + store_kind, + store_root, + store_id, + size_bytes, + item_count, + created_at, + completed_at, + failure_reason, + source_table_details_json, + owner_generation, + heartbeat_at, + } = t; + let store_kind = StoreKind::parse(&store_kind).ok_or_else(|| { + StorageError::Internal(format!( + "backup {backup_arn} carries an unknown store kind {store_kind:?}" + )) + })?; + Ok(BackupRow { + backup_id, + account_id, + table_name, + backup_arn, + backup_name, + status: BackupStatus::parse(&status)?, + store: StoreIdentity { + store_kind, + store_root, + store_id, + }, + size_bytes, + item_count, + created_at, + completed_at, + failure_reason, + source_table_details_json, + owner_generation: u64::try_from(owner_generation).unwrap_or_default(), + heartbeat_at, + }) +} + +/// The rows `read` fetches, as a stream: the read runs once, on the first +/// poll, and a failed read is one `Err` item. +fn rows_stream<'a, F>(read: F) -> BoxStream<'a, Result> +where + F: std::future::Future, StorageError>> + Send + 'a, +{ + use futures::StreamExt; + Box::pin( + futures::stream::once(async move { + match read.await { + Ok(rows) => rows.into_iter().map(row_from_tuple).collect::>(), + Err(e) => vec![Err(e)], + } + }) + .flat_map(futures::stream::iter), + ) +} + +/// A backup job's handle on its `backup_index` row. +struct PostgresBackupClaim { + session: Mutex, + generation: u64, + key: BackupKey, + pool: sqlx::PgPool, +} + +/// The heartbeat of a backup job: refresh its row while it carries this +/// generation. +const BACKUP_HEARTBEAT: &str = "UPDATE backup_index SET heartbeat_at = now() \ + WHERE account_id = $1 AND backup_id = $2 AND owner_generation = $3"; + +/// Whether a write's outcome cannot be told from its error: the statement +/// may have reached the server before the connection failed. +fn outcome_unknown(e: &sqlx::Error) -> bool { + match e { + sqlx::Error::Io(_) + | sqlx::Error::Tls(_) + | sqlx::Error::PoolTimedOut + | sqlx::Error::PoolClosed + | sqlx::Error::WorkerCrashed => true, + sqlx::Error::Database(d) => d.code().is_some_and(|code| { + code.starts_with("08") || matches!(code.as_ref(), "57P01" | "57P02" | "57P03") + }), + _ => false, + } +} + +impl JobClaim for PostgresBackupClaim { + fn generation(&self) -> u64 { + self.generation + } + + fn heartbeat(&self) -> BoxFuture<'_, Result> { + Box::pin(async move { + let mut session = self.session.lock().await; + let confirmed = session + .confirm( + BACKUP_HEARTBEAT, + &[&self.key.account_id, &self.key.backup_id], + ) + .await + .map_err(|e| StorageError::Internal(format!("claim session: {e}")))?; + Ok(if confirmed { + Heartbeat::Confirmed + } else { + Heartbeat::Lost + }) + }) + } + + fn set_status<'a>( + &'a self, + from: JobStatus, + to: JobStatus, + fields: &'a StatusFields, + ) -> BoxFuture<'a, Result> { + Box::pin(async move { + let mut session = self.session.lock().await; + let written = sqlx::query( + "UPDATE backup_index SET status = $4, \ + size_bytes = COALESCE($5, size_bytes), \ + item_count = COALESCE($6, item_count), \ + failure_reason = COALESCE($7, failure_reason), \ + completed_at = CASE WHEN $8 THEN now() ELSE completed_at END \ + WHERE account_id = $1 AND backup_id = $2 AND owner_generation = $3 \ + AND status = $9", + ) + .bind(&self.key.account_id) + .bind(&self.key.backup_id) + .bind(generation_column(self.generation)) + .bind(to.as_str()) + .bind(fields.size_bytes) + .bind(fields.item_count) + .bind(fields.failure_reason.as_deref()) + .bind(fields.completed) + .bind(from.as_str()) + .execute(session.connection()) + .await; + match written { + Ok(result) if result.rows_affected() == 1 => Ok(WriteOutcome::Matched), + Ok(_) => Ok(WriteOutcome::NotMatched), + Err(e) if outcome_unknown(&e) => Ok(WriteOutcome::Unknown), + Err(e) => Err(db_err(e)), + } + }) + } + + fn read(&self) -> BoxFuture<'_, Result> { + Box::pin(async move { + let row: Option<(String, i64, f64, Option)> = sqlx::query_as( + "SELECT status, owner_generation, EXTRACT(EPOCH FROM heartbeat_at)::FLOAT8, \ + failure_reason FROM backup_index WHERE account_id = $1 AND backup_id = $2", + ) + .bind(&self.key.account_id) + .bind(&self.key.backup_id) + .fetch_optional(&self.pool) + .await + .map_err(db_err)?; + let (status, generation, heartbeat_at, failure_reason) = + row.ok_or_else(|| not_found(&self.key.backup_arn))?; + Ok(JobRow { + status: JobStatus::parse(&status)?, + owner_generation: u64::try_from(generation).unwrap_or_default(), + heartbeat_at, + failure_reason, + }) + }) + } +} + +impl BackupCatalog for PostgresEngine { + fn catalog_id(&self) -> BoxFuture<'_, Result> { + Box::pin(async move { + let id: Option = + sqlx::query_scalar("SELECT value FROM settings WHERE key = 'catalog_id'") + .fetch_optional(&self.pool) + .await + .map_err(db_err)?; + id.ok_or_else(|| { + StorageError::Internal( + "the catalog has no catalog_id; run extenddb migrate --yes".to_owned(), + ) + }) + }) + } + + fn claim_backup<'a>( + &'a self, + row: &'a BackupRow, + ) -> BoxFuture<'a, Result, StorageError>> { + Box::pin(async move { + let Some(mut session) = + try_claim(&self.pool, &backup_lock_key(&row.backup_id), None).await? + else { + return Err(StorageError::AlreadyExists(format!( + "a live job owns backup {}", + row.backup_arn + ))); + }; + let inserted = sqlx::query( + "INSERT INTO backup_index (backup_id, account_id, table_name, backup_arn, \ + backup_name, status, store_kind, store_root, store_id, size_bytes, item_count, \ + created_at, source_table_details_json, owner_generation, heartbeat_at) \ + VALUES ($1, $2, $3, $4, $5, 'CREATING', $6, $7, $8, $9, $10, to_timestamp($11), \ + $12, $13, now()) ON CONFLICT DO NOTHING", + ) + .bind(&row.backup_id) + .bind(&row.account_id) + .bind(&row.table_name) + .bind(&row.backup_arn) + .bind(&row.backup_name) + .bind(row.store.store_kind.as_str()) + .bind(&row.store.store_root) + .bind(&row.store.store_id) + .bind(row.size_bytes) + .bind(row.item_count) + .bind(row.created_at) + .bind(&row.source_table_details_json) + .bind(generation_column(session.generation())) + .execute(session.connection()) + .await + .map_err(db_err)? + .rows_affected(); + if inserted == 0 { + return Err(StorageError::AlreadyExists(format!( + "backup {} already has a row", + row.backup_arn + ))); + } + let generation = session.generation(); + Ok(Box::new(PostgresBackupClaim { + session: Mutex::new(session), + generation, + key: row.key(), + pool: self.pool.clone(), + }) as Box) + }) + } + + fn list_backup_rows<'a>( + &'a self, + status: Option, + store: &'a StoreIdentity, + ) -> BoxStream<'a, Result> { + rows_stream(async move { + sqlx::query_as(&format!( + "SELECT {ROW_COLUMNS} FROM backup_index \ + WHERE store_kind = $1 AND store_root = $2 AND store_id = $3 \ + AND ($4::TEXT IS NULL OR status = $4) ORDER BY created_at, backup_arn" + )) + .bind(store.store_kind.as_str()) + .bind(&store.store_root) + .bind(&store.store_id) + .bind(status.map(BackupStatus::as_str)) + .fetch_all(&self.pool) + .await + .map_err(db_err) + }) + } + + fn list_account_backup_rows<'a>( + &'a self, + account_id: &'a str, + table_name: Option<&'a str>, + bounds: &'a TimeBounds, + ) -> BoxStream<'a, Result> { + rows_stream(async move { + sqlx::query_as(&format!( + "SELECT {ROW_COLUMNS} FROM backup_index WHERE account_id = $1 \ + AND ($2::TEXT IS NULL OR table_name = $2) \ + AND ($3::FLOAT8 IS NULL OR created_at >= to_timestamp($3)) \ + AND ($4::FLOAT8 IS NULL OR created_at < to_timestamp($4)) \ + ORDER BY created_at, backup_arn" + )) + .bind(account_id) + .bind(table_name) + .bind(bounds.lower) + .bind(bounds.upper) + .fetch_all(&self.pool) + .await + .map_err(db_err) + }) + } + + fn read_backup_row<'a>( + &'a self, + arn: &'a BackupArn, + ) -> BoxFuture<'a, Result, StorageError>> { + Box::pin(async move { + let row: Option = sqlx::query_as(&format!( + "SELECT {ROW_COLUMNS} FROM backup_index WHERE account_id = $1 AND backup_id = $2 \ + AND table_name = $3 AND backup_arn = $4" + )) + .bind(arn.account_id()) + .bind(arn.backup_id()) + .bind(arn.table_name()) + .bind(arn.to_arn_string()) + .fetch_optional(&self.pool) + .await + .map_err(db_err)?; + row.map(row_from_tuple).transpose() + }) + } + + fn start_backup_delete<'a>( + &'a self, + key: &'a BackupKey, + ) -> BoxFuture<'a, Result> { + Box::pin(async move { + let mut tx = self.pool.begin().await.map_err(db_err)?; + // FOR UPDATE is the contract a restore's registration of this + // row must meet: it locks the same row before its own in-use + // check, so the two order wholly one before the other (a + // registration that commits first leaves this `InUse`; one that + // waits behind this finds the row DELETING). On this head the + // legacy registration locks the legacy `backups` row and no + // restore of a `backup_index` row exists; PR 9a's registration + // is the first to take this lock. + let status: Option = sqlx::query_scalar( + "SELECT status FROM backup_index WHERE account_id = $1 AND backup_id = $2 \ + FOR UPDATE", + ) + .bind(&key.account_id) + .bind(&key.backup_id) + .fetch_optional(&mut *tx) + .await + .map_err(db_err)?; + let status = BackupStatus::parse(&status.ok_or_else(|| not_found(&key.backup_arn))?)?; + if status != BackupStatus::Available { + return Ok(DeleteStart::Status(status)); + } + if let Some(table) = creating_restore_target(&mut *tx, &key.backup_arn).await? { + return Ok(DeleteStart::InUse(table)); + } + sqlx::query( + "UPDATE backup_index SET status = 'DELETING' \ + WHERE account_id = $1 AND backup_id = $2 AND status = 'AVAILABLE'", + ) + .bind(&key.account_id) + .bind(&key.backup_id) + .execute(&mut *tx) + .await + .map_err(db_err)?; + tx.commit().await.map_err(db_err)?; + Ok(DeleteStart::Deleting) + }) + } + + fn set_backup_deleted<'a>( + &'a self, + key: &'a BackupKey, + ) -> BoxFuture<'a, Result> { + Box::pin(async move { + let affected = sqlx::query( + "UPDATE backup_index SET status = 'DELETED', completed_at = now() \ + WHERE account_id = $1 AND backup_id = $2 AND status = 'DELETING'", + ) + .bind(&key.account_id) + .bind(&key.backup_id) + .execute(&self.pool) + .await + .map_err(db_err)? + .rows_affected(); + Ok(affected == 1) + }) + } + + fn remove_backup_row<'a>( + &'a self, + key: &'a BackupKey, + ) -> BoxFuture<'a, Result> { + Box::pin(async move { + let affected = sqlx::query( + "DELETE FROM backup_index WHERE account_id = $1 AND backup_id = $2 \ + AND status IN ('DELETED', 'FAILED')", + ) + .bind(&key.account_id) + .bind(&key.backup_id) + .execute(&self.pool) + .await + .map_err(db_err)? + .rows_affected(); + Ok(affected == 1) + }) + } + + fn backup_in_use<'a>( + &'a self, + key: &'a BackupKey, + ) -> BoxFuture<'a, Result, StorageError>> { + Box::pin(async move { creating_restore_target(&self.pool, &key.backup_arn).await }) + } + + fn remove_restore_target<'a>( + &'a self, + key: &'a RestoreTargetKey, + ) -> BoxFuture<'a, Result> { + Box::pin(async move { + // The key must name a CREATING restore target with this table id; + // a table re-created under the name has another id and is not + // touched. `abort_restore` then claims it from CREATING in one + // statement, so it and the ACTIVE flip exclude each other. + let is_target: bool = sqlx::query_scalar( + "SELECT EXISTS(SELECT 1 FROM tables t JOIN table_restores r \ + ON r.table_id = t.table_id WHERE t.table_id = $1 AND t.account_id = $2 \ + AND t.table_name = $3 AND t.table_status = 'CREATING')", + ) + .bind(&key.table_id) + .bind(&key.account_id) + .bind(&key.table_name) + .fetch_one(&self.pool) + .await + .map_err(db_err)?; + if !is_target { + return Ok(false); + } + // Progress rows belong to a later release's restore path; a + // catalog without that table has none to delete. + let has_progress: bool = + sqlx::query_scalar("SELECT to_regclass('restore_progress') IS NOT NULL") + .fetch_one(&self.pool) + .await + .map_err(db_err)?; + if has_progress { + sqlx::query("DELETE FROM restore_progress WHERE table_id = $1") + .bind(&key.table_id) + .execute(&self.pool) + .await + .map_err(db_err)?; + } + self.abort_restore(&key.table_id).await + }) + } + + fn list_dead_restore_jobs( + &self, + stale_after: Duration, + ) -> BoxFuture<'_, Result, StorageError>> { + Box::pin(async move { + // Two arms: a RUNNING record whose heartbeat is stale (a dead + // owner), and a FAILED record whose table is still a CREATING + // restore target once the record is stale (the FAILED write + // landed and the removal did not; `fail_restore_job` and the + // job's own failure path stamp `completed_at`, and a record + // without one is aged by its heartbeat). The delay is what + // leaves a job still on its own failure path to finish it. + #[allow(clippy::type_complexity)] + let rows: Vec<(String, String, String, String, String, String, i64, f64)> = + sqlx::query_as( + "SELECT r.table_id, t.account_id, t.table_name, r.source_backup_arn, \ + r.kind, r.status, r.owner_generation, \ + EXTRACT(EPOCH FROM r.heartbeat_at)::FLOAT8 \ + FROM table_restores r JOIN tables t ON t.table_id = r.table_id \ + WHERE (r.status = 'RUNNING' \ + AND r.heartbeat_at < now() - make_interval(secs => $1)) \ + OR (r.status = 'FAILED' AND t.table_status = 'CREATING' \ + AND COALESCE(r.completed_at, r.heartbeat_at) \ + < now() - make_interval(secs => $1)) \ + ORDER BY t.account_id, t.table_name", + ) + .bind(stale_after.as_secs_f64()) + .fetch_all(&self.pool) + .await + .map_err(db_err)?; + rows.into_iter() + .map( + |( + table_id, + account_id, + table_name, + source_backup_arn, + kind, + status, + owner_generation, + heartbeat_at, + )| { + Ok(RestoreJobRow { + target: RestoreTargetKey { + account_id, + table_name, + table_id, + }, + source_backup_arn, + kind: RestoreKind::parse(&kind)?, + status: JobStatus::parse(&status)?, + owner_generation: u64::try_from(owner_generation).unwrap_or_default(), + heartbeat_at, + }) + }, + ) + .collect() + }) + } + + fn fail_restore_job<'a>( + &'a self, + key: &'a RestoreTargetKey, + reason: &'a str, + ) -> BoxFuture<'a, Result> { + Box::pin(async move { + let affected = sqlx::query( + "UPDATE table_restores r SET status = 'FAILED', failure_reason = $4, \ + completed_at = now() FROM tables t \ + WHERE t.table_id = r.table_id AND r.table_id = $1 AND t.account_id = $2 \ + AND t.table_name = $3 AND r.status = 'RUNNING'", + ) + .bind(&key.table_id) + .bind(&key.account_id) + .bind(&key.table_name) + .bind(reason) + .execute(&self.pool) + .await + .map_err(db_err)? + .rows_affected(); + Ok(affected == 1) + }) + } + + fn check_maintenance_claim(&self) -> BoxFuture<'_, Result> { + Box::pin(async move { + sqlx::query_scalar("SELECT EXISTS(SELECT 1 FROM backup_maintenance)") + .fetch_one(&self.pool) + .await + .map_err(db_err) + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_row_tuple_with_an_unknown_store_kind_or_status_is_internal() { + let good = || RowTuple { + backup_id: "01757500000123-0badcafe".to_owned(), + account_id: "123456789012".to_owned(), + table_name: "Music".to_owned(), + backup_arn: "arn".to_owned(), + backup_name: "b".to_owned(), + status: "AVAILABLE".to_owned(), + store_kind: "s3".to_owned(), + store_root: "s3://bucket/prefix".to_owned(), + store_id: "store-1".to_owned(), + size_bytes: 1, + item_count: 2, + created_at: 3.0, + completed_at: None, + failure_reason: None, + source_table_details_json: serde_json::json!({}), + owner_generation: 7, + heartbeat_at: 4.0, + }; + let row = row_from_tuple(good()).unwrap(); + assert_eq!(row.store.store_kind, StoreKind::S3); + assert_eq!(row.owner_generation, 7); + let mut bad_kind = good(); + bad_kind.store_kind = "gcs".to_owned(); + assert!(matches!( + row_from_tuple(bad_kind), + Err(StorageError::Internal(_)) + )); + let mut bad_status = good(); + bad_status.status = "ACTIVE".to_owned(); + assert!(matches!( + row_from_tuple(bad_status), + Err(StorageError::Internal(_)) + )); + } + + #[test] + fn only_a_connection_class_failure_leaves_a_write_unknown() { + assert!(outcome_unknown(&sqlx::Error::PoolClosed)); + assert!(outcome_unknown(&sqlx::Error::Io(std::io::Error::new( + std::io::ErrorKind::ConnectionReset, + "reset", + )))); + assert!(!outcome_unknown(&sqlx::Error::Protocol("x".to_owned()))); + assert!(!outcome_unknown(&sqlx::Error::ColumnNotFound( + "status".to_owned() + ))); + } +} diff --git a/crates/storage-postgres/src/backup_engine.rs b/crates/storage-postgres/src/backup_engine.rs index 3255dbb2e..0adceaa5a 100755 --- a/crates/storage-postgres/src/backup_engine.rs +++ b/crates/storage-postgres/src/backup_engine.rs @@ -1,16 +1,23 @@ // Copyright 2026 ExtendDB contributors // SPDX-License-Identifier: Apache-2.0 -//! Backup and restore implementation for `PostgreSQL` storage. +//! The in-catalog backup format on `PostgreSQL`: the create, describe, list, +//! delete, restore-source and restore paths of [`LegacyBackupReader`], frozen +//! apart from the restore's registration, which writes the restore job +//! fields and holds the job claim ([`ClaimSession`]) the catalog index's jobs +//! hold too (`backup_catalog.rs`). -use std::collections::HashMap; use std::sync::Arc; +use std::time::Duration; use extenddb_core::types::{ BackupDescription, BackupDetails, BackupSummary, ContinuousBackupsDescription, GsiInput, Item, KeySchemaElement, LsiInput, PointInTimeRecoveryDescription, Projection, RestoreSummary, ScalarAttributeType, SourceTableDetails, StreamSpecification, TableDescription, }; +use extenddb_storage::backup_catalog::{ + AbandonedLegacyBackup, JOB_HEARTBEAT_INTERVAL, JobBound, limit_exceeded_message, new_generation, +}; use extenddb_storage::backup_definition::{ BACKUP_DEFINITION_VERSION, BackupTableDefinition, COPY_BATCH_BYTES, COPY_BATCH_ITEMS, throughput_from_catalog, @@ -20,9 +27,8 @@ use extenddb_storage::util::{ SortKeyValue, backup_arn, backup_id_epoch_millis, composite_pk_to_text, new_backup_id, parse_sk, sk_column_n, }; -use extenddb_storage::{BackupEngine, BackupRestoreSource, RestoreRequest}; +use extenddb_storage::{BackupRestoreSource, JobPermit, LegacyBackupReader, RestoreRequest}; use futures::future::BoxFuture; -use tokio::sync::{OwnedSemaphorePermit, Semaphore}; use crate::PostgresEngine; use crate::create_table::RestoreRegistration; @@ -48,119 +54,15 @@ fn pg_timestamp_to_epoch(ts: time::OffsetDateTime) -> f64 { } } -/// Seed for the 64-bit restore lock key. Restore locks use the single-`bigint` -/// advisory lock form, which PostgreSQL keeps in a key space separate from the -/// two-`int4` form the migration and vector-build locks use, so they cannot -/// meet; the seed keeps this key space distinct from any other `bigint` user. -const RESTORE_LOCK_SEED: i64 = 0x0045_4452; // 'E', 'D', 'R' - -/// Restores one account may have in flight at once in this process. Each -/// holds one dedicated connection for its lock on top of its pooled ones. A -/// restore beyond the limit waits for one of its own account's slots, so one -/// account cannot hold every other account's restores back. -const RESTORE_SLOTS_PER_ACCOUNT: usize = 4; - -/// Upper limit on [`restore_server_slot_count`]. -const RESTORE_SERVER_SLOTS_MAX: usize = 16; - -/// Restores the whole process may have in flight at once, across accounts: -/// `min(16, pool_size / 2)`, at least 1. -/// -/// Each running copy holds one connection of the engine's catalog pool -/// (`pool_size`) for its whole duration, for the REPEATABLE READ snapshot it -/// reads the backup's items from. Without a process-wide bound, enough -/// accounts each running their per-account allowance would hold every -/// catalog connection and stall every other request that needs the catalog. -/// Half the pool stays free for them. A restore takes its account slot -/// first, then one of these; the two waits share one deadline of -/// [`RESTORE_SLOT_WAIT_SECS`]. -pub(crate) fn restore_server_slot_count(pool_size: u32) -> usize { - usize::try_from(pool_size / 2) - .unwrap_or(usize::MAX) - .clamp(1, RESTORE_SERVER_SLOTS_MAX) -} - -/// The two slots a running restore holds until its copy ends. -struct RestorePermits { - _account: OwnedSemaphorePermit, - _server: OwnedSemaphorePermit, -} - -/// Take an account slot, then a server slot, waiting at most `wait` for -/// both together. Running out of time on either is `LimitExceeded`, with a -/// message naming the limit that was full. -async fn acquire_restore_permits( - account: Arc, - server: Arc, - server_slots: usize, - wait: std::time::Duration, -) -> Result { - let deadline = tokio::time::Instant::now() + wait; - let closed = - |e: tokio::sync::AcquireError| StorageError::Internal(format!("restore slot: {e}")); - let account_permit = tokio::time::timeout_at(deadline, account.acquire_owned()) - .await - .map_err(|_| { - StorageError::LimitExceeded(format!( - "Too many restores are in progress for this account (at most \ - {RESTORE_SLOTS_PER_ACCOUNT} at once); retry later" - )) - })? - .map_err(closed)?; - let server_permit = tokio::time::timeout_at(deadline, server.acquire_owned()) - .await - .map_err(|_| { - StorageError::LimitExceeded(format!( - "Too many restores are in progress on this server (at most {server_slots} at \ - once); retry later" - )) - })? - .map_err(closed)?; - Ok(RestorePermits { - _account: account_permit, - _server: server_permit, - }) -} - -/// Per-account restore slots of one engine (shared by its clones, so one -/// per server process). An entry nobody holds (no permit, no waiter) is -/// pruned the next time any account takes a slot, so the map does not grow -/// with every account that ever restored. -#[derive(Clone, Default)] -pub(crate) struct RestoreAccountSlots(Arc>>>); - -impl RestoreAccountSlots { - /// The slot semaphore for `account_id`. - pub(crate) fn for_account(&self, account_id: &str) -> Arc { - let mut slots = self - .0 - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner); - // A permit and a waiter each hold a clone, so a count of one means - // the account has nothing in flight. - slots.retain(|_, s| Arc::strong_count(s) > 1); - Arc::clone( - slots - .entry(account_id.to_owned()) - .or_insert_with(|| Arc::new(Semaphore::new(RESTORE_SLOTS_PER_ACCOUNT))), - ) - } -} - -/// How long a restore waits for its account slot and a server slot, -/// together, before it is refused with `LimitExceededException`. -const RESTORE_SLOT_WAIT_SECS: u64 = 30; - -/// How often the restore's lock session is exercised while its copy runs. -/// Shorter than the idle-session timeouts managed services apply (minutes), -/// and a quarter of [`ABANDONED_RESTORE_GRACE_SECS`]. -const RESTORE_HEARTBEAT_SECS: u64 = 15; - -/// How old an unowned restore target must be before it is treated as -/// abandoned. The owner takes its lock before creating the target, so this -/// only has to cover clock skew between the catalog and the sweep, with a -/// wide margin. -const ABANDONED_RESTORE_GRACE_SECS: i32 = 60; +/// Seed for the 64-bit job lock keys: a restore's lock on its target name +/// and a backup job's lock on its backup id share this one key space. The +/// locks use the single-`bigint` advisory lock form, which PostgreSQL keeps +/// in a key space separate from the two-`int4` form the migration and +/// vector-build locks use, so they cannot meet; the seed keeps this key +/// space distinct from any other `bigint` user. Public for the tests that +/// hold or look up a lock; not an interface. +#[doc(hidden)] +pub const RESTORE_LOCK_SEED: i64 = 0x0045_4452; // 'E', 'D', 'R' /// A secondary index of a restore target, as the copy needs it. struct RestoreIndex { @@ -169,81 +71,133 @@ struct RestoreIndex { projection: Projection, } -/// Session-scoped ownership of one restore, held for the life of the copy. -/// -/// A session-level `pg_try_advisory_lock` on a dedicated connection: it dies -/// with the session, so a crashed process's claim disappears on its own and -/// the abandoned-restore sweep can tell a dead restore from a running one. +/// Session-scoped ownership of one job, held for the job's life: a +/// session-level `pg_try_advisory_lock` on a dedicated, non-pooled connection +/// to the catalog database, the job's `owner_generation`, and the job permit +/// the engine handed it. /// -/// This needs the connection to be one server session for its whole life: -/// a direct connection to the catalog database, or one through a +/// The lock dies with the session, so a crashed process's claim disappears on +/// its own. This needs the connection to be one server session for its whole +/// life: a direct connection to the catalog database, or one through a /// session-mode pooler. Behind a transaction-mode pooler (PgBouncer /// `pool_mode = transaction`) the lock stays on whichever server session ran /// the statement, is not released when this connection closes, and can be -/// observed as free by the sweep on that same session; restore is not -/// supported there. +/// observed as free on that same session; jobs are not supported there. /// -/// The connection would otherwise sit idle for the whole copy, and some +/// The connection would otherwise sit idle for the whole job, and some /// servers end idle sessions (`idle_session_timeout`, or a managed service's -/// own reaping). [`RestoreOwner::heartbeat`] issues -/// a statement on it every [`RESTORE_HEARTBEAT_SECS`]; when one fails, the -/// copy is stopped and the target removed, rather than copying on into a -/// target the sweep may claim. The sweep's claim and the copy's flip to -/// ACTIVE are both conditional on CREATING, so if the two still race (the -/// session ends between heartbeats), exactly one wins. -struct RestoreOwner { +/// own reaping). The job's heartbeats go through this connection every +/// [`JOB_HEARTBEAT_INTERVAL`] (shorter than the idle-session timeouts managed +/// services apply, and a small fraction of the stale window the reconciler +/// waits): each refreshes the record's `heartbeat_at` +/// conditioned on the generation, so a session the server dropped, or a +/// record a successor took over, surfaces as a lost heartbeat and the job +/// stops rather than continuing unowned. A free lock shows that the owner's +/// catalog session ended, not that its process stopped, so the reconciler +/// acts on a record only once its heartbeat is older than the stale window. +pub(crate) struct ClaimSession { conn: sqlx::PgConnection, - _slots: Option, + generation: u64, + _permit: Option, } -impl RestoreOwner { - /// Exercise the lock session every [`RESTORE_HEARTBEAT_SECS`] until a - /// statement on it fails; returns that failure. Never returns while the - /// session is alive. - async fn heartbeat(&mut self) -> sqlx::Error { - let period = std::time::Duration::from_secs(RESTORE_HEARTBEAT_SECS); +impl ClaimSession { + /// The fencing token this claim was taken with. + pub(crate) fn generation(&self) -> u64 { + self.generation + } + + /// The claim connection, for the job's conditional writes. + pub(crate) fn connection(&mut self) -> &mut sqlx::PgConnection { + &mut self.conn + } + + /// Run `statement`, bound with the record's key and this generation, on + /// the claim connection and report whether it refreshed exactly one row. + pub(crate) async fn confirm( + &mut self, + statement: &str, + key: &[&str], + ) -> Result { + let mut query = sqlx::query(statement); + for part in key { + query = query.bind(*part); + } + let affected = query + .bind(generation_column(self.generation)) + .execute(&mut self.conn) + .await? + .rows_affected(); + Ok(affected == 1) + } + + /// Heartbeat the record every [`JOB_HEARTBEAT_INTERVAL`] with + /// `statement` (as [`Self::confirm`] runs it) until a heartbeat fails or + /// refreshes no row; returns why. Never returns while the claim is + /// confirmed. + async fn heartbeat_loop(&mut self, statement: &str, key: &[&str]) -> StorageError { + let period = JOB_HEARTBEAT_INTERVAL; let mut tick = tokio::time::interval_at(tokio::time::Instant::now() + period, period); tick.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); loop { tick.tick().await; - if let Err(e) = sqlx::query("SELECT 1").execute(&mut self.conn).await { - return e; + match self.confirm(statement, key).await { + Ok(true) => {} + Ok(false) => { + return StorageError::Internal( + "the job record no longer carries this claim's generation".to_owned(), + ); + } + Err(e) => return StorageError::Internal(format!("claim session: {e}")), } } } } +/// An `owner_generation` as the catalog stores it (`BIGINT`); generations +/// are minted below `i64::MAX`. +pub(crate) fn generation_column(generation: u64) -> i64 { + i64::try_from(generation).unwrap_or(i64::MAX) +} + +/// The heartbeat of a legacy restore: refresh its job record while it is +/// RUNNING under this generation. +const RESTORE_HEARTBEAT: &str = "UPDATE table_restores SET heartbeat_at = now() \ + WHERE table_id = $1 AND status = 'RUNNING' AND owner_generation = $2"; + /// The advisory-lock key for a restore into `table_name` in `account_id`. /// /// Keyed by name rather than table id so the restore can hold it before the -/// target exists: the sweep must never see an unowned target, including in -/// the window while `create_table_impl` is still running DDL. The key is a -/// 64-bit `hashtextextended`, so a collision is negligible, and one would -/// only make the sweep skip a table while the other name's lock is held, -/// never remove a table it should not. +/// target exists: nothing must ever see an unowned target, including in the +/// window while `create_table_impl` is still running DDL. The key is a 64-bit +/// `hashtextextended`, so a collision is negligible, and one would only make +/// a second restore into the other name wait, never remove a table it should +/// not. fn restore_lock_key(account_id: &str, table_name: &str) -> String { format!("{account_id}/{table_name}") } -/// Take the restore lock for `table_name` on a new dedicated connection. -/// `Ok(None)`: another session holds it. +/// The advisory-lock key of a backup job's claim on `backup_id`. +pub(crate) fn backup_lock_key(backup_id: &str) -> String { + format!("backup:{backup_id}") +} + +/// Take the lock `key` on a new dedicated connection, with a fresh +/// generation. `Ok(None)`: another session holds it. /// /// A connection the server refuses for lack of connection slots is reported /// as `LimitExceeded`; any other failure to open it is `Internal` (see /// [`lock_connection_error`]). -async fn try_restore_lock( +pub(crate) async fn try_claim( pool: &sqlx::PgPool, - account_id: &str, - table_name: &str, - slots: Option, -) -> Result, StorageError> { + key: &str, + permit: Option, +) -> Result, StorageError> { let options = pool.connect_options(); let mut conn = ::connect_with(&options) .await .map_err(|e| { - tracing::warn!( - "could not open the restore lock connection for {account_id}/{table_name}: {e}" - ); + tracing::warn!("could not open the claim connection for {key}: {e}"); lock_connection_error(&e) })?; // A server-side idle_session_timeout would end this session between @@ -256,38 +210,54 @@ async fn try_restore_lock( .await { tracing::warn!( - "could not disable idle_session_timeout for the restore lock session; \ - continuing with the server setting: {e}" + "could not disable idle_session_timeout for the claim session; continuing with \ + the server setting: {e}" ); } let taken: bool = sqlx::query_scalar("SELECT pg_try_advisory_lock(hashtextextended($1, $2))") - .bind(restore_lock_key(account_id, table_name)) + .bind(key) .bind(RESTORE_LOCK_SEED) .fetch_one(&mut conn) .await - .map_err(|e| StorageError::Internal(format!("restore lock: {e}")))?; - Ok(taken.then_some(RestoreOwner { + .map_err(|e| StorageError::Internal(format!("claim lock: {e}")))?; + Ok(taken.then(|| ClaimSession { conn, - _slots: slots, + generation: new_generation(), + _permit: permit, })) } +/// Take the restore lock for `table_name` on a new dedicated connection. +async fn try_restore_lock( + pool: &sqlx::PgPool, + account_id: &str, + table_name: &str, + permit: JobPermit, +) -> Result, StorageError> { + try_claim( + pool, + &restore_lock_key(account_id, table_name), + Some(permit), + ) + .await +} + /// SQLSTATE `too_many_connections`: the server (or the role or database) /// has no connection slot left. const SQLSTATE_TOO_MANY_CONNECTIONS: &str = "53300"; -/// Map a failure to open the restore lock connection. Only the server's +/// Map a failure to open a claim connection. Only the server's /// `too_many_connections` refusal is `LimitExceeded` (a client should back -/// off and retry); authentication, TLS, DNS, network, and every other -/// failure, including one that carries no SQLSTATE, is `Internal`. +/// off and retry), in the one job-refusal text +/// ([`limit_exceeded_message`]) naming the server's connection slots as the +/// bound; authentication, TLS, DNS, network, and every other failure, +/// including one that carries no SQLSTATE, is `Internal`. fn lock_connection_error(e: &sqlx::Error) -> StorageError { let code = e.as_database_error().and_then(|d| d.code()); if code.as_deref() == Some(SQLSTATE_TOO_MANY_CONNECTIONS) { - StorageError::LimitExceeded( - "Too many restores are in progress on this server; retry later".to_owned(), - ) + StorageError::LimitExceeded(limit_exceeded_message(JobBound::ServerConnections)) } else { - StorageError::Internal(format!("restore lock connection: {e}")) + StorageError::Internal(format!("claim connection: {e}")) } } @@ -452,7 +422,8 @@ impl PostgresEngine { } /// Copy a backup's items into a freshly created restore target, populate - /// its secondary indexes, and mark it ACTIVE. + /// its secondary indexes, and mark it ACTIVE, which marks the job record + /// DONE in the same catalog transaction, conditioned on `generation`. /// /// Every key column is derived from the item itself, so the copy does not /// depend on how the source table's columns were laid out. Rows are plain @@ -474,6 +445,7 @@ impl PostgresEngine { &self, desc: &TableDescription, backup_arn: &str, + generation: u64, ) -> Result<(), StorageError> { use futures::TryStreamExt; @@ -561,36 +533,82 @@ impl PostgresEngine { .await .map_err(db_err)?; - // Mark the restored table ACTIVE now that every batch has committed. - // The table was created with the transition deferred, so this is the - // first point it can become ACTIVE, by ordering rather than by timing. + // Mark the restored table ACTIVE now that every batch has committed, + // and its job record DONE, in one catalog transaction. The table was + // created with the transition deferred, so this is the first point it + // can become ACTIVE, by ordering rather than by timing. // - // Only from CREATING. DeleteTable refuses a restore target, but the - // abandoned-restore sweep can claim one whose lock session was lost - // (moving it to DELETING); that claim wins, and flipping the row back - // to ACTIVE would leave a table whose data tables are being dropped. - let activated = sqlx::query( - "UPDATE tables SET item_count = $1, table_size_bytes = $2, table_status = 'ACTIVE', \ - status_transition_at = NULL WHERE table_id = $3 AND table_status = 'CREATING'", + // Only from CREATING, and only while the job record is RUNNING under + // this generation. DeleteTable refuses a restore target, but the + // reconciler can claim one whose owner it found dead (moving it to + // DELETING); that claim wins, and flipping the row back to ACTIVE + // would leave a table whose data tables are being dropped. + let mut tx = self.pool.begin().await.map_err(db_err)?; + let done = sqlx::query( + "UPDATE table_restores SET status = 'DONE', completed_at = now() \ + WHERE table_id = $1 AND status = 'RUNNING' AND owner_generation = $2", ) - .bind(item_count) - .bind(table_size) .bind(&desc.table_id) - .execute(&self.pool) + .bind(generation_column(generation)) + .execute(&mut *tx) .await - .map_err(db_err)?; - if activated.rows_affected() == 0 { - // The target was claimed for removal while the copy ran; the - // copy's rows go with it. + .map_err(db_err)? + .rows_affected(); + let activated = if done == 1 { + sqlx::query( + "UPDATE tables SET item_count = $1, table_size_bytes = $2, \ + table_status = 'ACTIVE', status_transition_at = NULL \ + WHERE table_id = $3 AND table_status = 'CREATING'", + ) + .bind(item_count) + .bind(table_size) + .bind(&desc.table_id) + .execute(&mut *tx) + .await + .map_err(db_err)? + .rows_affected() + } else { + 0 + }; + if activated == 0 { + tx.rollback().await.map_err(db_err)?; + // The target was claimed for removal while the copy ran, or the + // job record passed to another claim; the copy's rows go with it. return Err(StorageError::TableNotFound(format!( "restore of {backup_arn} into {} did not complete: the table was deleted \ while it was being restored", desc.table_name ))); } + tx.commit().await.map_err(db_err)?; Ok(()) } + /// RUNNING to FAILED on the job record of `table_id`, conditioned on + /// `generation`: the write a failing job makes before it removes its + /// target. `Ok(false)` when the record is not RUNNING under that + /// generation. + async fn fail_restore_job_as( + &self, + table_id: &str, + generation: u64, + reason: &str, + ) -> Result { + let affected = sqlx::query( + "UPDATE table_restores SET status = 'FAILED', failure_reason = $3, \ + completed_at = now() \ + WHERE table_id = $1 AND status = 'RUNNING' AND owner_generation = $2", + ) + .bind(table_id) + .bind(generation_column(generation)) + .bind(reason) + .execute(&self.pool) + .await + .map_err(db_err)? + .rows_affected(); + Ok(affected == 1) + } + /// Write and commit one batch of restored items: one multi-row INSERT /// into the base table, then one multi-row INSERT into each secondary /// index for the batch's items that carry that index's keys. @@ -698,7 +716,11 @@ impl PostgresEngine { /// exactly one wins. The claim also schedules the row for the ordinary /// DELETING removal, so if a drop below fails, the row stays DELETING with /// its index rows and the control plane finishes the job later. - async fn abort_restore(&self, table_id: &str) -> Result { + /// + /// Public for the test that runs it against an ACTIVE table; not an + /// interface. + #[doc(hidden)] + pub async fn abort_restore(&self, table_id: &str) -> Result { let mut tx = self.pool.begin().await.map_err(db_err)?; let claimed = sqlx::query( "UPDATE tables SET table_status = 'DELETING', status_transition_at = NOW() \ @@ -736,61 +758,16 @@ impl PostgresEngine { Ok(true) } - /// Remove restore targets whose restore died with its process. - /// - /// A restore target is the only table that sits CREATING with no - /// scheduled transition. One older than the grace period whose restore - /// lock is free has no live owner: the process that created it crashed or - /// was killed mid-copy, or its lock session ended, and nothing else would - /// ever move it on. Removing it frees the name; the client sees the table - /// disappear, as it would after a failed restore. A target whose lock is - /// held is being copied by a live process, here or on another instance, - /// and is left alone. - /// - /// Each removal is logged at warn with the account, the table, and the - /// backup it was being restored from, so a table that vanishes from - /// ListTables has a server record of why. - /// - /// Returns the names of the tables removed. - pub(crate) async fn sweep_abandoned_restores(&self) -> Result, StorageError> { - let candidates: Vec<(String, String, String, Option)> = sqlx::query_as( - "SELECT t.table_id, t.account_id, t.table_name, r.source_backup_arn FROM tables t \ - LEFT JOIN table_restores r ON r.table_id = t.table_id \ - WHERE t.table_status = 'CREATING' AND t.status_transition_at IS NULL \ - AND t.creation_date_time < NOW() - make_interval(secs => $1)", - ) - .bind(f64::from(ABANDONED_RESTORE_GRACE_SECS)) - .fetch_all(&self.pool) - .await - .map_err(db_err)?; - let mut removed = Vec::new(); - for (table_id, account_id, table_name, source_backup_arn) in candidates { - let Some(_owner) = try_restore_lock(&self.pool, &account_id, &table_name, None).await? - else { - continue; - }; - if self.abort_restore(&table_id).await? { - tracing::warn!( - "removed abandoned restore target {table_name} ({table_id}) in account \ - {account_id}, restoring from {}: its restore did not finish and no \ - process owns it", - source_backup_arn - .as_deref() - .unwrap_or("") - ); - removed.push(table_name); - } - } - Ok(removed) - } - - /// The detached task of RestoreTableFromBackup: wait for the account and - /// server slots, take the restore lock, register the CREATING target, - /// report the registration to the request (which may be gone), then copy. + /// The detached task of RestoreTableFromBackup: take the restore lock, + /// register the CREATING target with its job record, report the + /// registration to the request (which may be gone), then copy. `permit` + /// is the job slot the engine took, held by the claim until the copy + /// ends. async fn run_restore( self, account_id: String, request: RestoreRequest, + permit: JobPermit, registered: tokio::sync::oneshot::Sender>, ) { let RestoreRequest { @@ -803,7 +780,8 @@ impl PostgresEngine { // A refusal the request is no longer there to receive (its client // went away) would otherwise vanish, so it is logged. A refused restore // removes what it created; a cleanup that failed is logged on its own, - // and the abandoned-restore sweep finishes it. + // and the reconciler finishes it (the FAILED-target duty, once the + // record is stale). let refuse = |registered: tokio::sync::oneshot::Sender<_>, e: StorageError| { if let Err(Err(e)) = registered.send(Err(e)) { tracing::warn!( @@ -812,15 +790,13 @@ impl PostgresEngine { ); } }; - // The name first, unfenced, as on the other backends and before a - // slot is taken: a restore into a name that is a CREATING restore - // target is refused as the name (TableBeingRestored, the service's - // TableInUseException) at once, whatever state the backup is in and - // whether or not that target's restore is still running (an - // abandoned target holds no restore lock until the sweep removes - // it). A client retrying its own restore is answered here instead of - // waiting for a slot. The restore lock and the target insert stay - // the fence. + // The name first, unfenced, as on the other backends: a restore into + // a name that is a CREATING restore target is refused as the name + // (TableBeingRestored, the service's TableInUseException) at once, + // whatever state the backup is in and whether or not that target's + // restore is still running (an abandoned target holds no restore + // lock until the reconciler removes it). The restore lock and the + // target insert stay the fence. if let Some(source_backup_arn) = self.restoring_into(&account_id, &target).await { refuse( registered, @@ -831,32 +807,15 @@ impl PostgresEngine { ); return; } - // Admission: at most RESTORE_SLOTS_PER_ACCOUNT restores per account - // and restore_server_slot_count per process at once. The permits - // travel with the lock owner and are released when the copy ends. - let slots = match acquire_restore_permits( - self.restore_account_slots.for_account(&account_id), - Arc::clone(&self.restore_server_slots), - self.restore_server_slot_count, - std::time::Duration::from_secs(RESTORE_SLOT_WAIT_SECS), - ) - .await - { - Ok(slots) => slots, - Err(e) => { - refuse(registered, e); - return; - } - }; - // Ownership first, before the target exists, so the abandoned-restore - // sweep never sees this target without an owner, including while the - // DDL runs. It is held until the copy ends; a crash anywhere releases - // it with the connection, and the sweep then removes the target. A - // lock already held means another restore into this name is in - // flight right now: TableBeingRestored naming its source backup once - // its CREATING row is visible, else the name is taken as CreateTable - // would report it. - let owner = match try_restore_lock(&self.pool, &account_id, &target, Some(slots)).await { + // Ownership first, before the target exists, so nothing ever sees + // this target without an owner, including while the DDL runs. It is + // held until the copy ends; a crash anywhere releases it with the + // connection, and the reconciler removes the target once the job + // record's heartbeat is stale. A lock already held means another + // restore into this name is in flight right now: TableBeingRestored + // naming its source backup once its CREATING row is visible, else + // the name is taken as CreateTable would report it. + let owner = match try_restore_lock(&self.pool, &account_id, &target, permit).await { Ok(Some(owner)) => owner, Ok(None) => { let error = self.name_taken_error(&account_id, &target, true).await; @@ -877,6 +836,7 @@ impl PostgresEngine { backup_arn: &backup_arn, source_table_arn: &source_table_arn, restore_date_time, + owner_generation: owner.generation(), }; let mut desc = match self .create_table_for_restore(&account_id, input, ®istration) @@ -905,29 +865,31 @@ impl PostgresEngine { } /// The detached half of RestoreTableFromBackup: copy the backup into the - /// registered CREATING target while heartbeating the lock session, then - /// flip the target ACTIVE, or remove it on failure. Owns everything it - /// uses; the request that spawned it may be gone. + /// registered CREATING target while heartbeating the job record through + /// the claim session, then flip the target ACTIVE (which marks the job + /// record DONE, conditioned on the generation), or remove it on failure. + /// Owns everything it uses; the request that spawned it may be gone. async fn run_restore_copy( self, - mut owner: RestoreOwner, + mut owner: ClaimSession, account_id: String, desc: TableDescription, backup_arn: String, ) { let table_name = desc.table_name.clone(); let outcome = { - let copy = self.copy_backup_items(&desc, &backup_arn); - let heartbeat = owner.heartbeat(); + let key = [desc.table_id.as_str()]; + let copy = self.copy_backup_items(&desc, &backup_arn, owner.generation()); + let heartbeat = owner.heartbeat_loop(RESTORE_HEARTBEAT, &key); tokio::select! { copied = copy => copied, lost = heartbeat => { tracing::error!( "restore of {backup_arn} into {table_name} (account {account_id}): the \ - restore lock session failed its heartbeat ({lost}); stopping the copy" + claim session failed its heartbeat ({lost}); stopping the copy" ); Err(StorageError::Internal(format!( - "restore lock session lost: {lost}" + "claim session lost: {lost}" ))) } } @@ -943,17 +905,44 @@ impl PostgresEngine { }; // The target must not stay behind: it is CREATING with no scheduled // transition, so nothing else would move it on, and the name would - // stay taken. Removing it cascades its table_restores row. The lock - // is still held here (unless its session is what failed), so the - // sweep does not race this removal; both are conditional on CREATING - // in any case. + // stay taken. The job record is marked FAILED first, conditioned on + // the generation and on RUNNING; only if that matched is the target + // removed (which cascades the record). A record another generation + // owns, or a failed FAILED write (on a pooled connection, not the + // claim's), means the claim holder owns the target, and nothing + // further is touched. + let failed = self + .fail_restore_job_as(&desc.table_id, owner.generation(), &e.to_string()) + .await; + match failed { + Ok(true) => {} + Ok(false) => { + tracing::warn!( + "restore of {backup_arn} into {table_name} (account {account_id}) ended \ + with \"{e}\"; its job record was no longer RUNNING under this claim, so \ + the target ({}) was left to its owner", + desc.table_id + ); + drop(owner); + return; + } + Err(write) => { + tracing::error!( + "restore of {backup_arn} into {table_name} (account {account_id}) failed \ + ({e}), and its job record could not be marked FAILED ({write}); the \ + reconciler removes the target once its heartbeat is stale" + ); + drop(owner); + return; + } + } match self.abort_restore(&desc.table_id).await { Ok(true) => tracing::error!( "restore of {backup_arn} into {table_name} (account {account_id}) failed and \ the partial table was removed: {e}" ), - // Already claimed by the sweep (whose own removal is logged), or - // the activation committed before the copy was stopped. + // Already claimed by the reconciler (whose own removal is logged), + // or the activation committed before the copy was stopped. Ok(false) => tracing::warn!( "restore of {backup_arn} into {table_name} (account {account_id}) ended with \ \"{e}\"; the target ({}) was no longer CREATING, so it was left to its \ @@ -963,8 +952,8 @@ impl PostgresEngine { Err(cleanup) => tracing::error!( "restore of {backup_arn} into {table_name} (account {account_id}) failed ({e}), \ and the partial table ({}) could not be removed yet; the control plane (if it \ - reached DELETING) or the abandoned-restore sweep (if still CREATING) finishes \ - the removal: {cleanup}", + reached DELETING) or the reconciler (if still CREATING, once the FAILED record \ + is older than stale_job_after_secs) finishes the removal: {cleanup}", desc.table_id ), } @@ -972,17 +961,21 @@ impl PostgresEngine { } } -impl BackupEngine for PostgresEngine { - fn create_backup( +impl LegacyBackupReader for PostgresEngine { + fn create( &self, account_id: &str, table_name: &str, backup_name: &str, + permit: JobPermit, ) -> BoxFuture<'_, Result> { let account_id = account_id.to_string(); let table_name = table_name.to_string(); let backup_name = backup_name.to_string(); Box::pin(async move { + // The slot is the create's for its whole run; the copy is + // synchronous, so it is given back when this returns. + let _permit = permit; // The table's definition and its items must describe one instant. // They live in different databases, so no single snapshot covers // both. Instead the table row is held FOR SHARE from the first @@ -1265,7 +1258,7 @@ impl BackupEngine for PostgresEngine { }) } - fn describe_backup( + fn describe( &self, account_id: &str, backup_arn: &str, @@ -1349,7 +1342,7 @@ impl BackupEngine for PostgresEngine { }) } - fn list_backups( + fn list( &self, account_id: &str, table_name: Option<&str>, @@ -1425,7 +1418,7 @@ impl BackupEngine for PostgresEngine { }) } - fn delete_backup( + fn delete( &self, account_id: &str, backup_arn: &str, @@ -1435,7 +1428,7 @@ impl BackupEngine for PostgresEngine { Box::pin(async move { // Resolves account-scoped, so a backup owned by another account is // reported missing here and the writes below never run. - let desc = self.describe_backup(&account_id, &backup_arn).await?; + let desc = self.describe(&account_id, &backup_arn).await?; // One transaction, so a restore reading its snapshot sees either // the whole backup or a DELETED one, never AVAILABLE with its rows @@ -1455,18 +1448,10 @@ impl BackupEngine for PostgresEngine { .execute(&mut *tx) .await .map_err(db_err)?; - let restoring: Option = sqlx::query_scalar( - "SELECT t.table_name FROM table_restores r JOIN tables t ON t.table_id = r.table_id \ - WHERE r.source_backup_arn = $1 AND t.table_status = 'CREATING' LIMIT 1", - ) - .bind(&backup_arn) - .fetch_optional(&mut *tx) - .await - .map_err(db_err)?; - if let Some(table) = restoring { - return Err(StorageError::BackupInUse(format!( - "Backup is being used to restore table {table}: {backup_arn}" - ))); + if let Some(table) = + crate::backup_catalog::creating_restore_target(&mut *tx, &backup_arn).await? + { + return Err(crate::backup_catalog::backup_in_use(&table, &backup_arn)); } sqlx::query( "DELETE FROM backup_items WHERE backup_arn = $1 AND EXISTS (\ @@ -1510,7 +1495,7 @@ impl BackupEngine for PostgresEngine { }) } - fn backup_restore_source( + fn restore_source( &self, account_id: &str, backup_arn: &str, @@ -1628,32 +1613,40 @@ impl BackupEngine for PostgresEngine { /// runs in a detached task. /// /// The request hands everything to a `tokio::spawn`ed task that owns its - /// own clones: wait for a slot of its account's - /// ([`RESTORE_SLOTS_PER_ACCOUNT`]) and then a slot of the server's - /// ([`restore_server_slot_count`]), take the restore lock, create the - /// target and its `table_restores` row in one catalog transaction (which - /// re-checks the backup is AVAILABLE and owned by the account, holding it - /// FOR UPDATE), report that result back through a channel, then copy. - /// Dropping the request future at any point therefore changes nothing: - /// a restore refused for want of a slot creates nothing, the - /// registration still commits or fails on its own, and a registered - /// target is still copied and flipped ACTIVE, or removed on failure. - fn restore_table_from_backup( + /// own clones: take the restore lock, create the target and its + /// `table_restores` row (the job record, RUNNING under the claim's + /// generation) in one catalog transaction (which re-checks the backup is + /// AVAILABLE and owned by the account, holding it FOR UPDATE), report + /// that result back through a channel, then copy. Dropping the request + /// future at any point therefore changes nothing: the registration still + /// commits or fails on its own, and a registered target is still copied + /// and flipped ACTIVE, or removed on failure. + fn restore( &self, account_id: &str, request: RestoreRequest, + permit: JobPermit, ) -> BoxFuture<'_, Result> { let account_id = account_id.to_string(); Box::pin(async move { let (registered_tx, registered_rx) = tokio::sync::oneshot::channel(); let engine = self.clone(); - tokio::spawn(engine.run_restore(account_id, request, registered_tx)); + tokio::spawn(engine.run_restore(account_id, request, permit, registered_tx)); registered_rx.await.map_err(|_| { StorageError::Internal("the restore task ended before registering".to_owned()) })? }) } + /// The in-catalog create commits its row AVAILABLE in one transaction + /// with the items, so it never leaves a CREATING row. + fn abandoned_creating_backups( + &self, + _stale_after: Duration, + ) -> BoxFuture<'_, Result, StorageError>> { + Box::pin(async { Ok(Vec::new()) }) + } + fn describe_continuous_backups( &self, account_id: &str, @@ -1759,7 +1752,6 @@ impl BackupEngine for PostgresEngine { #[cfg(test)] mod tests { use std::borrow::Cow; - use std::time::Duration; use super::*; @@ -1804,10 +1796,14 @@ mod tests { #[test] fn only_too_many_connections_is_limit_exceeded() { - assert!(matches!( - lock_connection_error(&reply(Some("53300"))), - StorageError::LimitExceeded(_) - )); + match lock_connection_error(&reply(Some("53300"))) { + StorageError::LimitExceeded(m) => assert_eq!( + m, + limit_exceeded_message(JobBound::ServerConnections), + "the one job-refusal text, naming the server's connection slots" + ), + other => panic!("too_many_connections is LimitExceeded: {other:?}"), + } // invalid_password, invalid_catalog_name, cannot_connect_now, // insufficient_resources (53000, same class), and no SQLSTATE. for code in [ @@ -1893,91 +1889,4 @@ mod tests { ); } } - - #[test] - fn the_server_bound_is_half_the_pool_within_one_and_sixteen() { - assert_eq!(restore_server_slot_count(0), 1); - assert_eq!(restore_server_slot_count(1), 1); - assert_eq!(restore_server_slot_count(3), 1); - assert_eq!(restore_server_slot_count(10), 5); - assert_eq!(restore_server_slot_count(20), 10); - assert_eq!(restore_server_slot_count(32), 16); - assert_eq!(restore_server_slot_count(33), 16); - assert_eq!(restore_server_slot_count(1000), 16); - } - - /// A full server bound refuses with LimitExceeded naming the server, a - /// full account with LimitExceeded naming the account; a free slot of - /// each is granted, and releasing a held one lets the next restore in. - #[tokio::test] - async fn restore_permits_wait_then_limit_exceeded() { - let wait = Duration::from_millis(50); - let account = Arc::new(Semaphore::new(RESTORE_SLOTS_PER_ACCOUNT)); - let server = Arc::new(Semaphore::new(1)); - - let held = acquire_restore_permits(Arc::clone(&account), Arc::clone(&server), 1, wait) - .await - .expect("the first restore gets both slots"); - let other_account = Arc::new(Semaphore::new(RESTORE_SLOTS_PER_ACCOUNT)); - match acquire_restore_permits(Arc::clone(&other_account), Arc::clone(&server), 1, wait) - .await - { - Err(StorageError::LimitExceeded(m)) => { - assert!(m.contains("on this server (at most 1"), "{m}"); - } - other => panic!("a full server bound must refuse: {:?}", other.err()), - } - // The refused restore gave its account slot back. - assert_eq!(other_account.available_permits(), RESTORE_SLOTS_PER_ACCOUNT); - drop(held); - acquire_restore_permits(Arc::clone(&other_account), Arc::clone(&server), 1, wait) - .await - .expect("a freed server slot is granted"); - - let full_account = Arc::new(Semaphore::new(0)); - let roomy_server = Arc::new(Semaphore::new(4)); - match acquire_restore_permits(full_account, Arc::clone(&roomy_server), 4, wait).await { - Err(StorageError::LimitExceeded(m)) => { - assert!(m.contains("for this account"), "{m}"); - } - other => panic!("a full account must refuse: {:?}", other.err()), - } - assert_eq!(roomy_server.available_permits(), 4); - } - - /// One engine's account slots are shared by its clones and by nothing - /// else: a second engine in the same process has its own full allowance - /// for the same account. - #[test] - fn account_restore_slots_belong_to_one_engine() { - let slots = RestoreAccountSlots::default(); - let held = slots - .for_account("123456789012") - .try_acquire_owned() - .expect("a free slot"); - assert_eq!( - slots - .clone() - .for_account("123456789012") - .available_permits(), - RESTORE_SLOTS_PER_ACCOUNT - 1, - "a clone sees the same account map" - ); - assert_eq!( - RestoreAccountSlots::default() - .for_account("123456789012") - .available_permits(), - RESTORE_SLOTS_PER_ACCOUNT, - "another engine has its own" - ); - drop(held); - // Nothing in flight for the account: its entry is pruned on the next - // lookup of any account, and a fresh one has the full allowance. - slots.for_account("other"); - assert_eq!(slots.0.lock().unwrap().len(), 1); - assert_eq!( - slots.for_account("123456789012").available_permits(), - RESTORE_SLOTS_PER_ACCOUNT - ); - } } diff --git a/crates/storage-postgres/src/create_table.rs b/crates/storage-postgres/src/create_table.rs index daabe324e..7e4098a82 100755 --- a/crates/storage-postgres/src/create_table.rs +++ b/crates/storage-postgres/src/create_table.rs @@ -21,6 +21,9 @@ pub(crate) struct RestoreRegistration<'a> { /// Reported as `RestoreSummary.RestoreDateTime`, epoch seconds: the value /// the engine passes (the source backup's creation time), not "now". pub(crate) restore_date_time: f64, + /// The job claim's fencing token, written on the job record (RUNNING, + /// `kind = legacy`) with the target. + pub(crate) owner_generation: u64, } impl PostgresEngine { @@ -167,18 +170,10 @@ impl PostgresEngine { // One restore of a backup at a time, as the service: while a // target restored from it is CREATING the backup is in use. Same // predicate as DeleteBackup's. - let restoring: Option = sqlx::query_scalar( - "SELECT t.table_name FROM table_restores r JOIN tables t ON t.table_id = r.table_id \ - WHERE r.source_backup_arn = $1 AND t.table_status = 'CREATING' LIMIT 1", - ) - .bind(backup_arn) - .fetch_optional(&mut *tx) - .await - .map_err(|e| StorageError::Internal(e.to_string()))?; - if let Some(table) = restoring { - return Err(StorageError::BackupInUse(format!( - "Backup is being used to restore table {table}: {backup_arn}" - ))); + if let Some(table) = + crate::backup_catalog::creating_restore_target(&mut *tx, backup_arn).await? + { + return Err(crate::backup_catalog::backup_in_use(&table, backup_arn)); } } @@ -242,15 +237,20 @@ impl PostgresEngine { }; if let Some(r) = restore { + // The job record: RUNNING under the claim's generation, with a + // heartbeat on the catalog's clock, so no CREATING target ever + // exists without an owned record. sqlx::query( "INSERT INTO table_restores \ - (table_id, source_backup_arn, source_table_arn, restore_date_time) \ - VALUES ($1, $2, $3, to_timestamp($4))", + (table_id, source_backup_arn, source_table_arn, restore_date_time, \ + status, kind, owner_generation, heartbeat_at) \ + VALUES ($1, $2, $3, to_timestamp($4), 'RUNNING', 'legacy', $5, now())", ) .bind(&table_id) .bind(r.backup_arn) .bind(r.source_table_arn) .bind(r.restore_date_time) + .bind(crate::backup_engine::generation_column(r.owner_generation)) .execute(&mut *tx) .await .map_err(|e| StorageError::Internal(e.to_string()))?; diff --git a/crates/storage-postgres/src/delete_table.rs b/crates/storage-postgres/src/delete_table.rs index 81b75856b..6688b5caf 100755 --- a/crates/storage-postgres/src/delete_table.rs +++ b/crates/storage-postgres/src/delete_table.rs @@ -94,7 +94,7 @@ impl PostgresEngine { // filled by RestoreTableFromBackup. The service refuses to delete a // table it is still creating, and deleting this one would make the // restore fail part-way; refuse until the restore finishes. An - // abandoned target is removed by the restore sweep, not here. + // abandoned target is removed by the reconciler, not here. if row.table_status == "CREATING" { let restoring: bool = sqlx::query_scalar( "SELECT EXISTS(SELECT 1 FROM tables WHERE table_id = $1 \ diff --git a/crates/storage-postgres/src/lib.rs b/crates/storage-postgres/src/lib.rs index dfda9d9eb..9f2849477 100755 --- a/crates/storage-postgres/src/lib.rs +++ b/crates/storage-postgres/src/lib.rs @@ -10,6 +10,7 @@ mod admin_store; mod authorization_store; +mod backup_catalog; mod backup_engine; mod bootstrapper; mod catalog_store; @@ -39,6 +40,8 @@ mod vector_search; mod worker_store; mod workers; +#[doc(hidden)] +pub use backup_engine::RESTORE_LOCK_SEED; pub use bootstrapper::PostgresBootstrapper; pub use catalog_store::PostgresCatalogStore; pub use config::PostgresStorageConfig; @@ -261,15 +264,6 @@ pub struct PostgresEngine { /// the search path exists, so this backend still refuses vector operations /// over the wire; the flag is what that decision will read. pub(crate) vector_capable: bool, - /// Restores this engine (one per server process) may have in flight - /// across all accounts; see `backup_engine::restore_server_slot_count`. - pub(crate) restore_server_slots: Arc, - /// The number of permits `restore_server_slots` was created with, for - /// the refusal message. - pub(crate) restore_server_slot_count: usize, - /// Restores each account may run at once in this process; see - /// `backup_engine::RestoreAccountSlots`. - pub(crate) restore_account_slots: backup_engine::RestoreAccountSlots, /// The serving registration a server holds for its life, so `migrate /// --yes` refuses while it serves. `None` outside a server. pub(crate) serving: Option>, @@ -289,9 +283,10 @@ pub struct PostgresEngine { /// /// What the permits do not prevent: request load alone exhausting a pool /// (an open then waits for its connection up to the acquire timeout like - /// any request), and restores, which take their own slots (half the - /// pool, at most 16: `backup_engine::restore_server_slot_count`), so - /// snapshots and restores together can hold the whole pool. On the separate layout requests that block on + /// any request), and restores, which hold no snapshot slot (their bound + /// is the engine's `max_concurrent_jobs`, which the pool is raised to + /// twice plus ten), so snapshots and restores together can hold the + /// whole pool. On the separate layout requests that block on /// the `tables` row an open holds `FOR SHARE` (`UpdateTable`, /// `DeleteTable` of that table) hold catalog connections while they /// wait; the open needs only a data connection by then, so they delay it @@ -412,7 +407,6 @@ impl PostgresEngine { ), } - let server_slots = backup_engine::restore_server_slot_count(pool_size); Ok(Self { pool, data_pool, @@ -424,9 +418,6 @@ impl PostgresEngine { initial_gsi_delay, )), vector_capable, - restore_server_slots: Arc::new(tokio::sync::Semaphore::new(server_slots)), - restore_server_slot_count: server_slots, - restore_account_slots: backup_engine::RestoreAccountSlots::default(), serving: None, snapshot_open_slots: Arc::new(tokio::sync::Semaphore::new(snapshot_slot_count( pool_size, @@ -850,6 +841,24 @@ async fn register_and_check( } } +/// A pool size raised to what `max_concurrent_jobs` jobs need, when it is +/// smaller (`pool_size_for_jobs`: twice the jobs plus ten, so a snapshot +/// bound of half the pool admits every job the permit allows; the job's +/// claim connection is outside the pool). A raise is logged at warn, +/// naming both numbers, as the clamp to [`MIN_POOL_SIZE`] is; a size at or +/// above the floor is returned as configured, and the minimum clamp still +/// applies to it. +fn raise_pool_size_for_jobs(key: &str, configured: u32, floor: u32, jobs: usize) -> u32 { + if configured >= floor || floor <= MIN_POOL_SIZE { + return configured; + } + tracing::warn!( + "{key} = {configured} is below the {floor} connections backup.max_concurrent_jobs = \ + {jobs} needs (2 * max_concurrent_jobs + 10); raising it to {floor}" + ); + floor +} + /// Build server components for the Postgres backend (registered in [`register`]). fn server_components_factory( config: &(dyn extenddb_storage::config::StorageConfig + 'static), @@ -857,13 +866,27 @@ fn server_components_factory( // PostgreSQL bootstrap needs operator input (databases, roles, admin // credentials), so `bootstrap_if_uninitialized` is not honored here: // an uninitialized catalog fails with the explicit-`init` guidance. - _options: extenddb_storage::server_components::ServerComponentsOptions, + options: extenddb_storage::server_components::ServerComponentsOptions, ) -> std::pin::Pin< Box> + Send>, > { let connection_string = config.connection_config().to_string(); - let max_connections = config.max_connections(); - let max_catalog_connections = config.max_catalog_connections(); + // Both pools admit the jobs the instance may run at once (the rule is + // `pool_size_for_jobs`'s). + let job_floor = + extenddb_storage::backup_catalog::pool_size_for_jobs(options.max_concurrent_jobs); + let max_connections = raise_pool_size_for_jobs( + "storage.postgres.pool_size", + config.max_connections(), + job_floor, + options.max_concurrent_jobs, + ); + let max_catalog_connections = raise_pool_size_for_jobs( + "storage.postgres.catalog_pool_size", + config.max_catalog_connections(), + job_floor, + options.max_concurrent_jobs, + ); let region = region.to_string(); Box::pin(async move { let registration = register_and_check(&connection_string) @@ -981,3 +1004,25 @@ fn server_components_factory( }) }) } + +#[cfg(test)] +mod pool_size_tests { + use super::{MIN_POOL_SIZE, raise_pool_size_for_jobs}; + use extenddb_storage::backup_catalog::pool_size_for_jobs; + + #[test] + fn a_pool_under_the_job_floor_is_raised_and_one_at_it_is_kept() { + let floor = pool_size_for_jobs(4); + assert_eq!(floor, 18); + assert_eq!(raise_pool_size_for_jobs("pool_size", 12, floor, 4), 18); + assert_eq!(raise_pool_size_for_jobs("pool_size", 18, floor, 4), 18); + assert_eq!(raise_pool_size_for_jobs("pool_size", 20, floor, 4), 20); + // No jobs asked for: the floor is the minimum, which the minimum + // clamp applies on its own. + assert_eq!( + raise_pool_size_for_jobs("pool_size", 5, pool_size_for_jobs(0), 0), + 5 + ); + assert!(pool_size_for_jobs(0) <= MIN_POOL_SIZE); + } +} diff --git a/crates/storage-postgres/src/migrations/mod.rs b/crates/storage-postgres/src/migrations/mod.rs index f1fc5932c..1aa3cb2c7 100644 --- a/crates/storage-postgres/src/migrations/mod.rs +++ b/crates/storage-postgres/src/migrations/mod.rs @@ -111,6 +111,15 @@ pub fn migration(id: MigrationId) -> Migration { m0004_index_table_base_key::SOURCE, )], }, + MigrationId::BackupCatalogIndex => Migration::Applies { + kind: Kind::Breaking, + convergent: true, + bodies: vec![file_body!( + StoreKind::Catalog, + "migrations", + "0005_backup_catalog_index.sql" + )], + }, } } @@ -131,7 +140,8 @@ pub(crate) const fn runs_in(id: MigrationId) -> RunsIn { match id { MigrationId::Baseline | MigrationId::ResourceTagConditionKeys - | MigrationId::SortKeyDefinitions => RunsIn::Transaction, + | MigrationId::SortKeyDefinitions + | MigrationId::BackupCatalogIndex => RunsIn::Transaction, MigrationId::IndexTableBaseKey => RunsIn::Autocommit, } } diff --git a/crates/storage-postgres/src/snapshot_engine.rs b/crates/storage-postgres/src/snapshot_engine.rs index abb066cf3..3fbf6f83f 100644 --- a/crates/storage-postgres/src/snapshot_engine.rs +++ b/crates/storage-postgres/src/snapshot_engine.rs @@ -63,8 +63,8 @@ //! pool's 30 s acquire timeout). The `snapshot_open_slots` field doc says //! what the permits do not prevent. An open over the bound is refused with //! `LimitExceeded` rather than queued. The backup design's -//! `max_concurrent_jobs` is the caller-side bound; the restore path's slots -//! (`restore_server_slot_count`) do not count snapshots. +//! `max_concurrent_jobs` is the caller-side bound on jobs; a restore holds no +//! snapshot slot. //! //! `ACCESS SHARE` on the data table is held until the stream's last fetch //! (the transaction commits as soon as a short page shows the end), not diff --git a/crates/storage-postgres/src/worker_store.rs b/crates/storage-postgres/src/worker_store.rs index e24f8f38e..01145fb60 100755 --- a/crates/storage-postgres/src/worker_store.rs +++ b/crates/storage-postgres/src/worker_store.rs @@ -176,24 +176,6 @@ impl PostgresEngine { .await .map_err(|e| StorageError::Internal(e.to_string()))?; - // CREATING with no owner → removed. A restore whose process died - // mid-copy leaves its target CREATING with no scheduled transition. - // The poller is notification-driven with a 60-second idle timeout, so - // the 60-second grace means an idle abandoned target is removed about - // 60–120 seconds after ownership is lost. Every pass opens a fresh - // out-of-pool connection for each old candidate to probe its advisory - // lock; a still-owned restore closes that probe and is checked again on - // the next pass. Failures are logged and skipped, so they cannot hold - // up the transitions above. - match self.sweep_abandoned_restores().await { - Ok(names) => { - for name in names { - transitions.push((name, "abandoned restore → removed")); - } - } - Err(e) => tracing::warn!("abandoned restore sweep failed: {e}"), - } - Ok(ControlPlanePass { transitions, deferred_drops, diff --git a/crates/storage-postgres/tests/backup_restore.rs b/crates/storage-postgres/tests/backup_restore.rs index 5c0a6f588..f654f702f 100644 --- a/crates/storage-postgres/tests/backup_restore.rs +++ b/crates/storage-postgres/tests/backup_restore.rs @@ -14,7 +14,9 @@ //! - a backup with two rows for one key must fail rather than collapse them; //! - a backup of a multi-part key table (a preview gated by //! `enable_multipart_keys`) must be refused, not restored wrongly; -//! - the abandoned-restore sweep, which needs a crash to reach. +//! - the reconciler's duties on a dead or failed restore, which need a crash +//! to reach through the server and are driven here through the +//! `BackupCatalog` methods the reconciler calls. //! //! Each test builds a throwaway catalog and data database and drops them when //! it ends, passing or failing. The tests run one at a time: each opens its @@ -31,7 +33,7 @@ use extenddb_core::validation::validate_create_table; use extenddb_storage::error::StorageError; use extenddb_storage::migrations::Runner; use extenddb_storage::{ - BackupEngine, DataEngine, RestoreRequest, RestoreTableOverrides, TableEngine, + DataEngine, JobPermit, LegacyBackupReader, RestoreRequest, RestoreTableOverrides, TableEngine, }; use extenddb_storage_postgres::migrations::{PostgresMigrations, PostgresTarget}; use extenddb_storage_postgres::{PostgresConfig, PostgresEngine}; @@ -40,13 +42,10 @@ use sqlx::PgPool; use sqlx::postgres::PgPoolOptions; const ACCOUNT: &str = "123456789012"; -/// A second account, for the per-account restore slots. +/// A second account, for the cross-account checks. const OTHER_ACCOUNT: &str = "210987654321"; const REGION: &str = "us-east-1"; -/// The seed of the restore advisory-lock key (`backup_engine.rs`). -const RESTORE_LOCK_SEED: i64 = 0x0045_4452; -/// Restore slots per account (`backup_engine.rs`). -const RESTORE_SLOTS_PER_ACCOUNT: usize = 4; +use extenddb_storage_postgres::RESTORE_LOCK_SEED; /// A catalog database and a separate data database, the layout `extenddb /// init` creates. They must be separate here: the catalog schema carries a @@ -276,7 +275,7 @@ async fn legacy_backup_of( .expect("create the source table"); let backup = s .engine - .create_backup(ACCOUNT, &source_name, "legacy") + .create(ACCOUNT, &source_name, "legacy", JobPermit::detached()) .await .expect("back up the empty source"); for item in rows { @@ -327,7 +326,7 @@ async fn restore_request( target: &str, backup_arn: &str, ) -> Result { - let source = s.engine.backup_restore_source(account, backup_arn).await?; + let source = s.engine.restore_source(account, backup_arn).await?; source.ensure_restorable()?; let input = source.restore_input(target, &RestoreTableOverrides::default()); validate_create_table(&input, &LimitsConfig::default()) @@ -349,7 +348,10 @@ async fn start_restore_in( backup_arn: &str, ) -> Result { let request = restore_request(s, account, target, backup_arn).await?; - let desc = s.engine.restore_table_from_backup(account, request).await?; + let desc = s + .engine + .restore(account, request, JobPermit::detached()) + .await?; assert_eq!( desc.table_status, TableStatus::Creating, @@ -437,16 +439,28 @@ async fn await_copies_waiting(s: &Scratch, n: i64) { /// The backend pid holding the restore lock for `account`/`table`, if any. async fn restore_lock_holder(s: &Scratch, account: &str, table: &str) -> Option { + advisory_lock_holder(s, &format!("{account}/{table}")).await +} + +/// The backend pid holding the claim lock of backup `backup_id` +/// (`backup:` under the shared seed), if any. +async fn backup_lock_holder(s: &Scratch, backup_id: &str) -> Option { + advisory_lock_holder(s, &format!("backup:{backup_id}")).await +} + +/// The backend pid holding the advisory lock `hashtextextended(key, +/// RESTORE_LOCK_SEED)` on the catalog database, if any. +async fn advisory_lock_holder(s: &Scratch, key: &str) -> Option { sqlx::query_scalar( "SELECT pid FROM pg_locks WHERE locktype = 'advisory' AND granted \ AND database = (SELECT oid FROM pg_database WHERE datname = current_database()) \ AND ((classid::bigint << 32) | objid::bigint) = hashtextextended($1, $2)", ) - .bind(format!("{account}/{table}")) + .bind(key) .bind(RESTORE_LOCK_SEED) .fetch_optional(&s.catalog) .await - .expect("find the restore lock holder") + .expect("find the advisory lock holder") } /// Rows of `table_restores` for a target: (source backup ARN, source table @@ -826,7 +840,7 @@ async fn restore_recreates_indexes_and_throughput() { let src = indexed_source(&s, "idx_src").await; let backup = s .engine - .create_backup(ACCOUNT, "idx_src", "b") + .create(ACCOUNT, "idx_src", "b", JobPermit::detached()) .await .expect("backup"); let desc = restore(&s, "idx_dst", &backup.backup_arn) @@ -871,89 +885,118 @@ async fn restore_recreates_indexes_and_throughput() { s.cleanup().await; } +/// The reconciler's duty for a legacy restore whose owner died: a RUNNING +/// job record is dead once its heartbeat is older than the stale window on +/// the catalog's clock, whatever the table's age and whether or not the +/// advisory lock is free; a stale record is marked FAILED and its target +/// removed through `remove_restore_target`, with its data and index tables; +/// a fresh record, and a record that is no longer RUNNING, are left alone. #[tokio::test] -async fn abandoned_restore_is_removed_only_when_unowned_and_old() { +async fn abandoned_restore_is_removed_only_when_its_heartbeat_is_stale() { + use extenddb_storage::BackupCatalog; + use extenddb_storage::backup_catalog::{RestoreKind, RestoreTargetKey}; if base_conn().is_none() { - eprintln!("SKIP abandoned_restore_is_removed_only_when_unowned_and_old: no PostgreSQL"); + eprintln!( + "SKIP abandoned_restore_is_removed_only_when_its_heartbeat_is_stale: no PostgreSQL" + ); return; } + let stale = std::time::Duration::from_secs(300); let s = scratch().await; indexed_source(&s, "ab_src").await; let backup = s .engine - .create_backup(ACCOUNT, "ab_src", "b") + .create(ACCOUNT, "ab_src", "b", JobPermit::detached()) .await .expect("backup"); let desc = restore(&s, "ab_dst", &backup.backup_arn) .await .expect("restore"); assert_eq!(table_status(&s, "ab_dst").await.as_deref(), Some("ACTIVE")); + assert_eq!( + job_status(&s, &desc.table_id).await.as_deref(), + Some("DONE"), + "a finished restore's record is DONE" + ); let indexes = index_ids(&s, &desc.table_id).await; + let key = RestoreTargetKey { + account_id: ACCOUNT.to_owned(), + table_name: "ab_dst".to_owned(), + table_id: desc.table_id.clone(), + }; // The state a process killed mid-copy leaves: the target CREATING with no - // scheduled transition, some of its copy's batches committed. + // scheduled transition, its job record RUNNING with a heartbeat `age` + // old, some of its copy's batches committed. let crash = |age: &'static str| { let catalog = s.catalog.clone(); let id = desc.table_id.clone(); async move { + sqlx::query( + "UPDATE tables SET table_status = 'CREATING', status_transition_at = NULL \ + WHERE table_id = $1", + ) + .bind(&id) + .execute(&catalog) + .await + .expect("simulate a crash"); sqlx::query(&format!( - "UPDATE tables SET table_status = 'CREATING', status_transition_at = NULL, \ - creation_date_time = NOW() - INTERVAL '{age}' WHERE table_id = $1" + "UPDATE table_restores SET status = 'RUNNING', completed_at = NULL, \ + heartbeat_at = NOW() - INTERVAL '{age}' WHERE table_id = $1" )) .bind(&id) .execute(&catalog) .await - .expect("simulate a crash"); + .expect("age the heartbeat"); } }; - // Inside the grace period: a restore that has not taken its lock yet. + // A fresh heartbeat: a live restore, here or on another instance, even + // though the table itself is old. crash("1 second").await; - s.engine - .process_control_plane_transitions() + sqlx::query("UPDATE tables SET creation_date_time = NOW() - INTERVAL '10 minutes'") + .execute(&s.catalog) .await - .expect("sweep"); + .expect("age every table"); + assert!( + s.engine + .list_dead_restore_jobs(stale) + .await + .expect("list") + .is_empty() + ); assert_eq!( table_status(&s, "ab_dst").await.as_deref(), Some("CREATING") ); - // Old, but its lock is held: a live restore on another instance. The key - // is the account and name, so the lock covers the target before it exists. + // A stale heartbeat: dead, whether or not its lock session is still + // around (a free lock shows the session ended, not that the process did, + // and the record's heartbeat is what the reconciler waits on). crash("10 minutes").await; - let mut owner = s.catalog.acquire().await.expect("connection"); - sqlx::query("SELECT pg_advisory_lock(hashtextextended($2, $1))") - .bind(0x0045_4452_i64) - .bind(format!("{ACCOUNT}/ab_dst")) - .execute(&mut *owner) - .await - .expect("hold the restore lock"); - s.engine - .process_control_plane_transitions() - .await - .expect("sweep"); + let dead = s.engine.list_dead_restore_jobs(stale).await.expect("list"); + assert_eq!(dead.len(), 1, "{dead:?}"); + assert_eq!(dead[0].target, key); + assert_eq!(dead[0].kind, RestoreKind::Legacy); + assert_eq!(dead[0].source_backup_arn, backup.backup_arn); + assert!( + s.engine + .fail_restore_job(&key, "owner dead") + .await + .expect("fail") + ); assert_eq!( - table_status(&s, "ab_dst").await.as_deref(), - Some("CREATING") + job_status(&s, &desc.table_id).await.as_deref(), + Some("FAILED") ); - sqlx::query("SELECT pg_advisory_unlock(hashtextextended($2, $1))") - .bind(0x0045_4452_i64) - .bind(format!("{ACCOUNT}/ab_dst")) - .execute(&mut *owner) - .await - .expect("release the restore lock"); - drop(owner); - - // Old and unowned: removed, with its data and index tables. - let transitions = s - .engine - .process_control_plane_transitions() - .await - .expect("sweep"); assert!( - transitions.iter().any(|(n, _)| n == "ab_dst"), - "{transitions:?}" + !s.engine + .fail_restore_job(&key, "owner dead") + .await + .expect("fail again"), + "a record that is no longer RUNNING is not written twice" ); + assert!(s.engine.remove_restore_target(&key).await.expect("remove")); assert_eq!(table_status(&s, "ab_dst").await, None); assert!(!data_table_exists(&s, &desc.table_id).await); for (_, id) in &indexes { @@ -962,24 +1005,64 @@ async fn abandoned_restore_is_removed_only_when_unowned_and_old() { "index table {id} left behind" ); } + assert!( + !s.engine + .remove_restore_target(&key) + .await + .expect("remove again"), + "a removed target is removed once" + ); - // An ACTIVE table is never a candidate, and the name is free again. - restore(&s, "ab_dst", &backup.backup_arn) + // An ACTIVE table is never a candidate, the name is free again, and a + // key with the wrong table id (a table re-created under the name) does + // not remove the new table. + let again = restore(&s, "ab_dst", &backup.backup_arn) .await .expect("restore again"); assert_eq!(table_status(&s, "ab_dst").await.as_deref(), Some("ACTIVE")); - sqlx::query("UPDATE tables SET creation_date_time = NOW() - INTERVAL '10 minutes'") - .execute(&s.catalog) - .await - .expect("age every table"); - s.engine - .process_control_plane_transitions() - .await - .expect("sweep"); - assert_eq!(table_status(&s, "ab_dst").await.as_deref(), Some("ACTIVE")); + assert!( + s.engine + .list_dead_restore_jobs(stale) + .await + .expect("list") + .is_empty() + ); + assert!( + !s.engine + .remove_restore_target(&key) + .await + .expect("stale key") + ); + sqlx::query( + "UPDATE tables SET table_status = 'CREATING', status_transition_at = NULL WHERE table_id = $1", + ) + .bind(&again.table_id) + .execute(&s.catalog) + .await + .expect("make it a target again"); + assert!( + !s.engine + .remove_restore_target(&key) + .await + .expect("stale key"), + "the old table id never removes the new table" + ); + assert_eq!( + table_status(&s, "ab_dst").await.as_deref(), + Some("CREATING") + ); s.cleanup().await; } +/// The job record's status column for a target. +async fn job_status(s: &Scratch, table_id: &str) -> Option { + sqlx::query_scalar("SELECT status FROM table_restores WHERE table_id = $1") + .bind(table_id) + .fetch_optional(&s.catalog) + .await + .expect("read the job status") +} + #[tokio::test] async fn failed_restore_with_indexes_leaves_no_tables() { if base_conn().is_none() { @@ -990,7 +1073,7 @@ async fn failed_restore_with_indexes_leaves_no_tables() { indexed_source(&s, "fi_src").await; let backup = s .engine - .create_backup(ACCOUNT, "fi_src", "b") + .create(ACCOUNT, "fi_src", "b", JobPermit::detached()) .await .expect("backup"); sqlx::query("INSERT INTO backup_items (backup_arn, pk, item_data) VALUES ($1, 'x', $2)") @@ -1049,7 +1132,7 @@ async fn restore_keeps_sse_and_on_demand_limits_but_not_table_class() { let src = s.engine.create_table(ACCOUNT, input).await.expect("create"); let backup = s .engine - .create_backup(ACCOUNT, "cls_src", "b") + .create(ACCOUNT, "cls_src", "b", JobPermit::detached()) .await .expect("backup"); restore(&s, "cls_dst", &backup.backup_arn) @@ -1113,7 +1196,7 @@ async fn restore_crosses_batch_boundaries() { .collect(); let backup = s .engine - .create_backup(ACCOUNT, "many_src", "b") + .create(ACCOUNT, "many_src", "b", JobPermit::detached()) .await .expect("backup"); sqlx::query( @@ -1156,12 +1239,12 @@ async fn restore_into_a_name_whose_restore_lock_is_held_is_refused() { indexed_source(&s, "lk_src").await; let backup = s .engine - .create_backup(ACCOUNT, "lk_src", "b") + .create(ACCOUNT, "lk_src", "b", JobPermit::detached()) .await .expect("backup"); let mut other = s.catalog.acquire().await.expect("connection"); sqlx::query("SELECT pg_advisory_lock(hashtextextended($2, $1))") - .bind(0x0045_4452_i64) + .bind(RESTORE_LOCK_SEED) .bind(format!("{ACCOUNT}/lk_dst")) .execute(&mut *other) .await @@ -1202,21 +1285,22 @@ async fn a_restore_is_refused_while_a_target_of_its_backup_is_creating() { indexed_source(&s, "iu_src").await; let b1 = s .engine - .create_backup(ACCOUNT, "iu_src", "b1") + .create(ACCOUNT, "iu_src", "b1", JobPermit::detached()) .await .expect("backup 1"); let b2 = s .engine - .create_backup(ACCOUNT, "iu_src", "b2") + .create(ACCOUNT, "iu_src", "b2", JobPermit::detached()) .await .expect("backup 2"); - // Backups for the restores that fill the account's other slots, taken - // before the gate (a backup writes `backup_items`, which the gate locks). + // Backups for the other restores that run alongside, taken before the + // gate (a backup writes `backup_items`, which the gate locks). + const MORE_RESTORES: usize = 4; let mut more_backups = Vec::new(); - for i in 3..=(RESTORE_SLOTS_PER_ACCOUNT + 1) { + for i in 3..=(MORE_RESTORES + 1) { more_backups.push( s.engine - .create_backup(ACCOUNT, "iu_src", &format!("b{i}")) + .create(ACCOUNT, "iu_src", &format!("b{i}"), JobPermit::detached()) .await .expect("backup") .backup_arn, @@ -1249,9 +1333,31 @@ async fn a_restore_is_refused_while_a_target_of_its_backup_is_creating() { } other => panic!("a restore into a CREATING restore target: {other:?}"), } - // The name is checked before the backup and before a slot is taken: the - // same backup into the same name, and a backup that does not exist into - // it, are refused as the name, naming the target's source backup. + // What the engine's own name check reads before its job permit: the + // target describes as CREATING with its source backup in the + // RestoreSummary, which is what it answers TableInUse with. + let described = s + .engine + .describe_table( + ACCOUNT, + DescribeTableInput { + table_name: "iu_dst".to_owned(), + }, + ) + .await + .expect("describe the CREATING target"); + assert_eq!(described.table_status, TableStatus::Creating); + assert_eq!( + described + .restore_summary + .as_ref() + .and_then(|r| r.source_backup_arn.as_deref()), + Some(b1.backup_arn.as_str()), + "the CREATING target names its source backup" + ); + // The name is checked before the backup: the same backup into the same + // name, and a backup that does not exist into it, are refused as the + // name, naming the target's source backup. let missing_arn = flip_last_hex(&b2.backup_arn); let mut missing = restore_request(&s, ACCOUNT, "iu_dst", &b2.backup_arn) .await @@ -1264,7 +1370,9 @@ async fn a_restore_is_refused_while_a_target_of_its_backup_is_creating() { ), ( "a missing backup into a CREATING restore target", - s.engine.restore_table_from_backup(ACCOUNT, missing).await, + s.engine + .restore(ACCOUNT, missing, JobPermit::detached()) + .await, ), ] { match result { @@ -1280,23 +1388,23 @@ async fn a_restore_is_refused_while_a_target_of_its_backup_is_creating() { } // Only the first copy is waiting: the refused restores never started one. assert_eq!(copies_waiting(&s).await, 1); - // With every account slot taken, a client retrying its own restore is - // still answered at once, not after the 30 s slot wait. + // With several copies running, a client retrying its own restore is + // still answered at once. let mut more = Vec::new(); for (i, backup) in more_backups.iter().enumerate() { let target = format!("iu_more{i}"); start_restore(&s, &target, backup) .await - .expect("a restore within the account's slots"); + .expect("a restore alongside the first"); more.push(target); } - await_copies_waiting(&s, i64::try_from(RESTORE_SLOTS_PER_ACCOUNT).unwrap()).await; + await_copies_waiting(&s, i64::try_from(MORE_RESTORES).unwrap()).await; let retried = tokio::time::timeout( std::time::Duration::from_secs(5), start_restore(&s, "iu_dst", &b1.backup_arn), ) .await - .expect("a retry into a CREATING restore target does not wait for a slot"); + .expect("a retry into a CREATING restore target is answered at once"); assert!( matches!(&retried, Err(StorageError::TableBeingRestored { table_name, .. }) if table_name == "iu_dst"), "{retried:?}" @@ -1318,6 +1426,19 @@ async fn a_restore_is_refused_while_a_target_of_its_backup_is_creating() { matches!(&taken, Err(StorageError::TableAlreadyExists(n)) if n == "iu_dst"), "{taken:?}" ); + // The former target describes as ACTIVE, so the engine's name check + // answers TableAlreadyExists for it too. + let former = s + .engine + .describe_table( + ACCOUNT, + DescribeTableInput { + table_name: "iu_dst".to_owned(), + }, + ) + .await + .expect("describe the former target"); + assert_eq!(former.table_status, TableStatus::Active); s.cleanup().await; } @@ -1334,7 +1455,7 @@ async fn simultaneous_restores_of_one_backup_admit_exactly_one() { indexed_source(&s, "sim_src").await; let backup = s .engine - .create_backup(ACCOUNT, "sim_src", "b") + .create(ACCOUNT, "sim_src", "b", JobPermit::detached()) .await .expect("backup"); let gate = gate_copies(&s).await; @@ -1349,7 +1470,7 @@ async fn simultaneous_restores_of_one_backup_admit_exactly_one() { let results = futures::future::join_all( requests .into_iter() - .map(|request| s.engine.restore_table_from_backup(ACCOUNT, request)), + .map(|request| s.engine.restore(ACCOUNT, request, JobPermit::detached())), ) .await; let accepted: Vec = results @@ -1388,7 +1509,7 @@ async fn pre_0_0_4_backup_restores_after_migrating() { indexed_source(&s, "old_src").await; let backup = s .engine - .create_backup(ACCOUNT, "old_src", "b") + .create(ACCOUNT, "old_src", "b", JobPermit::detached()) .await .expect("backup"); sqlx::query("DROP TABLE backup_definitions") @@ -1416,7 +1537,7 @@ async fn pre_0_0_4_backup_restores_after_migrating() { // definitions; otherwise validate_create_table would refuse it. let source = s .engine - .backup_restore_source(ACCOUNT, &backup.backup_arn) + .restore_source(ACCOUNT, &backup.backup_arn) .await .expect("legacy source"); assert!(source.definition.is_none()); @@ -1474,7 +1595,9 @@ async fn production_create_backup_excludes_item_committed_after_snapshot_pin() { .await .expect("block CreateBackup's catalog insert"); - let mut backup_fut = s.engine.create_backup(ACCOUNT, "prod_snap", "b"); + let mut backup_fut = s + .engine + .create(ACCOUNT, "prod_snap", "b", JobPermit::detached()); let early = tokio::time::timeout(std::time::Duration::from_secs(2), &mut backup_fut).await; assert!( early.is_err(), @@ -1526,7 +1649,7 @@ async fn delete_backup_winning_before_restore_leaves_no_target() { snapshot_source(&s, "delete_first_src").await; let backup = s .engine - .create_backup(ACCOUNT, "delete_first_src", "b") + .create(ACCOUNT, "delete_first_src", "b", JobPermit::detached()) .await .expect("backup"); @@ -1537,7 +1660,7 @@ async fn delete_backup_winning_before_restore_leaves_no_target() { .execute(&mut *gate) .await .expect("block DeleteBackup after its row lock"); - let mut delete_fut = s.engine.delete_backup(ACCOUNT, &backup.backup_arn); + let mut delete_fut = s.engine.delete(ACCOUNT, &backup.backup_arn); let early_delete = tokio::time::timeout(std::time::Duration::from_secs(2), &mut delete_fut).await; assert!(early_delete.is_err(), "DeleteBackup did not reach the gate"); @@ -1593,7 +1716,7 @@ async fn restore_registration_winning_before_delete_returns_backup_in_use() { snapshot_source(&s, "restore_first_src").await; let backup = s .engine - .create_backup(ACCOUNT, "restore_first_src", "b") + .create(ACCOUNT, "restore_first_src", "b", JobPermit::detached()) .await .expect("backup"); @@ -1625,7 +1748,7 @@ async fn restore_registration_winning_before_delete_returns_backup_in_use() { let err = s .engine - .delete_backup(ACCOUNT, &backup.backup_arn) + .delete(ACCOUNT, &backup.backup_arn) .await .expect_err("a registered in-progress restore keeps its backup in use"); assert!(matches!(err, StorageError::BackupInUse(_)), "{err:?}"); @@ -1806,7 +1929,9 @@ async fn backup_definition_and_items_share_one_instant() { // The backup must wait for the writer rather than read around it. let arn = { - let backup = s.engine.create_backup(ACCOUNT, "bar_src", "b"); + let backup = s + .engine + .create(ACCOUNT, "bar_src", "b", JobPermit::detached()); tokio::pin!(backup); let early = tokio::time::timeout(std::time::Duration::from_millis(500), &mut backup).await; assert!( @@ -1851,7 +1976,7 @@ async fn backup_leaves_out_a_gsi_still_being_built() { .expect("make the GSI look half-built"); let backup = s .engine - .create_backup(ACCOUNT, "unb_src", "b") + .create(ACCOUNT, "unb_src", "b", JobPermit::detached()) .await .expect("backup"); let def: serde_json::Value = @@ -1903,7 +2028,7 @@ async fn delete_table_and_delete_backup_are_refused_during_the_copy() { indexed_source(&s, "dt_src").await; let backup = s .engine - .create_backup(ACCOUNT, "dt_src", "b") + .create(ACCOUNT, "dt_src", "b", JobPermit::detached()) .await .expect("backup"); let gate = gate_copies(&s).await; @@ -1925,7 +2050,7 @@ async fn delete_table_and_delete_backup_are_refused_during_the_copy() { assert!(matches!(err, StorageError::IndexesInUse(_)), "{err:?}"); let err = s .engine - .delete_backup(ACCOUNT, &backup.backup_arn) + .delete(ACCOUNT, &backup.backup_arn) .await .expect_err("DeleteBackup of a backup being restored from is refused"); assert!(matches!(err, StorageError::BackupInUse(_)), "{err:?}"); @@ -1943,7 +2068,7 @@ async fn delete_table_and_delete_backup_are_refused_during_the_copy() { ); assert_eq!(data_rows(&s, &desc.table_id).await, 30); s.engine - .delete_backup(ACCOUNT, &backup.backup_arn) + .delete(ACCOUNT, &backup.backup_arn) .await .expect("deletable once the restore is done"); s.engine @@ -1972,7 +2097,7 @@ async fn restore_summary_and_backup_in_use() { indexed_source(&s, "rs_src").await; let backup = s .engine - .create_backup(ACCOUNT, "rs_src", "b") + .create(ACCOUNT, "rs_src", "b", JobPermit::detached()) .await .expect("backup"); let gate = gate_copies(&s).await; @@ -1996,7 +2121,7 @@ async fn restore_summary_and_backup_in_use() { }; let source = s .engine - .backup_restore_source(ACCOUNT, &backup.backup_arn) + .restore_source(ACCOUNT, &backup.backup_arn) .await .expect("source"); // (g) The registration records what the engine passed: the source table's @@ -2023,7 +2148,7 @@ async fn restore_summary_and_backup_in_use() { assert!(describe("rs_src").await.restore_summary.is_none()); let err = s .engine - .delete_backup(ACCOUNT, &backup.backup_arn) + .delete(ACCOUNT, &backup.backup_arn) .await .expect_err("in use"); assert!(matches!(err, StorageError::BackupInUse(_)), "{err:?}"); @@ -2036,7 +2161,7 @@ async fn restore_summary_and_backup_in_use() { let done = describe("rs_dst").await.restore_summary.expect("summary"); assert!(!done.restore_in_progress); s.engine - .delete_backup(ACCOUNT, &backup.backup_arn) + .delete(ACCOUNT, &backup.backup_arn) .await .expect("deletable once the restore is done"); let after = describe("rs_dst").await.restore_summary.expect("summary"); @@ -2111,7 +2236,7 @@ async fn restore_returns_creating_and_the_detached_copy_finishes() { let src = indexed_source(&s, "cr_src").await; let backup = s .engine - .create_backup(ACCOUNT, "cr_src", "b") + .create(ACCOUNT, "cr_src", "b", JobPermit::detached()) .await .expect("backup"); add_backup_rows(&s, &backup.backup_arn, &bulk_rows(1_200)).await; @@ -2188,14 +2313,14 @@ async fn a_dropped_restore_request_still_reaches_active() { indexed_source(&s, "drop_src").await; let backup = s .engine - .create_backup(ACCOUNT, "drop_src", "b") + .create(ACCOUNT, "drop_src", "b", JobPermit::detached()) .await .expect("backup"); add_backup_rows(&s, &backup.backup_arn, &bulk_rows(1_200)).await; let request = restore_request(&s, ACCOUNT, "drop_dst", &backup.backup_arn) .await .expect("request"); - let fut = s.engine.restore_table_from_backup(ACCOUNT, request); + let fut = s.engine.restore(ACCOUNT, request, JobPermit::detached()); assert!( fut.now_or_never().is_none(), "the request cannot have finished on its first poll" @@ -2218,211 +2343,13 @@ async fn a_dropped_restore_request_still_reaches_active() { s.cleanup().await; } -/// (c) Restore slots are per account: an account with all of its slots held -/// by running copies makes its own next restore wait, and does not hold back -/// another account's. -#[tokio::test] -async fn one_accounts_restores_do_not_block_another_account() { - if base_conn().is_none() { - eprintln!("SKIP one_accounts_restores_do_not_block_another_account: no PostgreSQL"); - return; - } - // Each blocked copy holds a pooled catalog connection for its snapshot. - let s = scratch_with_pool(20).await; - indexed_source(&s, "sl_src").await; - // One backup per restore: a second restore of one backup while the - // first is CREATING is BackupInUse, not what is measured here. All are - // taken before the gate, which a CreateBackup would wait on too. - let mut backups_a = Vec::new(); - for i in 0..=RESTORE_SLOTS_PER_ACCOUNT + 1 { - backups_a.push( - s.engine - .create_backup(ACCOUNT, "sl_src", &format!("b{i}")) - .await - .expect("backup in A") - .backup_arn, - ); - } - let mut b_input = composite_table("sl_src_b"); - b_input.key_schema.truncate(1); - b_input.attribute_definitions.truncate(1); - s.engine - .create_table(OTHER_ACCOUNT, b_input) - .await - .expect("source in B"); - let backup_b = s - .engine - .create_backup(OTHER_ACCOUNT, "sl_src_b", "b") - .await - .expect("backup in B"); - - let gate = gate_copies(&s).await; - for (i, backup) in backups_a.iter().enumerate().take(RESTORE_SLOTS_PER_ACCOUNT) { - start_restore(&s, &format!("sl_a{i}"), backup) - .await - .expect("A restore within its slots"); - } - await_copies_waiting(&s, 4).await; - - // A's fifth waits for one of A's slots (the limit is real) ... - let mut fifth = Box::pin(start_restore(&s, "sl_a4", &backups_a[4])); - assert!( - tokio::time::timeout(std::time::Duration::from_secs(2), &mut fifth) - .await - .is_err(), - "account A's fifth restore must wait for one of A's slots" - ); - assert_eq!(table_status(&s, "sl_a4").await, None); - - // ... a sixth whose request is dropped while it waits for a slot (a - // client that gave up during the wait) still restores once a slot - // frees, and creates nothing before that ... - let request = restore_request(&s, ACCOUNT, "sl_a5", &backups_a[5]) - .await - .expect("request"); - let sixth = s.engine.restore_table_from_backup(ACCOUNT, request); - assert!( - sixth.now_or_never().is_none(), - "the sixth request cannot have finished on its first poll" - ); - tokio::time::sleep(std::time::Duration::from_millis(200)).await; - assert_eq!(table_status(&s, "sl_a5").await, None); - - // ... and B's does not. - tokio::time::timeout( - std::time::Duration::from_secs(10), - start_restore_in(&s, OTHER_ACCOUNT, "sl_b", &backup_b.backup_arn), - ) - .await - .expect("account A's full slots must not block account B") - .expect("B restore registers"); - await_copies_waiting(&s, 5).await; - - gate.commit().await.expect("release the copies"); - fifth.await.expect("A's fifth gets a slot once one frees"); - let deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(30); - while table_status(&s, "sl_a5").await.is_none() { - assert!( - tokio::time::Instant::now() < deadline, - "the dropped sixth request's restore must still create its target" - ); - tokio::time::sleep(std::time::Duration::from_millis(10)).await; - } - for i in 0..=RESTORE_SLOTS_PER_ACCOUNT + 1 { - assert_eq!( - settled_status(&s, &format!("sl_a{i}")).await.as_deref(), - Some("ACTIVE"), - "sl_a{i}" - ); - } - assert_eq!( - settled_status_in(&s, OTHER_ACCOUNT, "sl_b") - .await - .as_deref(), - Some("ACTIVE") - ); - s.cleanup().await; -} - -/// (A1) The process-wide bound: with a pool of 10, at most 5 restores run at -/// once across all accounts, so a third account's restore waits (creating -/// nothing) while A holds 4 and B holds 1, and proceeds once they finish. -#[tokio::test] -async fn restores_across_accounts_are_bounded_per_server() { - if base_conn().is_none() { - eprintln!("SKIP restores_across_accounts_are_bounded_per_server: no PostgreSQL"); - return; - } - const THIRD_ACCOUNT: &str = "333333333333"; - let s = scratch_with_pool(10).await; - sqlx::query("INSERT INTO accounts (account_id, account_name) VALUES ($1, $2)") - .bind(THIRD_ACCOUNT) - .bind(format!("acct-{}-c", s.db_names[0])) - .execute(&s.catalog) - .await - .expect("seed the third account"); - let mut backups = Vec::new(); - for account in [ACCOUNT, OTHER_ACCOUNT, THIRD_ACCOUNT] { - let mut input = composite_table("gb_src"); - input.key_schema.truncate(1); - input.attribute_definitions.truncate(1); - s.engine - .create_table(account, input) - .await - .expect("source table"); - backups.push( - s.engine - .create_backup(account, "gb_src", "b") - .await - .expect("backup") - .backup_arn, - ); - } - // A's four restores need four backups (one restore of a backup at a - // time). - let mut backups_a = vec![backups[0].clone()]; - for i in 1..RESTORE_SLOTS_PER_ACCOUNT { - backups_a.push( - s.engine - .create_backup(ACCOUNT, "gb_src", &format!("b{i}")) - .await - .expect("backup") - .backup_arn, - ); - } - - let gate = gate_copies(&s).await; - for (i, backup) in backups_a.iter().enumerate() { - start_restore_in(&s, ACCOUNT, &format!("gb_a{i}"), backup) - .await - .expect("A restore within its slots"); - } - start_restore_in(&s, OTHER_ACCOUNT, "gb_b", &backups[1]) - .await - .expect("B restore takes the last server slot"); - await_copies_waiting(&s, 5).await; - - let mut third = Box::pin(start_restore_in(&s, THIRD_ACCOUNT, "gb_c", &backups[2])); - assert!( - tokio::time::timeout(std::time::Duration::from_secs(2), &mut third) - .await - .is_err(), - "the third account's restore must wait for a server slot" - ); - assert_eq!(table_status_in(&s, THIRD_ACCOUNT, "gb_c").await, None); - - gate.commit().await.expect("release the copies"); - tokio::time::timeout(std::time::Duration::from_secs(25), third) - .await - .expect("the third restore gets a server slot once one frees") - .expect("the third restore registers"); - for i in 0..RESTORE_SLOTS_PER_ACCOUNT { - assert_eq!( - settled_status(&s, &format!("gb_a{i}")).await.as_deref(), - Some("ACTIVE") - ); - } - assert_eq!( - settled_status_in(&s, OTHER_ACCOUNT, "gb_b") - .await - .as_deref(), - Some("ACTIVE") - ); - assert_eq!( - settled_status_in(&s, THIRD_ACCOUNT, "gb_c") - .await - .as_deref(), - Some("ACTIVE") - ); - s.cleanup().await; -} - /// (d) The restore's lock session is terminated mid-copy. Either the /// heartbeat notices and the copy stops and removes its target, or the -/// sweep claims the unowned target first; in both cases there is one -/// outcome, nothing is left behind, and the name restores again. +/// reconciler finds the record's heartbeat stale first; in both cases there +/// is one outcome, nothing is left behind, and the name restores again. #[tokio::test] async fn losing_the_lock_session_mid_copy_has_one_outcome() { + use extenddb_storage::BackupCatalog; if base_conn().is_none() { eprintln!("SKIP losing_the_lock_session_mid_copy_has_one_outcome: no PostgreSQL"); return; @@ -2431,7 +2358,7 @@ async fn losing_the_lock_session_mid_copy_has_one_outcome() { indexed_source(&s, "hb_src").await; let backup = s .engine - .create_backup(ACCOUNT, "hb_src", "b") + .create(ACCOUNT, "hb_src", "b", JobPermit::detached()) .await .expect("backup"); let tables_before = data_table_count(&s).await; @@ -2473,9 +2400,10 @@ async fn losing_the_lock_session_mid_copy_has_one_outcome() { assert_eq!(data_table_count(&s).await, tables_before); gate.commit().await.expect("release the gate"); - // Sweep path: the session is lost and the sweep claims the old, unowned - // target before the heartbeat would; the copy, released afterwards, must - // not bring it back. + // Reconciler path: the session is lost and the reconciler finds the + // record's heartbeat stale before the copy's own heartbeat would notice; + // it fails the record and removes the target, and the copy, released + // afterwards, must not bring it back. let gate = gate_copies(&s).await; let second = start_restore(&s, "hb_dst", &backup.backup_arn) .await @@ -2486,20 +2414,29 @@ async fn losing_the_lock_session_mid_copy_has_one_outcome() { .expect("lock held"); terminate(pid).await; sqlx::query( - "UPDATE tables SET creation_date_time = NOW() - INTERVAL '10 minutes' WHERE table_id = $1", + "UPDATE table_restores SET heartbeat_at = NOW() - INTERVAL '10 minutes' WHERE table_id = $1", ) .bind(&second.table_id) .execute(&s.catalog) .await - .expect("age the target past the grace period"); - let transitions = s + .expect("age the heartbeat past the stale window"); + let dead = s .engine - .process_control_plane_transitions() + .list_dead_restore_jobs(std::time::Duration::from_secs(300)) .await - .expect("sweep"); + .expect("list"); + assert_eq!(dead.len(), 1, "{dead:?}"); + assert!( + s.engine + .fail_restore_job(&dead[0].target, "owner dead") + .await + .expect("fail") + ); assert!( - transitions.iter().any(|(n, _)| n == "hb_dst"), - "{transitions:?}" + s.engine + .remove_restore_target(&dead[0].target) + .await + .expect("remove") ); gate.commit().await.expect("release the copy"); // Past the next heartbeat, so the stopped copy has run its cleanup. @@ -2521,63 +2458,204 @@ async fn losing_the_lock_session_mid_copy_has_one_outcome() { /// (e) After a multi-batch restore, every base and index row is exactly the /// row the write path (PutItem) wrote for the source: same key columns, same /// projection, same membership. +/// (e) The restore heartbeat is conditioned on `status = 'RUNNING'`: when the +/// reconciler has failed a record whose copy is still alive (its heartbeat +/// found stale through a slow catalog, and its removal of the target then +/// errored, so the target is CREATING under a FAILED record), the copy's next +/// heartbeat refreshes nothing and the copy stops there, performing no +/// further store or table operation (design 4(e): "the owner itself aborts at +/// its next heartbeat"). Without the condition the heartbeat keeps confirming +/// on the FAILED record and the copy runs on. The copy is held at the gate +/// through the heartbeat, the record is written FAILED by hand (the state the +/// reconciler's write leaves), and the claim session's end is awaited before +/// the gate opens: the target is still CREATING, the record FAILED with its +/// heartbeat not refreshed, the data table empty, and the pair is what the +/// reconciler's FAILED-target duty then lists and removes. #[tokio::test] -async fn restored_index_rows_match_the_write_path() { +async fn a_copy_whose_record_the_reconciler_failed_stops_at_its_next_heartbeat() { + use extenddb_storage::BackupCatalog; if base_conn().is_none() { - eprintln!("SKIP restored_index_rows_match_the_write_path: no PostgreSQL"); + eprintln!( + "SKIP a_copy_whose_record_the_reconciler_failed_stops_at_its_next_heartbeat: no \ + PostgreSQL" + ); return; } let s = scratch().await; - let src = indexed_source_with(&s, "wp_src", 1_100).await; + indexed_source(&s, "hf_src").await; let backup = s .engine - .create_backup(ACCOUNT, "wp_src", "b") + .create(ACCOUNT, "hf_src", "b", JobPermit::detached()) .await .expect("backup"); - let desc = restore(&s, "wp_dst", &backup.backup_arn) + let gate = gate_copies(&s).await; + let desc = start_restore(&s, "hf_dst", &backup.backup_arn) .await - .expect("restore"); - assert_eq!(table_status(&s, "wp_dst").await.as_deref(), Some("ACTIVE")); - let rows = |id: String| { - let db = s.db.clone(); + .expect("registers"); + await_copies_waiting(&s, 1).await; + assert!( + restore_lock_holder(&s, ACCOUNT, "hf_dst").await.is_some(), + "the copy holds its claim session while it waits at the gate" + ); + let record = |s: &Scratch| { + let catalog = s.catalog.clone(); + let table_id = desc.table_id.clone(); async move { - sqlx::query_scalar::<_, String>(&format!( - "SELECT to_jsonb(t)::text FROM \"_ddb_{id}\" t ORDER BY 1" - )) - .fetch_all(&db) + let row: (String, f64, Option) = sqlx::query_as( + "SELECT status, EXTRACT(EPOCH FROM heartbeat_at)::FLOAT8, \ + EXTRACT(EPOCH FROM completed_at)::FLOAT8 FROM table_restores WHERE table_id = $1", + ) + .bind(&table_id) + .fetch_one(&catalog) .await - .expect("rows") + .expect("the job record"); + row } }; - let (src_gi, src_li) = gi_li(&s, &src).await; - let (dst_gi, dst_li) = gi_li(&s, &desc.table_id).await; - let base = rows(desc.table_id.clone()).await; - assert_eq!(base.len(), 1_100); - assert_eq!(base, rows(src.clone()).await); - for (a, b) in [(src_gi, dst_gi), (src_li, dst_li)] { - let want = rows(a).await; - assert!(!want.is_empty() && want.len() < 1_100, "partial membership"); - assert_eq!(rows(b).await, want); - } - assert!(xmins(&s, &desc.table_id).await.len() >= 3); - s.cleanup().await; -} + let (status, heartbeat_before, _) = record(&s).await; + assert_eq!(status, "RUNNING"); -/// The definition read reports a lost connection as `Transient`, not -/// `Internal`: DescribeBackup answers without feature details only for a -/// definition that does not decode, so a failover must not reach it as -/// `Internal`. The engine's catalog sessions are terminated, as a failover or -/// an administrator restart would; the next read runs on one of them (the -/// pool does not test a connection before handing it out). -#[tokio::test] -async fn a_definition_read_that_loses_its_connection_is_transient() { - if base_conn().is_none() { - eprintln!("SKIP a_definition_read_that_loses_its_connection_is_transient: no PostgreSQL"); - return; + // The reconciler's FAILED write, with its removal not landed. + let written = sqlx::query( + "UPDATE table_restores SET status = 'FAILED', completed_at = now(), \ + failure_reason = 'owner dead' WHERE table_id = $1 AND status = 'RUNNING'", + ) + .bind(&desc.table_id) + .execute(&s.catalog) + .await + .expect("fail the record") + .rows_affected(); + assert_eq!(written, 1); + + // The copy's next heartbeat matches no row; the copy stops and its claim + // session ends, with the gate still held. Observed through the advisory + // lock the session holds rather than a sleep; the deadline is well past + // one heartbeat period. + let started = std::time::Instant::now(); + let deadline = tokio::time::Instant::now() + + extenddb_storage::backup_catalog::JOB_HEARTBEAT_INTERVAL + + std::time::Duration::from_secs(25); + while restore_lock_holder(&s, ACCOUNT, "hf_dst").await.is_some() { + assert!( + tokio::time::Instant::now() < deadline, + "the copy did not stop at its next heartbeat after its record was failed" + ); + tokio::time::sleep(std::time::Duration::from_millis(100)).await; } - let s = scratch().await; + assert!( + started.elapsed() >= std::time::Duration::from_secs(5), + "the copy stopped at a heartbeat, not at the write: {:?}", + started.elapsed() + ); + let (status, heartbeat_after, completed_at) = record(&s).await; + assert_eq!( + status, "FAILED", + "the record is left as the reconciler wrote it" + ); + assert_eq!( + heartbeat_after, heartbeat_before, + "a heartbeat on a FAILED record refreshes nothing" + ); + assert!(completed_at.is_some()); + assert_eq!( + table_status(&s, "hf_dst").await.as_deref(), + Some("CREATING"), + "the target is left to the reconciler, not removed by the copy" + ); + gate.commit().await.expect("release the gate"); + tokio::time::sleep(std::time::Duration::from_secs(2)).await; + assert_eq!( + data_rows(&s, &desc.table_id).await, + 0, + "the stopped copy wrote nothing once the gate opened" + ); + assert_eq!( + table_status(&s, "hf_dst").await.as_deref(), + Some("CREATING") + ); + assert!(restore_lock_holder(&s, ACCOUNT, "hf_dst").await.is_none()); + + // The pair is the FAILED-target duty's: listed once the record is stale, + // and the removal lands. + tokio::time::sleep(std::time::Duration::from_millis(1_100)).await; + let dead = s + .engine + .list_dead_restore_jobs(std::time::Duration::from_secs(1)) + .await + .expect("list"); + assert_eq!(dead.len(), 1, "{dead:?}"); + assert_eq!( + dead[0].status, + extenddb_storage::backup_catalog::JobStatus::Failed + ); + assert!( + s.engine + .remove_restore_target(&dead[0].target) + .await + .expect("remove") + ); + assert_eq!(table_status(&s, "hf_dst").await, None); + assert!(!data_table_exists(&s, &desc.table_id).await); + s.cleanup().await; +} + +#[tokio::test] +async fn restored_index_rows_match_the_write_path() { + if base_conn().is_none() { + eprintln!("SKIP restored_index_rows_match_the_write_path: no PostgreSQL"); + return; + } + let s = scratch().await; + let src = indexed_source_with(&s, "wp_src", 1_100).await; + let backup = s + .engine + .create(ACCOUNT, "wp_src", "b", JobPermit::detached()) + .await + .expect("backup"); + let desc = restore(&s, "wp_dst", &backup.backup_arn) + .await + .expect("restore"); + assert_eq!(table_status(&s, "wp_dst").await.as_deref(), Some("ACTIVE")); + let rows = |id: String| { + let db = s.db.clone(); + async move { + sqlx::query_scalar::<_, String>(&format!( + "SELECT to_jsonb(t)::text FROM \"_ddb_{id}\" t ORDER BY 1" + )) + .fetch_all(&db) + .await + .expect("rows") + } + }; + let (src_gi, src_li) = gi_li(&s, &src).await; + let (dst_gi, dst_li) = gi_li(&s, &desc.table_id).await; + let base = rows(desc.table_id.clone()).await; + assert_eq!(base.len(), 1_100); + assert_eq!(base, rows(src.clone()).await); + for (a, b) in [(src_gi, dst_gi), (src_li, dst_li)] { + let want = rows(a).await; + assert!(!want.is_empty() && want.len() < 1_100, "partial membership"); + assert_eq!(rows(b).await, want); + } + assert!(xmins(&s, &desc.table_id).await.len() >= 3); + s.cleanup().await; +} + +/// The definition read reports a lost connection as `Transient`, not +/// `Internal`: DescribeBackup answers without feature details only for a +/// definition that does not decode, so a failover must not reach it as +/// `Internal`. The engine's catalog sessions are terminated, as a failover or +/// an administrator restart would; the next read runs on one of them (the +/// pool does not test a connection before handing it out). +#[tokio::test] +async fn a_definition_read_that_loses_its_connection_is_transient() { + if base_conn().is_none() { + eprintln!("SKIP a_definition_read_that_loses_its_connection_is_transient: no PostgreSQL"); + return; + } + let s = scratch().await; let missing = format!("arn:aws:dynamodb:{REGION}:{ACCOUNT}:table/t/backup/01"); - match s.engine.backup_restore_source(ACCOUNT, &missing).await { + match s.engine.restore_source(ACCOUNT, &missing).await { Err(StorageError::Validation(m)) => assert!(m.contains("Backup not found"), "{m}"), other => panic!("a missing backup before the failover: {:?}", other.err()), } @@ -2609,16 +2687,14 @@ async fn a_definition_read_that_loses_its_connection_is_transient() { tokio::time::sleep(std::time::Duration::from_millis(50)).await; } - match s.engine.backup_restore_source(ACCOUNT, &missing).await { + match s.engine.restore_source(ACCOUNT, &missing).await { Err(StorageError::Transient(m)) => assert!(m.contains("Database error"), "{m}"), other => panic!("a read on a terminated session must be Transient: {other:?}"), } // And the engine recovers: a later read reconnects. let mut recovered = false; for _ in 0..10 { - if let Err(StorageError::Validation(_)) = - s.engine.backup_restore_source(ACCOUNT, &missing).await - { + if let Err(StorageError::Validation(_)) = s.engine.restore_source(ACCOUNT, &missing).await { recovered = true; break; } @@ -2626,3 +2702,1125 @@ async fn a_definition_read_that_loses_its_connection_is_transient() { assert!(recovered, "the pool reconnects after the failover"); s.cleanup().await; } + +// --- the catalog index ----------------------------------------------------------- + +mod catalog_index { + use extenddb_core::backup_format::BackupArn; + use extenddb_storage::BackupCatalog; + use extenddb_storage::backup_catalog::{ + BackupRow, BackupStatus, DeleteStart, Heartbeat, JobStatus, RestoreTargetKey, StatusFields, + StoreIdentity, StoreKind, TimeBounds, UNOWNED_GENERATION, WriteOutcome, + }; + use futures::TryStreamExt; + + use super::*; + + fn store() -> StoreIdentity { + StoreIdentity { + store_kind: StoreKind::Filesystem, + store_root: "/var/lib/extenddb/backups".to_owned(), + store_id: "store-1".to_owned(), + } + } + + /// A row for a backup of `table` in `account` minted now, with the + /// manifest fixture's source table. + fn row(account: &str, table: &str, name: &str) -> BackupRow { + let backup_id = extenddb_storage::util::new_backup_id(); + let backup_arn = extenddb_storage::util::backup_arn(REGION, account, table, &backup_id); + let manifest: serde_json::Value = serde_json::from_str(include_str!( + "../../core/testdata/extenddb-manifest/v1.json" + )) + .expect("fixture"); + BackupRow { + backup_id: backup_id.clone(), + account_id: account.to_owned(), + table_name: table.to_owned(), + backup_arn: backup_arn.clone(), + backup_name: name.to_owned(), + status: BackupStatus::Creating, + store: store(), + size_bytes: 0, + item_count: 0, + created_at: extenddb_storage::util::backup_creation_date_time(&backup_id).unwrap(), + completed_at: None, + failure_reason: None, + source_table_details_json: manifest["source_table"].clone(), + owner_generation: 0, + heartbeat_at: 0.0, + } + } + + fn arn_of(row: &BackupRow) -> BackupArn { + BackupArn::parse(&row.backup_arn).unwrap() + } + + /// The claim: a second claim of the same backup is refused while the + /// first is held, the heartbeat confirms the claim and refreshes the + /// row, a status write is conditioned on the generation and the `from` + /// status, and the row is read back by account, backup id, table and ARN. + #[tokio::test] + async fn claim_backup_excludes_a_second_owner_and_fences_its_writes() { + if base_conn().is_none() { + eprintln!( + "SKIP claim_backup_excludes_a_second_owner_and_fences_its_writes: no PostgreSQL" + ); + return; + } + let s = scratch().await; + assert!( + uuid::Uuid::parse_str(&s.engine.catalog_id().await.expect("id")).is_ok(), + "the catalog has an id from the migration" + ); + assert!(!s.engine.check_maintenance_claim().await.expect("check")); + let row = row(ACCOUNT, "Music", "nightly"); + assert_eq!(backup_lock_holder(&s, &row.backup_id).await, None); + let claim = s.engine.claim_backup(&row).await.expect("claim"); + assert_ne!(claim.generation(), UNOWNED_GENERATION); + // The claim is the advisory lock on the claim session, held for the + // claim's life: what excludes a second owner before any row exists. + let holder = backup_lock_holder(&s, &row.backup_id) + .await + .expect("the claim session holds the backup's advisory lock"); + let second = s.engine.claim_backup(&row).await.err(); + assert!( + matches!(second, Some(StorageError::AlreadyExists(_))), + "{second:?}" + ); + assert_eq!( + backup_lock_holder(&s, &row.backup_id).await, + Some(holder), + "the refused claim took nothing" + ); + let stored = s + .engine + .read_backup_row(&arn_of(&row)) + .await + .expect("read") + .expect("the row exists"); + assert_eq!(stored.status, BackupStatus::Creating); + assert_eq!(stored.owner_generation, claim.generation()); + assert_eq!(stored.store, store()); + assert_eq!(stored.created_at, row.created_at); + assert_eq!( + stored.source_table_details_json, + row.source_table_details_json + ); + let before = stored.heartbeat_at; + tokio::time::sleep(std::time::Duration::from_millis(20)).await; + assert_eq!( + claim.heartbeat().await.expect("heartbeat"), + Heartbeat::Confirmed + ); + let after = s + .engine + .read_backup_row(&arn_of(&row)) + .await + .unwrap() + .unwrap() + .heartbeat_at; + assert!(after > before, "{after} > {before}"); + + // The same id under another account, table or region finds nothing. + let other_account = + BackupArn::new("aws", REGION, OTHER_ACCOUNT, "Music", &row.backup_id).unwrap(); + assert!( + s.engine + .read_backup_row(&other_account) + .await + .unwrap() + .is_none() + ); + let other_table = BackupArn::new("aws", REGION, ACCOUNT, "Other", &row.backup_id).unwrap(); + assert!( + s.engine + .read_backup_row(&other_table) + .await + .unwrap() + .is_none() + ); + let other_region = + BackupArn::new("aws", "eu-west-1", ACCOUNT, "Music", &row.backup_id).unwrap(); + assert!( + s.engine + .read_backup_row(&other_region) + .await + .unwrap() + .is_none() + ); + + // A write from the wrong state matches nothing; the right one matches + // once, and the row then carries what it set. + let fields = StatusFields { + size_bytes: Some(1234), + item_count: Some(5), + failure_reason: None, + completed: true, + }; + assert_eq!( + claim + .set_status(JobStatus::Running, JobStatus::Available, &fields) + .await + .unwrap(), + WriteOutcome::NotMatched + ); + assert_eq!( + claim + .set_status(JobStatus::Creating, JobStatus::Available, &fields) + .await + .unwrap(), + WriteOutcome::Matched + ); + assert_eq!( + claim + .set_status(JobStatus::Creating, JobStatus::Available, &fields) + .await + .unwrap(), + WriteOutcome::NotMatched + ); + let read = claim.read().await.expect("read"); + assert_eq!(read.status, JobStatus::Available); + assert_eq!(read.owner_generation, claim.generation()); + let stored = s + .engine + .read_backup_row(&arn_of(&row)) + .await + .unwrap() + .unwrap(); + assert_eq!(stored.status, BackupStatus::Available); + assert_eq!(stored.size_bytes, 1234); + assert_eq!(stored.item_count, 5); + assert!(stored.completed_at.is_some()); + + // A row another generation took over confirms nothing for this claim. + sqlx::query( + "UPDATE backup_index SET owner_generation = owner_generation + 1 WHERE backup_id = $1", + ) + .bind(&row.backup_id) + .execute(&s.catalog) + .await + .expect("take over"); + assert_eq!(claim.heartbeat().await.expect("heartbeat"), Heartbeat::Lost); + assert_eq!( + claim + .set_status( + JobStatus::Available, + JobStatus::Failed, + &StatusFields::default() + ) + .await + .unwrap(), + WriteOutcome::NotMatched + ); + drop(claim); + // The lock went with the claim's session (the server releases it when + // the connection closes, a moment after the drop): the row can be + // claimed by a successor only through its insert, which the existing + // row refuses. + let deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(10); + while backup_lock_holder(&s, &row.backup_id).await.is_some() { + assert!( + tokio::time::Instant::now() < deadline, + "the advisory lock outlived the dropped claim" + ); + tokio::time::sleep(std::time::Duration::from_millis(20)).await; + } + let third = s.engine.claim_backup(&row).await.err(); + assert!( + matches!(third, Some(StorageError::AlreadyExists(_))), + "{third:?}" + ); + s.cleanup().await; + } + + /// `start_backup_delete` runs under the `backup_index` row's lock: a + /// transaction holding the row `FOR UPDATE` while it registers a CREATING + /// target makes the delete wait, and when it commits the delete sees the + /// registration and answers `InUse`, with the row left AVAILABLE. Without + /// the `FOR UPDATE` the delete reads the row and the in-use rule past the + /// uncommitted registration and then waits on the row in its DELETING + /// write instead, so a wait alone cannot tell the two apart: the + /// `pg_stat_activity` probe below is scoped to the `SELECT ... FOR + /// UPDATE` text, and a delete that waits in its `UPDATE` never satisfies + /// it and fails the probe's deadline. That probe is what pins the lock; + /// the `InUse` answer after the commit is the second check. + #[tokio::test] + async fn start_backup_delete_waits_for_the_row_lock_and_then_sees_the_registration() { + if base_conn().is_none() { + eprintln!( + "SKIP start_backup_delete_waits_for_the_row_lock_and_then_sees_the_registration: no PostgreSQL" + ); + return; + } + let s = scratch().await; + let row = row(ACCOUNT, "Music", "nightly"); + let key = row.key(); + let claim = s.engine.claim_backup(&row).await.expect("claim"); + assert_eq!( + claim + .set_status( + JobStatus::Creating, + JobStatus::Available, + &StatusFields::default() + ) + .await + .unwrap(), + WriteOutcome::Matched + ); + indexed_source(&s, "lk_target").await; + let target_id = table_id_of(&s, "lk_target").await.unwrap(); + + // The registration's side: lock the row, then register the target, + // in one transaction that stays open. + let mut registration = s.catalog.begin().await.expect("begin"); + let status: String = sqlx::query_scalar( + "SELECT status FROM backup_index WHERE account_id = $1 AND backup_id = $2 FOR UPDATE", + ) + .bind(&key.account_id) + .bind(&key.backup_id) + .fetch_one(&mut *registration) + .await + .expect("lock the row"); + assert_eq!(status, "AVAILABLE"); + sqlx::query( + "INSERT INTO table_restores (table_id, source_backup_arn, source_table_arn, \ + restore_date_time, status, kind, owner_generation, heartbeat_at) \ + VALUES ($1, $2, 'arn:aws:dynamodb:us-east-1:123456789012:table/Music', now(), \ + 'RUNNING', 'manifest', 7, now())", + ) + .bind(&target_id) + .bind(&row.backup_arn) + .execute(&mut *registration) + .await + .expect("register a target"); + sqlx::query( + "UPDATE tables SET table_status = 'CREATING', status_transition_at = NULL \ + WHERE table_id = $1", + ) + .bind(&target_id) + .execute(&mut *registration) + .await + .expect("make it CREATING"); + + let engine = s.engine.clone(); + let deleting = tokio::spawn(async move { + let key = key; + engine.start_backup_delete(&key).await + }); + // The delete is waiting on the registration's transaction, in its + // `SELECT ... FOR UPDATE` (scoped to this database and that statement: + // other tests of this binary produce row-lock waits of their own, and + // without the lock the wait would be the DELETING write's instead). + let deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(30); + loop { + let waiting: i64 = sqlx::query_scalar( + "SELECT count(*) FROM pg_stat_activity WHERE datname = current_database() \ + AND wait_event_type = 'Lock' \ + AND query LIKE 'SELECT status FROM backup_index%FOR UPDATE'", + ) + .fetch_one(&s.catalog) + .await + .expect("count waiters"); + if waiting >= 1 { + break; + } + assert!( + tokio::time::Instant::now() < deadline, + "start_backup_delete did not wait for the row lock" + ); + assert!( + !deleting.is_finished(), + "the delete answered under the lock" + ); + tokio::time::sleep(std::time::Duration::from_millis(20)).await; + } + assert!( + !deleting.is_finished(), + "the delete answered under the lock" + ); + registration + .commit() + .await + .expect("commit the registration"); + let answer = tokio::time::timeout(std::time::Duration::from_secs(30), deleting) + .await + .expect("the delete finishes once the lock is released") + .expect("task") + .expect("start_backup_delete"); + assert_eq!(answer, DeleteStart::InUse("lk_target".to_owned())); + assert_eq!( + s.engine + .read_backup_row(&arn_of(&row)) + .await + .unwrap() + .unwrap() + .status, + BackupStatus::Available, + "the refused delete leaves the row AVAILABLE" + ); + drop(claim); + s.cleanup().await; + } + + /// The FAILED-target duty's listing: a FAILED record whose table is + /// still a CREATING restore target (the FAILED write landed and the + /// removal did not) is not listed while the record is fresh (the job + /// may still be on its own failure path), is listed once `completed_at` + /// is older than the stale window, with its status, and its target is + /// then removed with no second FAILED write; a FAILED record without a + /// `completed_at` is aged by its heartbeat; a FAILED record whose table + /// is ACTIVE is never listed. While the target is CREATING under the + /// FAILED record the backup is in use (the rule reads the target's + /// status, not the job's) and DeleteBackup is refused; the removal frees + /// it. + #[tokio::test] + async fn a_failed_record_whose_target_removal_failed_is_listed_once_stale_and_removed() { + use extenddb_storage::backup_catalog::RestoreKind; + if base_conn().is_none() { + eprintln!( + "SKIP a_failed_record_whose_target_removal_failed_is_listed_once_stale_and_removed: no PostgreSQL" + ); + return; + } + let stale = std::time::Duration::from_secs(300); + let s = scratch().await; + indexed_source(&s, "ft_src").await; + let backup = s + .engine + .create(ACCOUNT, "ft_src", "b", JobPermit::detached()) + .await + .expect("backup"); + let desc = restore(&s, "ft_dst", &backup.backup_arn) + .await + .expect("restore"); + let key = RestoreTargetKey { + account_id: ACCOUNT.to_owned(), + table_name: "ft_dst".to_owned(), + table_id: desc.table_id.clone(), + }; + // The state the probe left: a dead restore, failed by the reconciler + // (or by the job's own failure path), whose target removal failed. + sqlx::query( + "UPDATE tables SET table_status = 'CREATING', status_transition_at = NULL \ + WHERE table_id = $1", + ) + .bind(&desc.table_id) + .execute(&s.catalog) + .await + .expect("interrupt"); + sqlx::query( + "UPDATE table_restores SET status = 'RUNNING', completed_at = NULL, \ + heartbeat_at = NOW() - INTERVAL '10 minutes' WHERE table_id = $1", + ) + .bind(&desc.table_id) + .execute(&s.catalog) + .await + .expect("age the heartbeat"); + let dead = s.engine.list_dead_restore_jobs(stale).await.expect("list"); + assert_eq!(dead.len(), 1); + assert_eq!(dead[0].status, JobStatus::Running); + assert!( + s.engine + .fail_restore_job(&key, "owner dead") + .await + .expect("fail") + ); + assert_eq!( + job_status(&s, &desc.table_id).await.as_deref(), + Some("FAILED") + ); + assert_eq!( + table_status(&s, "ft_dst").await.as_deref(), + Some("CREATING") + ); + + // Fresh: the record was just failed; whoever failed it may still be + // removing the target. + assert!( + s.engine + .list_dead_restore_jobs(stale) + .await + .expect("list") + .is_empty(), + "a FAILED record within the stale window is left to its writer" + ); + // Stale: listed, as FAILED, for the FAILED-target duty. + sqlx::query( + "UPDATE table_restores SET completed_at = NOW() - INTERVAL '10 minutes' \ + WHERE table_id = $1", + ) + .bind(&desc.table_id) + .execute(&s.catalog) + .await + .expect("age the FAILED record"); + let dead = s.engine.list_dead_restore_jobs(stale).await.expect("list"); + assert_eq!(dead.len(), 1, "{dead:?}"); + assert_eq!(dead[0].target, key); + assert_eq!(dead[0].status, JobStatus::Failed); + assert_eq!(dead[0].kind, RestoreKind::Legacy); + assert_eq!(dead[0].source_backup_arn, backup.backup_arn); + // A FAILED record without `completed_at` (a failure path that did + // not stamp one) is aged by its heartbeat: fresh, not listed; stale, + // listed. + sqlx::query( + "UPDATE table_restores SET completed_at = NULL, heartbeat_at = NOW() \ + WHERE table_id = $1", + ) + .bind(&desc.table_id) + .execute(&s.catalog) + .await + .expect("clear completed_at, fresh heartbeat"); + assert!( + s.engine + .list_dead_restore_jobs(stale) + .await + .expect("list") + .is_empty() + ); + sqlx::query( + "UPDATE table_restores SET heartbeat_at = NOW() - INTERVAL '10 minutes' \ + WHERE table_id = $1", + ) + .bind(&desc.table_id) + .execute(&s.catalog) + .await + .expect("age the heartbeat"); + assert_eq!( + s.engine + .list_dead_restore_jobs(stale) + .await + .expect("list") + .len(), + 1 + ); + // A FAILED record whose table the removal already moved to DELETING + // (the claim landed, the drop did not) is the control plane's row. + sqlx::query("UPDATE tables SET table_status = 'DELETING' WHERE table_id = $1") + .bind(&desc.table_id) + .execute(&s.catalog) + .await + .expect("DELETING"); + assert!( + s.engine + .list_dead_restore_jobs(stale) + .await + .expect("list") + .is_empty() + ); + sqlx::query("UPDATE tables SET table_status = 'CREATING' WHERE table_id = $1") + .bind(&desc.table_id) + .execute(&s.catalog) + .await + .expect("CREATING again"); + // The in-use rule reads the target's status, not the job's: with the + // record FAILED and the target still CREATING, the backup is in use + // (naming the target) and the legacy delete is refused. + let backup_key = extenddb_storage::backup_catalog::BackupKey::from_arn( + &extenddb_core::backup_format::BackupArn::parse(&backup.backup_arn).expect("arn"), + ); + assert_eq!( + s.engine + .backup_in_use(&backup_key) + .await + .expect("in use") + .as_deref(), + Some("ft_dst"), + "a CREATING target under a FAILED record keeps its backup in use" + ); + let refused = s.engine.delete(ACCOUNT, &backup.backup_arn).await; + assert!( + matches!(refused, Err(StorageError::BackupInUse(_))), + "DeleteBackup while the target is CREATING: {refused:?}" + ); + // The reconciler's FAILED-target duty: no second FAILED write, the + // target removed. + assert!( + !s.engine + .fail_restore_job(&key, "owner dead") + .await + .expect("fail again"), + "a FAILED record is not RUNNING" + ); + assert!(s.engine.remove_restore_target(&key).await.expect("remove")); + assert_eq!(table_status(&s, "ft_dst").await, None); + assert!(!data_table_exists(&s, &desc.table_id).await); + // The removal frees the backup: no longer in use, and deletable. + assert_eq!( + s.engine.backup_in_use(&backup_key).await.expect("in use"), + None + ); + assert!( + s.engine + .list_dead_restore_jobs(stale) + .await + .expect("list") + .is_empty() + ); + + // A FAILED record whose table is ACTIVE (a flip that won against a + // late FAILED write) is nobody's duty. + let again = restore(&s, "ft_dst", &backup.backup_arn) + .await + .expect("restore again"); + assert_eq!(table_status(&s, "ft_dst").await.as_deref(), Some("ACTIVE")); + sqlx::query( + "UPDATE table_restores SET status = 'FAILED', \ + completed_at = NOW() - INTERVAL '10 minutes' WHERE table_id = $1", + ) + .bind(&again.table_id) + .execute(&s.catalog) + .await + .expect("fail the record of an ACTIVE table"); + assert!( + s.engine + .list_dead_restore_jobs(stale) + .await + .expect("list") + .is_empty(), + "an ACTIVE table is never a target to remove" + ); + s.engine + .delete(ACCOUNT, &backup.backup_arn) + .await + .expect("DeleteBackup once no CREATING target reads the backup"); + s.cleanup().await; + } + + /// The two conditions that make a target's removal exclusive with its + /// ACTIVE flip and with every other table: `abort_restore`'s claim is + /// conditional on CREATING, so against an ACTIVE table with the right id + /// it does nothing (as does `remove_restore_target`); and + /// `remove_restore_target` matches the account and name as well as the + /// id, so the right id under another account or name removes nothing. + #[tokio::test] + async fn a_target_is_removed_only_while_creating_and_only_under_its_full_key() { + if base_conn().is_none() { + eprintln!( + "SKIP a_target_is_removed_only_while_creating_and_only_under_its_full_key: no PostgreSQL" + ); + return; + } + let s = scratch().await; + indexed_source(&s, "ex_src").await; + let backup = s + .engine + .create(ACCOUNT, "ex_src", "b", JobPermit::detached()) + .await + .expect("backup"); + let desc = restore(&s, "ex_dst", &backup.backup_arn) + .await + .expect("restore"); + assert_eq!(table_status(&s, "ex_dst").await.as_deref(), Some("ACTIVE")); + let key = RestoreTargetKey { + account_id: ACCOUNT.to_owned(), + table_name: "ex_dst".to_owned(), + table_id: desc.table_id.clone(), + }; + + // ACTIVE with the right id: the ordering where the copy's flip won. + assert!( + !s.engine + .abort_restore(&desc.table_id) + .await + .expect("abort against ACTIVE"), + "abort_restore claims only a CREATING table" + ); + assert_eq!(table_status(&s, "ex_dst").await.as_deref(), Some("ACTIVE")); + assert!(data_table_exists(&s, &desc.table_id).await); + assert!( + !s.engine + .remove_restore_target(&key) + .await + .expect("remove against ACTIVE") + ); + assert_eq!(table_status(&s, "ex_dst").await.as_deref(), Some("ACTIVE")); + + // CREATING, the right id, the wrong account or name: nothing. + sqlx::query( + "UPDATE tables SET table_status = 'CREATING', status_transition_at = NULL \ + WHERE table_id = $1", + ) + .bind(&desc.table_id) + .execute(&s.catalog) + .await + .expect("make it a target"); + let other_account = RestoreTargetKey { + account_id: OTHER_ACCOUNT.to_owned(), + ..key.clone() + }; + let other_name = RestoreTargetKey { + table_name: "ex_other".to_owned(), + ..key.clone() + }; + for wrong in [&other_account, &other_name] { + assert!( + !s.engine + .remove_restore_target(wrong) + .await + .expect("remove under the wrong key"), + "{wrong:?}" + ); + assert_eq!( + table_status(&s, "ex_dst").await.as_deref(), + Some("CREATING") + ); + assert!(data_table_exists(&s, &desc.table_id).await); + } + // The full key removes it. + assert!(s.engine.remove_restore_target(&key).await.expect("remove")); + assert_eq!(table_status(&s, "ex_dst").await, None); + assert!(!data_table_exists(&s, &desc.table_id).await); + s.cleanup().await; + } + + /// DeleteBackup's first step and the tombstone writes: only an AVAILABLE + /// row moves to DELETING; a CREATING row answers its status; a backup a + /// CREATING restore target reads answers the target's name; DELETING + /// moves to DELETED once; a DELETED row is removed once; a missing row is + /// "Backup not found". + #[tokio::test] + async fn start_backup_delete_checks_the_status_and_the_in_use_rule_under_the_lock() { + if base_conn().is_none() { + eprintln!( + "SKIP start_backup_delete_checks_the_status_and_the_in_use_rule_under_the_lock: no PostgreSQL" + ); + return; + } + let s = scratch().await; + let row = row(ACCOUNT, "Music", "nightly"); + let key = row.key(); + assert!(matches!( + s.engine.start_backup_delete(&key).await, + Err(StorageError::Validation(m)) if m.contains("Backup not found") + )); + let claim = s.engine.claim_backup(&row).await.expect("claim"); + assert_eq!( + s.engine.start_backup_delete(&key).await.unwrap(), + DeleteStart::Status(BackupStatus::Creating) + ); + assert_eq!( + claim + .set_status( + JobStatus::Creating, + JobStatus::Available, + &StatusFields::default() + ) + .await + .unwrap(), + WriteOutcome::Matched + ); + assert_eq!(s.engine.backup_in_use(&key).await.unwrap(), None); + + // A CREATING restore target registered from the backup: in use, + // naming the target; an ACTIVE one is not. + indexed_source(&s, "del_target").await; + let target_id = table_id_of(&s, "del_target").await.unwrap(); + sqlx::query( + "INSERT INTO table_restores (table_id, source_backup_arn, source_table_arn, \ + restore_date_time, status, kind, owner_generation, heartbeat_at) \ + VALUES ($1, $2, 'arn:aws:dynamodb:us-east-1:123456789012:table/Music', now(), \ + 'RUNNING', 'manifest', 7, now())", + ) + .bind(&target_id) + .bind(&row.backup_arn) + .execute(&s.catalog) + .await + .expect("register a target"); + sqlx::query("UPDATE tables SET table_status = 'CREATING', status_transition_at = NULL WHERE table_id = $1") + .bind(&target_id) + .execute(&s.catalog) + .await + .expect("make it CREATING"); + assert_eq!( + s.engine.backup_in_use(&key).await.unwrap().as_deref(), + Some("del_target") + ); + assert_eq!( + s.engine.start_backup_delete(&key).await.unwrap(), + DeleteStart::InUse("del_target".to_owned()) + ); + assert_eq!( + s.engine + .read_backup_row(&arn_of(&row)) + .await + .unwrap() + .unwrap() + .status, + BackupStatus::Available, + "a refused delete leaves the row AVAILABLE" + ); + sqlx::query("UPDATE tables SET table_status = 'ACTIVE' WHERE table_id = $1") + .bind(&target_id) + .execute(&s.catalog) + .await + .expect("make it ACTIVE"); + assert_eq!(s.engine.backup_in_use(&key).await.unwrap(), None); + + assert_eq!( + s.engine.start_backup_delete(&key).await.unwrap(), + DeleteStart::Deleting + ); + assert_eq!( + s.engine.start_backup_delete(&key).await.unwrap(), + DeleteStart::Status(BackupStatus::Deleting), + "a retry finds the row DELETING" + ); + assert!( + !s.engine.remove_backup_row(&key).await.unwrap(), + "DELETING is not a tombstone" + ); + assert!(s.engine.set_backup_deleted(&key).await.unwrap()); + assert!(!s.engine.set_backup_deleted(&key).await.unwrap()); + let stored = s + .engine + .read_backup_row(&arn_of(&row)) + .await + .unwrap() + .unwrap(); + assert_eq!(stored.status, BackupStatus::Deleted); + assert!(stored.completed_at.is_some()); + assert!(s.engine.remove_backup_row(&key).await.unwrap()); + assert!(!s.engine.remove_backup_row(&key).await.unwrap()); + assert!( + s.engine + .read_backup_row(&arn_of(&row)) + .await + .unwrap() + .is_none() + ); + drop(claim); + s.cleanup().await; + } + + /// The listings: by account with an optional table and the time bounds + /// (lower inclusive, upper exclusive) in (created_at, ARN) order, and by + /// store and status. + #[tokio::test] + async fn the_listings_are_scoped_and_ordered() { + if base_conn().is_none() { + eprintln!("SKIP the_listings_are_scoped_and_ordered: no PostgreSQL"); + return; + } + let s = scratch().await; + let mut rows = Vec::new(); + for (account, table, name) in [ + (ACCOUNT, "Music", "a"), + (ACCOUNT, "Music", "b"), + (ACCOUNT, "Other", "c"), + (OTHER_ACCOUNT, "Music", "d"), + ] { + tokio::time::sleep(std::time::Duration::from_millis(5)).await; + let row = row(account, table, name); + let claim = s.engine.claim_backup(&row).await.expect("claim"); + if name == "b" { + assert_eq!( + claim + .set_status( + JobStatus::Creating, + JobStatus::Failed, + &StatusFields::default() + ) + .await + .unwrap(), + WriteOutcome::Matched + ); + } + rows.push((row, claim)); + } + let names = |listed: Vec| -> Vec { + listed.into_iter().map(|r| r.backup_name).collect() + }; + let all: Vec = s + .engine + .list_account_backup_rows(ACCOUNT, None, &TimeBounds::default()) + .try_collect() + .await + .unwrap(); + assert_eq!( + names(all.clone()), + ["a", "b", "c"], + "every status, in creation order" + ); + assert_eq!(all[1].status, BackupStatus::Failed); + let music: Vec = s + .engine + .list_account_backup_rows(ACCOUNT, Some("Music"), &TimeBounds::default()) + .try_collect() + .await + .unwrap(); + assert_eq!(names(music), ["a", "b"]); + let bounds = TimeBounds { + lower: Some(rows[1].0.created_at), + upper: Some(rows[2].0.created_at), + }; + let bounded: Vec = s + .engine + .list_account_backup_rows(ACCOUNT, None, &bounds) + .try_collect() + .await + .unwrap(); + assert_eq!(names(bounded), ["b"], "lower inclusive, upper exclusive"); + let other: Vec = s + .engine + .list_account_backup_rows(OTHER_ACCOUNT, None, &TimeBounds::default()) + .try_collect() + .await + .unwrap(); + assert_eq!(names(other), ["d"]); + + let by_store: Vec = s + .engine + .list_backup_rows(None, &store()) + .try_collect() + .await + .unwrap(); + assert_eq!(names(by_store), ["a", "b", "c", "d"]); + let failed: Vec = s + .engine + .list_backup_rows(Some(BackupStatus::Failed), &store()) + .try_collect() + .await + .unwrap(); + assert_eq!(names(failed), ["b"]); + let elsewhere = StoreIdentity { + store_id: "store-2".to_owned(), + ..store() + }; + let none: Vec = s + .engine + .list_backup_rows(None, &elsewhere) + .try_collect() + .await + .unwrap(); + assert!(none.is_empty(), "another store's rows are not listed"); + + // Two rows with one `created_at` order by the ARN in byte order + // (`backup_arn` is `COLLATE "C"`), the order the engine merges and + // pages by, not the database's collation: under `en_US` "a" sorts + // before "B", in byte order "B" (0x42) before "a" (0x61). The + // column's collation is asserted first, so the cell has teeth on a + // cluster whose default collation is already `C` (where the two + // orders agree and the rows alone would show nothing). + let collation: String = sqlx::query_scalar( + "SELECT c.collname FROM pg_attribute a JOIN pg_collation c ON c.oid = a.attcollation \ + WHERE a.attrelid = 'backup_index'::regclass AND a.attname = 'backup_arn'", + ) + .fetch_one(&s.catalog) + .await + .expect("the column's collation"); + assert_eq!(collation, "C", "backup_index.backup_arn is COLLATE \"C\""); + const CASE_ACCOUNT: &str = "333333333333"; + let mut same_instant = Vec::new(); + for table in ["a", "B"] { + let mut r = row(CASE_ACCOUNT, table, table); + r.created_at = 1_700_000_000.0; + let claim = s.engine.claim_backup(&r).await.expect("claim"); + same_instant.push((r, claim)); + } + let listed: Vec = s + .engine + .list_account_backup_rows(CASE_ACCOUNT, None, &TimeBounds::default()) + .try_collect() + .await + .unwrap(); + assert_eq!( + names(listed), + ["B", "a"], + "equal created_at: byte order of the ARN, whatever the database's collation" + ); + drop(same_instant); + drop(rows); + s.cleanup().await; + } + + /// A catalog at the shape before migration 5 holding a finished restore + /// and a restore the stop interrupted (its target CREATING) migrates with + /// the registrations backfilled: the finished one DONE, the interrupted + /// one RUNNING with `heartbeat_at` at the epoch and the unowned + /// generation, so the reconciler's first pass fails it and removes its + /// target. + #[tokio::test] + async fn migration_5_backfills_the_registrations_and_the_reconciler_removes_the_interrupted_one() + { + if base_conn().is_none() { + eprintln!( + "SKIP migration_5_backfills_the_registrations_and_the_reconciler_removes_the_interrupted_one: no PostgreSQL" + ); + return; + } + let s = scratch().await; + indexed_source(&s, "bf_src").await; + let backup = s + .engine + .create(ACCOUNT, "bf_src", "b", JobPermit::detached()) + .await + .expect("backup"); + let finished = restore(&s, "bf_done", &backup.backup_arn) + .await + .expect("restore"); + let interrupted = restore(&s, "bf_cut", &backup.backup_arn) + .await + .expect("restore"); + let catalog_id = s.engine.catalog_id().await.expect("id"); + + // Back to the shape before migration 5: the job record's columns and + // index gone, the registrations as the old release wrote them, the + // interrupted target CREATING, and the migration forgotten. + sqlx::raw_sql( + "DROP INDEX IF EXISTS idx_table_restores_status; \ + ALTER TABLE table_restores DROP COLUMN status, DROP COLUMN kind, \ + DROP COLUMN store_kind, DROP COLUMN store_root, DROP COLUMN store_id, \ + DROP COLUMN owner_generation, DROP COLUMN heartbeat_at, DROP COLUMN spool_host, \ + DROP COLUMN file_index, DROP COLUMN items_applied, DROP COLUMN failure_reason, \ + DROP COLUMN completed_at;", + ) + .execute(&s.catalog) + .await + .expect("drop the job record's columns"); + sqlx::query( + "UPDATE tables SET table_status = 'CREATING', status_transition_at = NULL WHERE table_id = $1", + ) + .bind(&interrupted.table_id) + .execute(&s.catalog) + .await + .expect("interrupt the second restore"); + let target = + PostgresTarget::new(s.catalog.connect_options().as_ref().clone(), &s.db_names[1]); + let runner = PostgresMigrations::open_unlocked(&target) + .await + .expect("open the runner"); + let migrations = Runner::new(&runner, "test"); + migrations.forget(5).await.expect("forget migration 5"); + let report = migrations.migrate().await.expect("migrate again"); + runner.close().await; + assert_eq!(report.applied.iter().map(|a| a.id).collect::>(), [5]); + + #[allow(clippy::type_complexity)] + let records: Vec<(String, String, String, i64, f64, Option)> = sqlx::query_as( + "SELECT t.table_name, r.status, r.kind, r.owner_generation, \ + EXTRACT(EPOCH FROM r.heartbeat_at)::FLOAT8, \ + EXTRACT(EPOCH FROM r.completed_at - t.creation_date_time)::FLOAT8 \ + FROM table_restores r JOIN tables t ON t.table_id = r.table_id ORDER BY t.table_name", + ) + .fetch_all(&s.catalog) + .await + .expect("read the records"); + assert_eq!(records.len(), 2, "{records:?}"); + let (name, status, kind, generation, heartbeat, completed) = &records[0]; + assert_eq!(name, "bf_cut"); + assert_eq!(status, "RUNNING"); + assert_eq!(kind, "legacy"); + assert_eq!(*generation, i64::try_from(UNOWNED_GENERATION).unwrap()); + assert_eq!(*heartbeat, 0.0, "the epoch, not NULL"); + assert_eq!(*completed, None); + let (name, status, kind, _, _, completed) = &records[1]; + assert_eq!(name, "bf_done"); + assert_eq!(status, "DONE"); + assert_eq!(kind, "legacy"); + assert_eq!( + *completed, + Some(0.0), + "completed when the table was created" + ); + assert_eq!( + s.engine.catalog_id().await.expect("id"), + catalog_id, + "a second run keeps the catalog id" + ); + + // The first reconciler pass: the interrupted restore is dead. + let dead = s + .engine + .list_dead_restore_jobs(std::time::Duration::from_secs(300)) + .await + .expect("list"); + assert_eq!(dead.len(), 1, "{dead:?}"); + assert_eq!( + dead[0].target, + RestoreTargetKey { + account_id: ACCOUNT.to_owned(), + table_name: "bf_cut".to_owned(), + table_id: interrupted.table_id.clone(), + } + ); + assert_eq!(dead[0].owner_generation, UNOWNED_GENERATION); + assert!( + s.engine + .fail_restore_job(&dead[0].target, "owner dead") + .await + .unwrap() + ); + assert!( + s.engine + .remove_restore_target(&dead[0].target) + .await + .unwrap() + ); + assert_eq!(table_status(&s, "bf_cut").await, None); + assert!(!data_table_exists(&s, &interrupted.table_id).await); + assert_eq!(table_status(&s, "bf_done").await.as_deref(), Some("ACTIVE")); + assert_eq!(table_status(&s, "bf_src").await.as_deref(), Some("ACTIVE")); + let _ = finished; + s.cleanup().await; + } + + /// The legacy restore registers through the job record: RUNNING under + /// the claim's generation while the copy runs, DONE with the ACTIVE flip. + #[tokio::test] + async fn a_legacy_restore_registers_a_running_job_record_and_finishes_it_done() { + if base_conn().is_none() { + eprintln!( + "SKIP a_legacy_restore_registers_a_running_job_record_and_finishes_it_done: no PostgreSQL" + ); + return; + } + let s = scratch().await; + indexed_source(&s, "jr_src").await; + let backup = s + .engine + .create(ACCOUNT, "jr_src", "b", JobPermit::detached()) + .await + .expect("backup"); + let gate = gate_copies(&s).await; + let desc = start_restore(&s, "jr_dst", &backup.backup_arn) + .await + .expect("registers"); + await_copies_waiting(&s, 1).await; + let record: (String, String, i64, f64) = sqlx::query_as( + "SELECT status, kind, owner_generation, EXTRACT(EPOCH FROM now() - heartbeat_at)::FLOAT8 \ + FROM table_restores WHERE table_id = $1", + ) + .bind(&desc.table_id) + .fetch_one(&s.catalog) + .await + .expect("record"); + assert_eq!(record.0, "RUNNING"); + assert_eq!(record.1, "legacy"); + assert_ne!(record.2, i64::try_from(UNOWNED_GENERATION).unwrap()); + assert!(record.3 < 60.0, "a fresh heartbeat: {} s old", record.3); + assert!( + s.engine + .list_dead_restore_jobs(std::time::Duration::from_secs(300)) + .await + .unwrap() + .is_empty(), + "a running restore is not dead" + ); + gate.commit().await.expect("release"); + assert_eq!( + settled_status(&s, "jr_dst").await.as_deref(), + Some("ACTIVE") + ); + let record: (String, Option) = sqlx::query_as( + "SELECT status, EXTRACT(EPOCH FROM completed_at)::FLOAT8 FROM table_restores WHERE table_id = $1", + ) + .bind(&desc.table_id) + .fetch_one(&s.catalog) + .await + .expect("record"); + assert_eq!(record.0, "DONE"); + assert!(record.1.is_some()); + s.cleanup().await; + } +} diff --git a/crates/storage-postgres/tests/migrations_fresh.rs b/crates/storage-postgres/tests/migrations_fresh.rs index 48df4f8f6..c6531eb5e 100644 --- a/crates/storage-postgres/tests/migrations_fresh.rs +++ b/crates/storage-postgres/tests/migrations_fresh.rs @@ -99,8 +99,11 @@ async fn a_fresh_install_is_current_and_a_second_run_changes_nothing() { settings_rows.get("data_database_connection_string"), Some(&s.data_url()) ); - // No baseline seeds a setting: the version and the connection rows only. - assert_eq!(settings_rows.len(), 3, "{settings_rows:?}"); + // No baseline seeds a setting: the version, the connection rows, and the + // catalog id migration 5 writes once. + assert_eq!(settings_rows.len(), 4, "{settings_rows:?}"); + let catalog_id = settings_rows.get("catalog_id").expect("a catalog id"); + assert!(uuid::Uuid::parse_str(catalog_id).is_ok(), "{catalog_id}"); assert_eq!( legacy_rows(&catalog).await, [ diff --git a/crates/storage-postgres/tests/table_snapshot.rs b/crates/storage-postgres/tests/table_snapshot.rs index 7a985c88d..9dbc4a928 100644 --- a/crates/storage-postgres/tests/table_snapshot.rs +++ b/crates/storage-postgres/tests/table_snapshot.rs @@ -1227,16 +1227,18 @@ on_both_layouts!(an_open_held_with_deletes_queued_completes); /// A restored table's snapshot description carries the same /// `restore_summary` as `describe_table`'s (both read the `table_restores` -/// row). The record is written directly, as `RestoreTableFromBackup` leaves -/// it. +/// row). The record is written directly, as a finished +/// `RestoreTableFromBackup` leaves it: a DONE legacy job record. async fn a_restored_table_describes_its_restore(shared: bool) { let s = scratch_with(shared).await; create_active(&s, snapshot_conformance::table_input("restored")).await; snapshot_conformance::populate(&s.engine, ACCOUNT, "restored", 5).await; sqlx::query( "INSERT INTO table_restores \ - (table_id, source_backup_arn, source_table_arn, restore_date_time) \ - SELECT table_id, $1, $2, '2026-10-01T12:00:00.250Z' FROM tables \ + (table_id, source_backup_arn, source_table_arn, restore_date_time, \ + status, kind, owner_generation, heartbeat_at, completed_at) \ + SELECT table_id, $1, $2, '2026-10-01T12:00:00.250Z', \ + 'DONE', 'legacy', 0, now(), now() FROM tables \ WHERE table_name = 'restored'", ) .bind("arn:aws:dynamodb:us-east-1:123456789012:table/src/backup/01759320000250-0a1b2c3d") @@ -1569,7 +1571,7 @@ on_both_layouts!(cancelling_an_open_wakes_the_active_poller); /// first drop attempt, and its end wakes the poller, which drops the table /// within 3 s rather than at its next timed attempt about 9.5 s later. async fn the_in_catalog_backup_wakes_the_active_poller(shared: bool) { - use extenddb_storage::BackupEngine as _; + use extenddb_storage::{JobPermit, LegacyBackupReader as _}; let s = scratch_with(shared).await; create_active(&s, snapshot_conformance::table_input("backed")).await; snapshot_conformance::populate(&s.engine, ACCOUNT, "backed", 20).await; @@ -1585,7 +1587,9 @@ async fn the_in_catalog_backup_wakes_the_active_poller(shared: bool) { .await .expect("hold backups"); - let backup = s.engine.create_backup(ACCOUNT, "backed", "held"); + let backup = s + .engine + .create(ACCOUNT, "backed", "held", JobPermit::detached()); let controller = async { let mut watcher = PgConnection::connect(&format!("{}/postgres", s.base)) .await diff --git a/crates/storage-postgres/tests/vector_control_plane.rs b/crates/storage-postgres/tests/vector_control_plane.rs index 6ff14fd35..1f6fc3213 100644 --- a/crates/storage-postgres/tests/vector_control_plane.rs +++ b/crates/storage-postgres/tests/vector_control_plane.rs @@ -33,7 +33,7 @@ use extenddb_core::types::{ }; use extenddb_storage::error::StorageError; use extenddb_storage::{ - BackupEngine, DataEngine, RestoreRequest, RestoreTableOverrides, TableEngine, + DataEngine, JobPermit, LegacyBackupReader, RestoreRequest, RestoreTableOverrides, TableEngine, }; use extenddb_storage_postgres::{PostgresConfig, PostgresEngine}; use sqlx::PgPool; @@ -415,11 +415,11 @@ async fn restore( target: &str, backup_arn: &str, ) -> Result, StorageError> { - let source = s.engine.backup_restore_source(ACCOUNT, backup_arn).await?; + let source = s.engine.restore_source(ACCOUNT, backup_arn).await?; source.ensure_restorable()?; let input = source.restore_input(target, &RestoreTableOverrides::default()); s.engine - .restore_table_from_backup( + .restore( ACCOUNT, RestoreRequest { backup_arn: backup_arn.to_owned(), @@ -427,6 +427,7 @@ async fn restore( restore_date_time: source.backup_creation_date_time, input, }, + JobPermit::detached(), ) .await?; let deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(60); @@ -990,7 +991,7 @@ async fn restoring_a_backup_that_carries_vector_indexes_is_refused() { .expect("create a table with a vector index"); let details = s .engine - .create_backup(ACCOUNT, "t_source", "b_vector") + .create(ACCOUNT, "t_source", "b_vector", JobPermit::detached()) .await .expect("back up a table with a vector index"); @@ -1019,7 +1020,7 @@ async fn restoring_a_backup_that_carries_vector_indexes_is_refused() { .expect("drop the definition row"); let legacy = s .engine - .backup_restore_source(ACCOUNT, &details.backup_arn) + .restore_source(ACCOUNT, &details.backup_arn) .await .expect("legacy source"); assert!(legacy.definition.is_none()); @@ -1047,7 +1048,7 @@ async fn restoring_a_backup_that_carries_vector_indexes_is_refused() { .expect("create a plain table"); let plain = s .engine - .create_backup(ACCOUNT, "t_plain_source", "b_plain") + .create(ACCOUNT, "t_plain_source", "b_plain", JobPermit::detached()) .await .expect("back up a plain table"); let status = restore(&s, "t_plain_restored", &plain.backup_arn) @@ -1302,7 +1303,7 @@ async fn a_backup_snapshot_carries_the_wire_shape_behind_a_version() { .expect("create a table with a vector index"); let details = s .engine - .create_backup(ACCOUNT, "t_snapshot", "b_snapshot") + .create(ACCOUNT, "t_snapshot", "b_snapshot", JobPermit::detached()) .await .expect("back up the table"); @@ -1356,7 +1357,7 @@ async fn a_backup_taken_before_the_snapshot_column_existed_still_restores() { .expect("create a plain table"); let details = s .engine - .create_backup(ACCOUNT, "t_legacy", "b_legacy") + .create(ACCOUNT, "t_legacy", "b_legacy", JobPermit::detached()) .await .expect("back up the table"); @@ -1378,7 +1379,7 @@ async fn a_backup_taken_before_the_snapshot_column_existed_still_restores() { .expect("drop the definition row"); let source = s .engine - .backup_restore_source(ACCOUNT, &details.backup_arn) + .restore_source(ACCOUNT, &details.backup_arn) .await .expect("source"); assert!(source.definition.is_none()); diff --git a/crates/storage-sqlite/src/backup.rs b/crates/storage-sqlite/src/backup.rs index 6d5d5b22c..6c3d9ebce 100644 --- a/crates/storage-sqlite/src/backup.rs +++ b/crates/storage-sqlite/src/backup.rs @@ -1,7 +1,7 @@ // Copyright 2026 ExtendDB contributors // SPDX-License-Identifier: Apache-2.0 -//! `BackupEngine` for the SQLite backend. +//! The in-catalog backup format on SQLite: its `LegacyBackupReader`. //! //! A backup snapshots every item's `item_data` into `backup_items` and the //! table's definition (secondary indexes, billing mode and throughput, table @@ -10,11 +10,49 @@ //! //! RestoreTableFromBackup receives the target's `CreateTableInput` already //! resolved and validated by the engine. It creates the target CREATING and -//! registers the restore in one write transaction, returns, and copies the +//! registers the restore, with its job record (RUNNING, `kind = legacy`, the +//! claim's generation), in one write transaction, returns, and copies the //! snapshot in a detached task, filling the secondary indexes as it goes in -//! bounded batches. The task flips the target ACTIVE, or removes it if the -//! copy fails; the startup sweep removes a target a process exit left -//! CREATING. +//! bounded batches while a heartbeat refreshes the record. The task flips +//! the target ACTIVE and the record DONE in one transaction, or marks the +//! record FAILED and removes the target if the copy fails; a target a +//! process exit left CREATING is removed by the engine's reconciler once its +//! record's heartbeat is stale, and a restart ages every record of the +//! stopped process at startup so that happens on the first cycle. +//! +//! A create commits a CREATING `backups` row before its copy and holds an +//! in-process slot for the row's life; a CREATING row whose slot no task +//! holds (a create that died, or one of a process that stopped) is what +//! `abandoned_creating_backups` reports for the reconciler to remove. Its +//! `stale_after` plays no part on this backend: a slot no task holds is +//! proof of death, and `backups` has no heartbeat column to age. The +//! reader's `delete` of such a row removes the row whole, with its items and +//! definition, and leaves no DELETED tombstone: a backup that never finished +//! was never visible as one, and a DELETED row would describe one. The +//! engine's DeleteBackup reaches the same `delete` for a legacy ARN, so in +//! the window before the reconciler's next cycle a client's DeleteBackup of +//! a backup ListBackups shows CREATING answers DELETED (the row is removed), +//! where main answered BackupNotFound; the service's BackupInUse for a +//! CREATING backup is the catalog-index row's rule, not this reader's. +//! +//! A restore's heartbeat runs every +//! [`JOB_HEARTBEAT_INTERVAL`](extenddb_storage::backup_catalog::JOB_HEARTBEAT_INTERVAL), +//! the one constant every backend's heartbeat loop reads and the stale floor +//! is computed from (`[backup] heartbeat_interval_secs` reaches the backends +//! in a later release), under the engine write lock, as every writer does, +//! so a long write transaction delays it; the stale window the reconciler +//! waits (`stale_job_after_secs`, at least three heartbeats) is what absorbs +//! that. +//! +//! A create holds one of [`backup_server_slot_count`] server slots for its +//! snapshot's life, besides the engine's `max_concurrent_jobs` permit it +//! arrives with. The slot stays until a later release folds it into +//! `max_concurrent_jobs`: it is what bounds the pooled connections a flood +//! of snapshot opens holds, and the two bounds can differ. A create whose +//! slot is not free is refused at once with the one `LimitExceededException` +//! text ([`limit_exceeded_message`]), as the engine's permit refuses, rather +//! than waiting for one: a queued create would hold nothing yet, but the +//! caller would learn nothing for as long as it waited. use extenddb_core::types::{ AttributeDefinition, BackupDescription, BackupDetails, BackupSummary, @@ -22,15 +60,22 @@ use extenddb_core::types::{ PointInTimeRecoveryDescription, Projection, SourceTableDetails, StreamSpecification, TableDescription, TimeToLiveDescription, TimeToLiveStatus, }; +use extenddb_storage::backup_catalog::{ + AbandonedLegacyBackup, JOB_HEARTBEAT_INTERVAL, JobBound, JobClaim, JobStatus, StatusFields, + WriteOutcome, limit_exceeded_message, new_generation, +}; use extenddb_storage::backup_definition::{ BACKUP_DEFINITION_VERSION, BackupTableDefinition, COPY_BATCH_BYTES, COPY_BATCH_ITEMS, throughput_from_catalog, }; use extenddb_storage::error::StorageError; use extenddb_storage::util::{composite_pk_to_text, parse_sk, sk_column_n}; -use extenddb_storage::{BackupEngine, BackupRestoreSource, RestoreRequest}; +use extenddb_storage::{BackupRestoreSource, JobPermit, LegacyBackupReader, RestoreRequest}; use futures::future::BoxFuture; +use crate::backup_catalog::{ + JobSlots, NOW, SqliteJobClaim, generation_column, legacy_backup_slot_key, +}; use crate::create_table::RestoreRegistration; use crate::data::{ BoundValue, all_sort_key_info, data_table_name, index_table_name, insert_index_row_multi, @@ -42,10 +87,6 @@ use crate::store::SqliteEngine; /// Upper limit on [`backup_server_slot_count`]. const BACKUP_SERVER_SLOTS_MAX: usize = 16; -/// How long a CreateBackup waits for a server slot before it is refused with -/// `LimitExceeded`. -pub(crate) const BACKUP_SLOT_WAIT: std::time::Duration = std::time::Duration::from_secs(30); - /// Backups and table snapshots the whole process may run at once, across /// accounts. The formula is stated once, on `SqliteConfig::pool_size` /// (`config.rs`); this computes it. @@ -66,28 +107,34 @@ pub(crate) fn backup_server_slot_count(pool_size: u32) -> usize { .clamp(1, BACKUP_SERVER_SLOTS_MAX) } -/// Take a server slot for a backup, waiting at most `wait`; `LimitExceeded` -/// when none frees up in time. -pub(crate) async fn acquire_backup_permit( - slots: std::sync::Arc, +/// How the slot refusal names its bound, beside the number: the engine's +/// `max_concurrent_jobs` refusal carries the same text with its own name, so +/// an operator can tell which bound a request met. The shared text reads +/// "the bound", so the name is the setting the count derives from and +/// a qualifier, with no comma: "the [storage.sqlite] pool_size snapshot-slot +/// bound". +pub(crate) const BACKUP_SERVER_SLOTS_NAME: &str = "[storage.sqlite] pool_size snapshot-slot"; + +/// Take a server slot for a backup or a table snapshot (`snapshot_engine.rs`) +/// without waiting; `LimitExceeded`, in the one text every job refusal +/// carries, naming this bound, when none is free. The slots are shared, so with snapshots +/// holding every slot a `CreateBackup` is refused with no backup running. +pub(crate) fn acquire_backup_permit( + slots: &std::sync::Arc, slot_count: usize, - wait: std::time::Duration, ) -> Result { - tokio::time::timeout(wait, slots.acquire_owned()) - .await - .map_err(|_| slots_exhausted(slot_count))? - .map_err(|e| StorageError::Internal(format!("backup slot: {e}"))) -} - -/// The refusal when every server slot is held. Backups and table snapshots -/// (`snapshot_engine.rs`) share the slots, so the message names both: with -/// snapshots holding every slot a `CreateBackup` is refused with no backup -/// running. -pub(crate) fn slots_exhausted(slot_count: usize) -> StorageError { - StorageError::LimitExceeded(format!( - "Too many backups or table snapshots are in progress on this server (at most \ - {slot_count} at once); retry later" - )) + use tokio::sync::TryAcquireError; + std::sync::Arc::clone(slots) + .try_acquire_owned() + .map_err(|e| match e { + TryAcquireError::NoPermits => { + StorageError::LimitExceeded(limit_exceeded_message(JobBound::Counted { + name: BACKUP_SERVER_SLOTS_NAME, + limit: slot_count, + })) + } + TryAcquireError::Closed => StorageError::Internal(format!("backup slot: {e}")), + }) } /// Read the next batch of `(cursor, item_data)` from `table` after @@ -272,12 +319,14 @@ impl SqliteEngine { /// each RANGE part into its typed column), and each row is a plain /// INSERT, so two backup rows for one key fail the restore instead of /// collapsing into one item. Items are read and written in byte-bounded - /// batches, each in its own write transaction, and the target is flipped - /// to ACTIVE in the last one. Returns the number of items copied. + /// batches, each in its own write transaction, and in the last one the + /// job record is marked DONE, conditioned on `generation`, and the + /// target flipped to ACTIVE. Returns the number of items copied. async fn copy_backup_items( &self, desc: &TableDescription, backup_arn: &str, + generation: u64, ) -> Result { let ddb_table = data_table_name(&desc.table_id); let sort_keys = all_sort_key_info(&desc.key_schema, &desc.attribute_definitions); @@ -321,14 +370,14 @@ impl SqliteEngine { // no one sees it part-filled. Each batch first re-checks, under // the write lock, that the backup is still AVAILABLE (DeleteBackup // removes its rows and marks it DELETED in one transaction) and - // that the target is still this restore's. + // that the target is still this restore's, under this generation. let _writer = self.write_lock.lock().await; let mut tx = self .pool .begin_with("BEGIN IMMEDIATE") .await .map_err(db_err)?; - Self::ensure_restore_can_continue(&mut tx, desc, backup_arn).await?; + Self::ensure_restore_can_continue(&mut tx, desc, backup_arn, generation).await?; let batch = next_batch( &mut tx, "backup_items", @@ -344,8 +393,30 @@ impl SqliteEngine { .fetch_one(&mut *tx) .await .map_err(db_err)?; - // Only from CREATING, which the check above confirmed under - // the same lock. + // The job record DONE, only while it is RUNNING under this + // generation, and the table ACTIVE only from CREATING, which + // the check above confirmed under the same lock. A record + // another claim took over leaves the table to that owner. + let done = sqlx::query(&format!( + "UPDATE table_restores SET status = 'DONE', items_applied = ?, \ + completed_at = {NOW} \ + WHERE table_id = ? AND status = 'RUNNING' AND owner_generation = ?" + )) + .bind(count) + .bind(&desc.table_id) + .bind(generation_column(generation)) + .execute(&mut *tx) + .await + .map_err(db_err)? + .rows_affected(); + if done != 1 { + tx.rollback().await.map_err(db_err)?; + return Err(StorageError::TableNotFound(format!( + "restore of {backup_arn} into {} did not complete: its job record \ + passed to another owner", + desc.table_name + ))); + } sqlx::query( "UPDATE tables SET item_count = ?, table_size_bytes = ?, \ table_status = 'ACTIVE', status_transition_at = NULL \ @@ -411,12 +482,14 @@ impl SqliteEngine { } } - /// Fail a restore whose backup was deleted or whose target is no longer - /// the one it created. + /// Fail a restore whose backup was deleted, whose target is no longer the + /// one it created, or whose job record is no longer RUNNING under + /// `generation`. async fn ensure_restore_can_continue( tx: &mut sqlx::Transaction<'_, sqlx::Sqlite>, desc: &TableDescription, backup_arn: &str, + generation: u64, ) -> Result<(), StorageError> { let available: bool = sqlx::query_scalar( "SELECT EXISTS(SELECT 1 FROM backups WHERE backup_arn = ? \ @@ -435,10 +508,12 @@ impl SqliteEngine { "SELECT EXISTS(SELECT 1 FROM tables t \ JOIN table_restores r ON r.table_id = t.table_id \ WHERE t.table_id = ? AND r.source_backup_arn = ? \ - AND t.table_status = 'CREATING' AND t.status_transition_at IS NULL)", + AND t.table_status = 'CREATING' AND t.status_transition_at IS NULL \ + AND r.status = 'RUNNING' AND r.owner_generation = ?)", ) .bind(&desc.table_id) .bind(backup_arn) + .bind(generation_column(generation)) .fetch_one(&mut **tx) .await .map_err(db_err)?; @@ -453,38 +528,95 @@ impl SqliteEngine { } /// The detached half of RestoreTableFromBackup: copy the backup into the - /// registered CREATING target, which flips it ACTIVE in the copy's last - /// batch, or remove the target if the copy fails. Logs either outcome; a - /// process exit part-way is cleaned up by the startup sweep. - async fn finish_restore(&self, desc: &TableDescription, backup_arn: &str) { + /// registered CREATING target while the claim heartbeats the job record, + /// which flips it ACTIVE and the record DONE in the copy's last batch, or + /// mark the record FAILED and remove the target if the copy fails. Logs + /// either outcome. + async fn finish_restore( + &self, + desc: &TableDescription, + backup_arn: &str, + claim: &SqliteJobClaim, + ) { let target = &desc.table_name; let started = std::time::Instant::now(); - match self.copy_backup_items(desc, backup_arn).await { - Ok(items) => tracing::info!( - "restore of {backup_arn} into table '{target}' finished: {items} items in \ - {:.3}s; the table is ACTIVE", - started.elapsed().as_secs_f64() - ), - Err(e) => { + let outcome = { + let copy = self.copy_backup_items(desc, backup_arn, claim.generation()); + let heartbeat = claim.heartbeat_loop(JOB_HEARTBEAT_INTERVAL); + tokio::select! { + copied = copy => copied, + lost = heartbeat => { + tracing::error!( + "restore of {backup_arn} into table '{target}': the claim lost its \ + heartbeat ({lost}); stopping the copy" + ); + Err(StorageError::Internal(format!("claim lost: {lost}"))) + } + } + }; + let e = match outcome { + Ok(items) => { + tracing::info!( + "restore of {backup_arn} into table '{target}' finished: {items} items in \ + {:.3}s; the table is ACTIVE", + started.elapsed().as_secs_f64() + ); + return; + } + Err(e) => e, + }; + // The target must not stay behind: it is CREATING with no scheduled + // transition, so nothing else would move it on, and the name would + // stay taken. The job record is marked FAILED first, conditioned on + // the generation and on RUNNING; only if that matched is the target + // removed (which cascades the record). A record another generation + // owns means its holder owns the target, and nothing further is + // touched. + let failed = StatusFields { + failure_reason: Some(e.to_string()), + completed: true, + ..StatusFields::default() + }; + match claim + .set_status(JobStatus::Running, JobStatus::Failed, &failed) + .await + { + Ok(WriteOutcome::Matched) => {} + Ok(_) => { + tracing::warn!( + "restore of {backup_arn} into table '{target}' ended with \"{e}\"; its job \ + record was no longer RUNNING under this claim, so the target ({}) was \ + left to its owner", + desc.table_id + ); + return; + } + Err(write) => { tracing::error!( - "restore of {backup_arn} into table '{target}' failed, removing the \ - partial table: {e}" + "restore of {backup_arn} into table '{target}' failed ({e}), and its job \ + record could not be marked FAILED ({write}); the reconciler removes the \ + target once its heartbeat is stale" ); - match self.abort_restore(&desc.table_id).await { - Ok(true) => tracing::info!( - "removed partially restored table '{target}' ({})", - desc.table_id - ), - // Deleted or already gone: nothing of this restore remains. - Ok(false) => {} - Err(cleanup) => tracing::error!( - "could not remove partially restored table '{target}' ({}); the next \ - startup will: {cleanup}", - desc.table_id - ), - } + return; } } + tracing::error!( + "restore of {backup_arn} into table '{target}' failed, removing the partial \ + table: {e}" + ); + match self.abort_restore(&desc.table_id).await { + Ok(true) => tracing::info!( + "removed partially restored table '{target}' ({})", + desc.table_id + ), + // Deleted or already gone: nothing of this restore remains. + Ok(false) => {} + Err(cleanup) => tracing::error!( + "could not remove partially restored table '{target}' ({}); the reconciler \ + will: {cleanup}", + desc.table_id + ), + } } /// Remove a restore target whose copy failed or was abandoned: its @@ -492,7 +624,7 @@ impl SqliteEngine { /// target still CREATING with no scheduled transition is removed, and by /// table id, so this can never touch a table the restore did not create /// or one a client already deleted. - async fn abort_restore(&self, table_id: &str) -> Result { + pub(crate) async fn abort_restore(&self, table_id: &str) -> Result { let _writer = self.write_lock.lock().await; let mut tx = self .pool @@ -815,62 +947,15 @@ impl SqliteEngine { tx.commit().await.map_err(db_err)?; Ok(removed) } - - /// Remove backups left CREATING by a process that died during its copy. - /// The backups delete cascades to definitions and any copied item rows. - pub async fn sweep_incomplete_backups(&self) -> Result, StorageError> { - let _writer = self.write_lock.lock().await; - let mut tx = self - .pool - .begin_with("BEGIN IMMEDIATE") - .await - .map_err(db_err)?; - let removed: Vec = sqlx::query_scalar( - "DELETE FROM backups WHERE backup_status = 'CREATING' RETURNING backup_arn", - ) - .fetch_all(&mut *tx) - .await - .map_err(db_err)?; - tx.commit().await.map_err(db_err)?; - Ok(removed) - } - - /// Remove restore targets left by a process that died mid-restore. - /// - /// A restore target is the only table that sits CREATING with no - /// scheduled transition. Called once at startup, before any request is - /// served, so no restore of this process can be in flight and every such - /// table is abandoned. - /// - /// This assumes one server process per database file, which the backend - /// already depends on: writers are serialized by an in-process lock, so a - /// second process writing the same file is unsupported regardless. A - /// second process started against the file while the first is mid-restore - /// would remove that restore's target. - pub(crate) async fn sweep_abandoned_restores(&self) -> Result, StorageError> { - let candidates: Vec<(String, String)> = sqlx::query_as( - "SELECT table_id, table_name FROM tables \ - WHERE table_status = 'CREATING' AND status_transition_at IS NULL", - ) - .fetch_all(&self.pool) - .await - .map_err(db_err)?; - let mut removed = Vec::new(); - for (table_id, table_name) in candidates { - if self.abort_restore(&table_id).await? { - removed.push(table_name); - } - } - Ok(removed) - } } -impl BackupEngine for SqliteEngine { - fn create_backup( +impl LegacyBackupReader for SqliteEngine { + fn create( &self, account_id: &str, table_name: &str, backup_name: &str, + job_permit: JobPermit, ) -> BoxFuture<'_, Result> { let engine = self.clone(); let account_id = account_id.to_owned(); @@ -880,20 +965,20 @@ impl BackupEngine for SqliteEngine { // Admission: at most backup_server_slot_count backups run at once, // so a flood of CreateBackups cannot hold every pooled connection // (each holds its snapshot reader for its whole duration). The - // permit travels with the detached task and is released when the - // copy ends, whatever happened to the request. + // slot is tried, not waited for. The permit travels with the + // detached task and is released when the copy ends, whatever + // happened to the request. let permit = acquire_backup_permit( - std::sync::Arc::clone(&engine.backup_server_slots), + &engine.backup_server_slots, engine.backup_server_slot_count, - BACKUP_SLOT_WAIT, - ) - .await?; + )?; // Detach the operation from the request future. Once the CREATING // row commits, dropping an HTTP request must not drop the copy and // strand wire-visible state. A dropped JoinHandle leaves its Tokio // task running, so the copy reaches AVAILABLE or executes cleanup. let task = tokio::spawn(async move { let _permit = permit; + let _job_permit = job_permit; // One id gives the ARN and the creation time, so the ARN, the // reported time, and the ListBackups cursor agree. let backup_id = extenddb_storage::util::new_backup_id(); @@ -907,6 +992,15 @@ impl BackupEngine for SqliteEngine { .ok_or_else(|| { StorageError::Internal(format!("backup id {backup_id} has no timestamp")) })?; + // The in-process slot that marks the CREATING row as owned, + // taken before the row exists and held until the row is + // AVAILABLE or removed. The id is fresh, so the slot is free. + let _slot = JobSlots::take(&engine.job_slots, legacy_backup_slot_key(&backup_arn)) + .ok_or_else(|| { + StorageError::Internal(format!( + "backup {backup_arn} is already being created" + )) + })?; let created = if engine.in_memory { engine .create_backup_single_connection( @@ -932,7 +1026,7 @@ impl BackupEngine for SqliteEngine { if let Err(cleanup) = engine.remove_incomplete_backup(&backup_arn).await { tracing::error!( "backup {backup_arn} failed ({error}); could not remove its CREATING \ - row: {cleanup}; the next startup will retry cleanup" + row: {cleanup}; the reconciler removes it once this task is gone" ); } return Err(error); @@ -945,7 +1039,7 @@ impl BackupEngine for SqliteEngine { }) } - fn describe_backup( + fn describe( &self, account_id: &str, backup_arn: &str, @@ -1007,7 +1101,7 @@ impl BackupEngine for SqliteEngine { }) } - fn list_backups( + fn list( &self, account_id: &str, table_name: Option<&str>, @@ -1066,7 +1160,7 @@ impl BackupEngine for SqliteEngine { }) } - fn delete_backup( + fn delete( &self, account_id: &str, backup_arn: &str, @@ -1076,8 +1170,14 @@ impl BackupEngine for SqliteEngine { Box::pin(async move { // Resolves account-scoped, so a backup owned by another account is // reported missing here and the writes below never run. - let desc = self.describe_backup(&account_id, &backup_arn).await?; - if desc.backup_details.backup_status != "AVAILABLE" { + let desc = self.describe(&account_id, &backup_arn).await?; + // A CREATING row whose create is dead (no task of this process + // holds its slot) is deletable: nothing will ever finish it, and + // the reconciler removes it through this path. One whose create + // is running is not a backup yet. + let dead_creating = desc.backup_details.backup_status == "CREATING" + && !self.job_slots.holds(&legacy_backup_slot_key(&backup_arn)); + if desc.backup_details.backup_status != "AVAILABLE" && !dead_creating { return Err(StorageError::Validation(format!( "Backup not found: {backup_arn}" ))); @@ -1101,18 +1201,10 @@ impl BackupEngine for SqliteEngine { // Refuse while a restore from this backup is still running, as the // service does. Under the write lock, which the restore also takes // to record itself, so the check cannot miss one. - let restoring: Option = sqlx::query_scalar( - "SELECT t.table_name FROM table_restores r JOIN tables t ON t.table_id = r.table_id \ - WHERE r.source_backup_arn = ? AND t.table_status = 'CREATING' LIMIT 1", - ) - .bind(&backup_arn) - .fetch_optional(&mut *tx) - .await - .map_err(db_err)?; - if let Some(table) = restoring { - return Err(StorageError::BackupInUse(format!( - "Backup is being used to restore table {table}: {backup_arn}" - ))); + if let Some(table) = + crate::backup_catalog::creating_restore_target(&mut *tx, &backup_arn).await? + { + return Err(crate::backup_catalog::backup_in_use(&table, &backup_arn)); } sqlx::query( "DELETE FROM backup_items WHERE backup_arn = ?1 AND EXISTS (\ @@ -1132,15 +1224,29 @@ impl BackupEngine for SqliteEngine { .execute(&mut *tx) .await .map_err(|e| StorageError::Internal(e.to_string()))?; - sqlx::query( - "UPDATE backups SET backup_status = 'DELETED' \ - WHERE backup_arn = ? AND account_id = ?", - ) - .bind(&backup_arn) - .bind(&account_id) - .execute(&mut *tx) - .await - .map_err(|e| StorageError::Internal(e.to_string()))?; + if dead_creating { + // Never a backup: no tombstone, as the failed create would + // have left none. + sqlx::query( + "DELETE FROM backups WHERE backup_arn = ? AND account_id = ? \ + AND backup_status = 'CREATING'", + ) + .bind(&backup_arn) + .bind(&account_id) + .execute(&mut *tx) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + } else { + sqlx::query( + "UPDATE backups SET backup_status = 'DELETED' \ + WHERE backup_arn = ? AND account_id = ?", + ) + .bind(&backup_arn) + .bind(&account_id) + .execute(&mut *tx) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + } tx.commit().await.map_err(db_err)?; Ok(BackupDescription { backup_details: BackupDetails { @@ -1153,7 +1259,7 @@ impl BackupEngine for SqliteEngine { }) } - fn backup_restore_source( + fn restore_source( &self, account_id: &str, backup_arn: &str, @@ -1252,10 +1358,11 @@ impl BackupEngine for SqliteEngine { }) } - fn restore_table_from_backup( + fn restore( &self, account_id: &str, request: RestoreRequest, + permit: JobPermit, ) -> BoxFuture<'_, Result> { let engine = self.clone(); let account_id = account_id.to_owned(); @@ -1263,7 +1370,8 @@ impl BackupEngine for SqliteEngine { // The whole restore runs in a task that owns its inputs, so the // request future being dropped (a client timeout) stops neither the // registration nor the copy. The request waits only until the - // CREATING target and its table_restores row have committed. + // CREATING target and its job record have committed. The job + // permit lives with the claim and is freed when the copy ends. let (registered_tx, registered_rx) = tokio::sync::oneshot::channel(); tokio::spawn(async move { let RestoreRequest { @@ -1274,12 +1382,16 @@ impl BackupEngine for SqliteEngine { } = request; // One write transaction: re-check the backup is AVAILABLE and // owned by the account, insert the CREATING target, record the - // restore. DeleteBackup takes the same lock, so it sees either - // no target or a CREATING target that names this backup. + // restore with its job record under a fresh generation (the + // insert is the claim). DeleteBackup takes the same lock, so + // it sees either no target or a CREATING target that names + // this backup. + let generation = new_generation(); let registration = RestoreRegistration { backup_arn: &backup_arn, source_table_arn: &source_table_arn, restore_date_time, + owner_generation: generation, }; let registered = engine .create_table_for_restore(&account_id, input, ®istration) @@ -1300,9 +1412,11 @@ impl BackupEngine for SqliteEngine { return; } }; + let claim = + SqliteJobClaim::for_restore(&engine, &desc.table_id, generation, permit); // A closed receiver means the request is gone; the copy goes on. let _ = registered_tx.send(Ok(desc.clone())); - engine.finish_restore(&desc, &backup_arn).await; + engine.finish_restore(&desc, &backup_arn, &claim).await; }); registered_rx.await.map_err(|_| { StorageError::Internal( @@ -1312,6 +1426,35 @@ impl BackupEngine for SqliteEngine { }) } + /// CREATING `backups` rows no running create holds: a create commits its + /// row before the copy and holds an in-process slot for it until the row + /// is AVAILABLE or removed, so a row with no slot was left by a create + /// that died or by a process that stopped (a restart frees every slot). + /// A slot no task holds is proof of death here, so `stale_after` plays no + /// part. + fn abandoned_creating_backups( + &self, + _stale_after: std::time::Duration, + ) -> BoxFuture<'_, Result, StorageError>> { + Box::pin(async move { + let creating: Vec<(String, String)> = sqlx::query_as( + "SELECT account_id, backup_arn FROM backups WHERE backup_status = 'CREATING' \ + ORDER BY backup_arn", + ) + .fetch_all(&self.pool) + .await + .map_err(db_err)?; + Ok(creating + .into_iter() + .filter(|(_, arn)| !self.job_slots.holds(&legacy_backup_slot_key(arn))) + .map(|(account_id, backup_arn)| AbandonedLegacyBackup { + account_id, + backup_arn, + }) + .collect()) + }) + } + fn describe_continuous_backups( &self, account_id: &str, @@ -1408,10 +1551,15 @@ mod restore_tests { use extenddb_core::types::{Item, TableDescription, TableKeyInfo}; use extenddb_storage::error::StorageError; - use extenddb_storage::{BackupEngine, DataEngine, RestoreRequest, RestoreTableOverrides}; + use extenddb_storage::{ + DataEngine, JobPermit, LegacyBackupReader, RestoreRequest, RestoreTableOverrides, + }; use serde_json::json; - use super::{acquire_backup_permit, backup_server_slot_count}; + use super::{ + BACKUP_SERVER_SLOTS_NAME, JobBound, acquire_backup_permit, backup_server_slot_count, + limit_exceeded_message, + }; use crate::SqliteEngine; use crate::create_table::RestoreRegistration; @@ -1424,7 +1572,7 @@ mod restore_tests { target: &str, backup_arn: &str, ) -> Result { - let source = engine.backup_restore_source(ACCOUNT, backup_arn).await?; + let source = engine.restore_source(ACCOUNT, backup_arn).await?; source.ensure_restorable()?; let input = source.restore_input(target, &RestoreTableOverrides::default()); Ok(RestoreRequest { @@ -1450,7 +1598,9 @@ mod restore_tests { backup_arn: &str, ) -> Result { let request = restore_request(engine, target, backup_arn).await?; - engine.restore_table_from_backup(ACCOUNT, request).await + engine + .restore(ACCOUNT, request, JobPermit::detached()) + .await } /// Poll the target's catalog row until its detached copy is done: `true` @@ -1609,7 +1759,7 @@ mod restore_tests { let engine = engine().await; let src = indexed_table(&engine, "src").await; let backup = engine - .create_backup(ACCOUNT, "src", "bkp") + .create(ACCOUNT, "src", "bkp", JobPermit::detached()) .await .expect("backup"); let desc = restore(&engine, "dst", &backup.backup_arn) @@ -1653,41 +1803,133 @@ mod restore_tests { ); } + /// The job record's status column for a target. + async fn job_status(engine: &SqliteEngine, table_id: &str) -> Option { + sqlx::query_scalar("SELECT status FROM table_restores WHERE table_id = ?") + .bind(table_id) + .fetch_optional(&engine.pool) + .await + .expect("read the job status") + } + + /// A restore target whose owner died is found by the reconciler only once + /// its job record's heartbeat is older than the stale window, whatever + /// the table's age; failing the record and removing the target each + /// happen once; a key with the wrong table id never removes a table + /// re-created under the name. #[tokio::test] - async fn startup_sweep_removes_an_abandoned_restore() { + async fn abandoned_restore_is_removed_only_when_its_heartbeat_is_stale() { + use extenddb_storage::BackupCatalog; + use extenddb_storage::backup_catalog::{RestoreKind, RestoreTargetKey}; + let stale = Duration::from_secs(300); let engine = engine().await; indexed_table(&engine, "src").await; let backup = engine - .create_backup(ACCOUNT, "src", "bkp") + .create(ACCOUNT, "src", "bkp", JobPermit::detached()) .await .expect("backup"); let desc = restore(&engine, "dst", &backup.backup_arn) .await .expect("restore"); + assert_eq!( + job_status(&engine, &desc.table_id).await.as_deref(), + Some("DONE"), + "a finished restore's record is DONE" + ); let index_ids: Vec = index_tables(&engine, &desc.table_id) .await .into_iter() .map(|(_, id)| id) .collect(); + let key = RestoreTargetKey { + account_id: ACCOUNT.to_owned(), + table_name: "dst".to_owned(), + table_id: desc.table_id.clone(), + }; // The state a process killed mid-copy leaves: the target CREATING with - // no scheduled transition (its copy transaction rolled back). + // no scheduled transition, its job record RUNNING with a heartbeat + // `age_secs` old. + let crash = |age_secs: i64| { + let pool = engine.pool.clone(); + let id = desc.table_id.clone(); + async move { + sqlx::query( + "UPDATE tables SET table_status = 'CREATING', status_transition_at = NULL \ + WHERE table_id = ?", + ) + .bind(&id) + .execute(&pool) + .await + .expect("simulate a crash"); + sqlx::query( + "UPDATE table_restores SET status = 'RUNNING', completed_at = NULL, \ + heartbeat_at = strftime('%Y-%m-%dT%H:%M:%fZ', 'now', ? || ' seconds') \ + WHERE table_id = ?", + ) + .bind(-age_secs) + .bind(&id) + .execute(&pool) + .await + .expect("age the heartbeat"); + } + }; + + // A fresh heartbeat: a live restore, even though the table is old. + crash(1).await; sqlx::query( - "UPDATE tables SET table_status = 'CREATING', status_transition_at = NULL \ - WHERE table_id = ?", + "UPDATE tables SET creation_date_time = strftime('%Y-%m-%dT%H:%M:%fZ', 'now', \ + '-10 minutes')", ) - .bind(&desc.table_id) .execute(&engine.pool) .await - .expect("simulate crash"); + .expect("age every table"); + assert!( + engine + .list_dead_restore_jobs(stale) + .await + .expect("list") + .is_empty() + ); + assert_eq!( + status_of(&engine, &desc.table_id).await.as_deref(), + Some("CREATING") + ); - let removed = engine.sweep_abandoned_restores().await.expect("sweep"); - assert_eq!(removed, vec!["dst".to_owned()]); - let rows: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM tables WHERE table_name = 'dst'") - .fetch_one(&engine.pool) - .await - .expect("rows"); - assert_eq!(rows, 0); + // A stale heartbeat: dead. + crash(600).await; + let dead = engine.list_dead_restore_jobs(stale).await.expect("list"); + assert_eq!(dead.len(), 1, "{dead:?}"); + assert_eq!(dead[0].target, key); + assert_eq!(dead[0].kind, RestoreKind::Legacy); + assert_eq!(dead[0].source_backup_arn, backup.backup_arn); + assert!(dead[0].heartbeat_at > 0.0); + assert!( + engine + .fail_restore_job(&key, "owner dead") + .await + .expect("fail") + ); + assert_eq!( + job_status(&engine, &desc.table_id).await.as_deref(), + Some("FAILED") + ); + let reason: Option = + sqlx::query_scalar("SELECT failure_reason FROM table_restores WHERE table_id = ?") + .bind(&desc.table_id) + .fetch_one(&engine.pool) + .await + .expect("reason"); + assert_eq!(reason.as_deref(), Some("owner dead")); + assert!( + !engine + .fail_restore_job(&key, "owner dead") + .await + .expect("fail again"), + "a record that is no longer RUNNING is not written twice" + ); + assert!(engine.remove_restore_target(&key).await.expect("remove")); + assert_eq!(status_of(&engine, &desc.table_id).await, None); let mut leftover = vec![desc.table_id.clone()]; leftover.extend(index_ids); for id in leftover { @@ -1700,96 +1942,362 @@ mod restore_tests { .expect("exists"); assert!(!exists, "data table _ddb_{id} left behind"); } + let registrations: i64 = + sqlx::query_scalar("SELECT COUNT(*) FROM table_restores WHERE table_id = ?") + .bind(&desc.table_id) + .fetch_one(&engine.pool) + .await + .expect("registrations"); + assert_eq!(registrations, 0, "the record goes with its target"); + assert!( + !engine + .remove_restore_target(&key) + .await + .expect("remove again"), + "a removed target is removed once" + ); - // A live table is never a candidate, and the name is free again. + // An ACTIVE table is never a candidate, the name is free again, and a + // key with the wrong table id (a table re-created under the name) + // does not remove the new table. + let again = restore(&engine, "dst", &backup.backup_arn) + .await + .expect("restore again"); assert!( engine - .sweep_abandoned_restores() + .list_dead_restore_jobs(stale) .await - .expect("sweep") + .expect("list") .is_empty() ); - restore(&engine, "dst", &backup.backup_arn) - .await - .expect("restore again"); + assert!(!engine.remove_restore_target(&key).await.expect("stale key")); + sqlx::query( + "UPDATE tables SET table_status = 'CREATING', status_transition_at = NULL \ + WHERE table_id = ?", + ) + .bind(&again.table_id) + .execute(&engine.pool) + .await + .expect("make it a target again"); + assert!( + !engine.remove_restore_target(&key).await.expect("stale key"), + "the old table id never removes the new table" + ); + assert_eq!( + status_of(&engine, &again.table_id).await.as_deref(), + Some("CREATING") + ); } + /// The FAILED-target duty's listing: a FAILED record whose table is still + /// a CREATING restore target (the FAILED write landed and the removal did + /// not) is not listed while the record is fresh (the task that failed it + /// may still be removing the target), is listed once `completed_at` is + /// older than the stale window, as FAILED, and its target is then removed + /// with no second FAILED write; a FAILED record without a `completed_at` + /// is aged by its heartbeat; a FAILED record whose table is DELETING or + /// ACTIVE is never listed. #[tokio::test] - async fn startup_sweep_removes_an_incomplete_backup_and_its_rows() { + async fn a_failed_record_whose_target_removal_failed_is_listed_once_stale_and_removed() { + use extenddb_storage::BackupCatalog; + use extenddb_storage::backup_catalog::{JobStatus, RestoreKind, RestoreTargetKey}; + let stale = Duration::from_secs(300); let engine = engine().await; indexed_table(&engine, "src").await; let backup = engine - .create_backup(ACCOUNT, "src", "bkp") + .create(ACCOUNT, "src", "bkp", JobPermit::detached()) .await .expect("backup"); - sqlx::query("UPDATE backups SET backup_status = 'CREATING' WHERE backup_arn = ?") - .bind(&backup.backup_arn) - .execute(&engine.pool) - .await - .expect("simulate crash"); - let listed = engine - .list_backups(ACCOUNT, Some("src")) - .await - .expect("list CREATING backup"); - assert_eq!(listed.len(), 1); - assert_eq!(listed[0].backup_status, "CREATING"); - let described = engine - .describe_backup(ACCOUNT, &backup.backup_arn) - .await - .expect("describe CREATING backup"); - assert_eq!(described.backup_details.backup_status, "CREATING"); - engine - .delete_backup(ACCOUNT, &backup.backup_arn) - .await - .expect_err("DeleteBackup only accepts AVAILABLE backups"); - - let removed = engine - .sweep_incomplete_backups() + let desc = restore(&engine, "dst", &backup.backup_arn) .await - .expect("backup sweep"); - assert_eq!(removed, vec![backup.backup_arn.clone()]); - let backup_rows: i64 = - sqlx::query_scalar("SELECT COUNT(*) FROM backups WHERE backup_arn = ?") - .bind(&backup.backup_arn) - .fetch_one(&engine.pool) - .await - .expect("backup rows"); - let item_rows: i64 = - sqlx::query_scalar("SELECT COUNT(*) FROM backup_items WHERE backup_arn = ?") - .bind(&backup.backup_arn) - .fetch_one(&engine.pool) - .await - .expect("item rows"); - let definition_rows: i64 = - sqlx::query_scalar("SELECT COUNT(*) FROM backup_definitions WHERE backup_arn = ?") - .bind(&backup.backup_arn) - .fetch_one(&engine.pool) + .expect("restore"); + let key = RestoreTargetKey { + account_id: ACCOUNT.to_owned(), + table_name: "dst".to_owned(), + table_id: desc.table_id.clone(), + }; + let set_record = |sql: &'static str| { + let pool = engine.pool.clone(); + let id = desc.table_id.clone(); + async move { + sqlx::query(sql) + .bind(&id) + .execute(&pool) + .await + .expect("set the record"); + } + }; + let set_table_status = |status: &'static str| { + let pool = engine.pool.clone(); + let id = desc.table_id.clone(); + async move { + sqlx::query( + "UPDATE tables SET table_status = ?, status_transition_at = NULL \ + WHERE table_id = ?", + ) + .bind(status) + .bind(&id) + .execute(&pool) .await - .expect("definition rows"); - assert_eq!((backup_rows, item_rows, definition_rows), (0, 0, 0)); + .expect("set the table status"); + } + }; + let listed = || async { engine.list_dead_restore_jobs(stale).await.expect("list") }; + + // The state a dead restore leaves once the reconciler (or the copy's + // own failure path) failed its record and the target's removal did + // not land. + set_table_status("CREATING").await; + set_record( + "UPDATE table_restores SET status = 'RUNNING', completed_at = NULL, \ + heartbeat_at = strftime('%Y-%m-%dT%H:%M:%fZ', 'now', '-10 minutes') \ + WHERE table_id = ?", + ) + .await; + let dead = listed().await; + assert_eq!(dead.len(), 1, "{dead:?}"); + assert_eq!(dead[0].status, JobStatus::Running); assert!( engine - .sweep_incomplete_backups() + .fail_restore_job(&key, "owner dead") .await - .expect("second sweep") - .is_empty() + .expect("fail") + ); + assert_eq!( + job_status(&engine, &desc.table_id).await.as_deref(), + Some("FAILED") + ); + assert_eq!( + status_of(&engine, &desc.table_id).await.as_deref(), + Some("CREATING") ); - } - #[tokio::test] - async fn failed_backup_copy_removes_its_creating_row() { - let (engine, path) = file_engine().await; - let source = plain_table(&engine, "source").await; - let table = crate::data::data_table_name(&source.table_id); - sqlx::query(&format!( - "INSERT INTO {table} (pk, item_data) VALUES ('broken', 'not json')" - )) - .execute(&engine.pool) - .await - .expect("corrupt source row"); + // Fresh: `fail_restore_job` stamped `completed_at` a moment ago, and + // whoever failed it may still be removing the target. + assert!( + listed().await.is_empty(), + "a FAILED record within the stale window is left to its writer" + ); + // Stale: listed, as FAILED, for the FAILED-target duty. + set_record( + "UPDATE table_restores SET completed_at = \ + strftime('%Y-%m-%dT%H:%M:%fZ', 'now', '-10 minutes') WHERE table_id = ?", + ) + .await; + let dead = listed().await; + assert_eq!(dead.len(), 1, "{dead:?}"); + assert_eq!(dead[0].target, key); + assert_eq!(dead[0].status, JobStatus::Failed); + assert_eq!(dead[0].kind, RestoreKind::Legacy); + assert_eq!(dead[0].source_backup_arn, backup.backup_arn); + // A FAILED record without `completed_at` is aged by its heartbeat: + // fresh, not listed; stale, listed. + set_record( + "UPDATE table_restores SET completed_at = NULL, \ + heartbeat_at = strftime('%Y-%m-%dT%H:%M:%fZ', 'now') WHERE table_id = ?", + ) + .await; + assert!( + listed().await.is_empty(), + "a FAILED record with a fresh heartbeat and no completed_at is fresh" + ); + set_record( + "UPDATE table_restores SET heartbeat_at = \ + strftime('%Y-%m-%dT%H:%M:%fZ', 'now', '-10 minutes') WHERE table_id = ?", + ) + .await; + assert_eq!(listed().await.len(), 1); + // A FAILED record whose table the removal already moved to DELETING + // (the claim landed, the drop did not) is the control plane's row. + set_table_status("DELETING").await; + assert!( + listed().await.is_empty(), + "a DELETING table is not a target" + ); + set_table_status("CREATING").await; + + // The reconciler's FAILED-target duty: no second FAILED write, the + // target removed, with its record and its data tables. + assert!( + !engine + .fail_restore_job(&key, "owner dead") + .await + .expect("fail again"), + "a FAILED record is not RUNNING and is not written FAILED again" + ); + assert!(engine.remove_restore_target(&key).await.expect("remove")); + assert_eq!(status_of(&engine, &desc.table_id).await, None); + let data_table: bool = sqlx::query_scalar( + "SELECT EXISTS(SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = ?)", + ) + .bind(format!("_ddb_{}", desc.table_id)) + .fetch_one(&engine.pool) + .await + .expect("exists"); + assert!(!data_table, "data table left behind"); + let registrations: i64 = + sqlx::query_scalar("SELECT COUNT(*) FROM table_restores WHERE table_id = ?") + .bind(&desc.table_id) + .fetch_one(&engine.pool) + .await + .expect("registrations"); + assert_eq!(registrations, 0, "the record goes with its target"); + assert!(listed().await.is_empty()); + engine + .delete(ACCOUNT, &backup.backup_arn) + .await + .expect("the backup is free once the target is gone"); + + // A FAILED record whose table is ACTIVE (a flip that won against a + // late FAILED write) is nobody's duty. + let backup = engine + .create(ACCOUNT, "src", "bkp2", JobPermit::detached()) + .await + .expect("backup again"); + let again = restore(&engine, "dst", &backup.backup_arn) + .await + .expect("restore again"); + assert_eq!( + status_of(&engine, &again.table_id).await.as_deref(), + Some("ACTIVE") + ); + sqlx::query( + "UPDATE table_restores SET status = 'FAILED', completed_at = \ + strftime('%Y-%m-%dT%H:%M:%fZ', 'now', '-10 minutes') WHERE table_id = ?", + ) + .bind(&again.table_id) + .execute(&engine.pool) + .await + .expect("fail the record of an ACTIVE table"); + assert!( + listed().await.is_empty(), + "a FAILED record of an ACTIVE table is not listed" + ); + } + + /// A CREATING backup row is reported as abandoned only while no task of + /// this process holds its slot: a running create is left alone and its + /// row is not deletable; a dead one (a crash, or the previous run) is + /// reported, and the reader's `delete` removes it with its rows, as the + /// reconciler does. A finished backup is never reported. + #[tokio::test] + async fn an_abandoned_creating_backup_is_reported_and_removed_through_delete() { + use crate::backup_catalog::{JobSlots, legacy_backup_slot_key}; + let stale = Duration::from_secs(300); + let engine = engine().await; + indexed_table(&engine, "src").await; + let backup = engine + .create(ACCOUNT, "src", "bkp", JobPermit::detached()) + .await + .expect("backup"); + assert!( + engine + .abandoned_creating_backups(stale) + .await + .expect("list") + .is_empty(), + "an AVAILABLE backup is not abandoned" + ); + sqlx::query("UPDATE backups SET backup_status = 'CREATING' WHERE backup_arn = ?") + .bind(&backup.backup_arn) + .execute(&engine.pool) + .await + .expect("simulate a crash"); + let listed = engine + .list(ACCOUNT, Some("src")) + .await + .expect("list CREATING backup"); + assert_eq!(listed.len(), 1); + assert_eq!(listed[0].backup_status, "CREATING"); + let described = engine + .describe(ACCOUNT, &backup.backup_arn) + .await + .expect("describe CREATING backup"); + assert_eq!(described.backup_details.backup_status, "CREATING"); + + // The slot a running create holds: not abandoned, not deletable. + let held = JobSlots::take( + &engine.job_slots, + legacy_backup_slot_key(&backup.backup_arn), + ) + .expect("free slot"); + assert!( + engine + .abandoned_creating_backups(stale) + .await + .expect("list") + .is_empty(), + "a CREATING row whose create is running is not abandoned" + ); + let err = engine + .delete(ACCOUNT, &backup.backup_arn) + .await + .expect_err("a backup still being created is not deletable"); + assert!( + matches!(&err, StorageError::Validation(m) if m.starts_with("Backup not found")), + "{err:?}" + ); + drop(held); + + // No slot: dead. The reconciler removes it through `delete`. + let abandoned = engine + .abandoned_creating_backups(stale) + .await + .expect("list"); + assert_eq!(abandoned.len(), 1, "{abandoned:?}"); + assert_eq!(abandoned[0].account_id, ACCOUNT); + assert_eq!(abandoned[0].backup_arn, backup.backup_arn); + let removed = engine + .delete(&abandoned[0].account_id, &abandoned[0].backup_arn) + .await + .expect("delete the abandoned backup"); + assert_eq!(removed.backup_details.backup_status, "DELETED"); + let leftovers: (i64, i64, i64) = sqlx::query_as( + "SELECT \ + (SELECT COUNT(*) FROM backups WHERE backup_arn = ?1), \ + (SELECT COUNT(*) FROM backup_items WHERE backup_arn = ?1), \ + (SELECT COUNT(*) FROM backup_definitions WHERE backup_arn = ?1)", + ) + .bind(&backup.backup_arn) + .fetch_one(&engine.pool) + .await + .expect("leftovers"); + assert_eq!( + leftovers, + (0, 0, 0), + "a backup that never finished leaves no tombstone" + ); + assert!( + engine + .abandoned_creating_backups(stale) + .await + .expect("second list") + .is_empty() + ); + let err = engine + .describe(ACCOUNT, &backup.backup_arn) + .await + .expect_err("gone"); + assert!( + matches!(&err, StorageError::Validation(m) if m.starts_with("Backup not found")), + "{err:?}" + ); + } + + #[tokio::test] + async fn failed_backup_copy_removes_its_creating_row() { + let (engine, path) = file_engine().await; + let source = plain_table(&engine, "source").await; + let table = crate::data::data_table_name(&source.table_id); + sqlx::query(&format!( + "INSERT INTO {table} (pk, item_data) VALUES ('broken', 'not json')" + )) + .execute(&engine.pool) + .await + .expect("corrupt source row"); engine - .create_backup(ACCOUNT, "source", "broken") + .create(ACCOUNT, "source", "broken", JobPermit::detached()) .await .expect_err("invalid source item must fail the backup"); let leftovers: (i64, i64, i64) = sqlx::query_as( @@ -1812,7 +2320,7 @@ mod restore_tests { let engine = engine().await; indexed_table(&engine, "src").await; let backup = engine - .create_backup(ACCOUNT, "src", "bkp") + .create(ACCOUNT, "src", "bkp", JobPermit::detached()) .await .expect("backup"); // Corrupt one backup row so the copy fails part-way. @@ -1859,7 +2367,7 @@ mod restore_tests { let engine = engine().await; indexed_table(&engine, "src").await; let backup = engine - .create_backup(ACCOUNT, "src", "bkp") + .create(ACCOUNT, "src", "bkp", JobPermit::detached()) .await .expect("backup"); let input = serde_json::from_value(json!({ @@ -1881,6 +2389,7 @@ mod restore_tests { backup_arn: &backup.backup_arn, source_table_arn: "arn:aws:dynamodb:us-east-1:000000000000:table/src", restore_date_time: backup.backup_creation_date_time, + owner_generation: extenddb_storage::backup_catalog::new_generation(), }, ) .await @@ -1902,7 +2411,7 @@ mod restore_tests { assert_eq!(state, (true, true)); let err = engine - .delete_backup(ACCOUNT, &backup.backup_arn) + .delete(ACCOUNT, &backup.backup_arn) .await .expect_err("restore intent must block deletion"); assert!( @@ -1919,7 +2428,7 @@ mod restore_tests { .expect("intent count"); assert_eq!(intents, 0, "target cleanup must remove restore intent"); engine - .delete_backup(ACCOUNT, &backup.backup_arn) + .delete(ACCOUNT, &backup.backup_arn) .await .expect("backup is deletable after abort"); } @@ -1954,7 +2463,7 @@ mod restore_tests { ) .await; let backup = engine - .create_backup(ACCOUNT, "m", "bkp") + .create(ACCOUNT, "m", "bkp", JobPermit::detached()) .await .expect("backup"); // The refusal is the engine's (`ensure_restorable` in `restore`); this @@ -1993,7 +2502,7 @@ mod restore_tests { ) .await; let backup = engine - .create_backup(ACCOUNT, "c", "bkp") + .create(ACCOUNT, "c", "bkp", JobPermit::detached()) .await .expect("backup"); restore(&engine, "c2", &backup.backup_arn) @@ -2077,7 +2586,7 @@ mod restore_tests { .expect("put"); } let backup = engine - .create_backup(ACCOUNT, "src", "bkp") + .create(ACCOUNT, "src", "bkp", JobPermit::detached()) .await .expect("backup"); let desc = restore(&engine, "dst", &backup.backup_arn) @@ -2288,7 +2797,7 @@ mod restore_tests { let engine = engine().await; indexed_table(&engine, "src").await; let backup = engine - .create_backup(ACCOUNT, "src", "bkp") + .create(ACCOUNT, "src", "bkp", JobPermit::detached()) .await .expect("backup"); let desc = restore(&engine, "dst", &backup.backup_arn) @@ -2324,7 +2833,7 @@ mod restore_tests { let engine = engine().await; indexed_table(&engine, "src").await; let backup = engine - .create_backup(ACCOUNT, "src", "bkp") + .create(ACCOUNT, "src", "bkp", JobPermit::detached()) .await .expect("backup"); let desc = restore(&engine, "dst", &backup.backup_arn) @@ -2373,7 +2882,7 @@ mod restore_tests { .restore_in_progress ); let err = engine - .delete_backup(ACCOUNT, &backup.backup_arn) + .delete(ACCOUNT, &backup.backup_arn) .await .expect_err("in use"); assert!( @@ -2387,7 +2896,7 @@ mod restore_tests { .await .expect("finished"); engine - .delete_backup(ACCOUNT, &backup.backup_arn) + .delete(ACCOUNT, &backup.backup_arn) .await .expect("deletable once the restore is done"); let after = describe("dst").await.restore_summary.expect("summary"); @@ -2549,7 +3058,7 @@ mod restore_tests { let started = Instant::now(); engine - .create_backup(ACCOUNT, "source", "concurrent") + .create(ACCOUNT, "source", "concurrent", JobPermit::detached()) .await .expect("backup"); let finished = Instant::now(); @@ -2594,7 +3103,7 @@ mod restore_tests { let backup_engine = engine.clone(); let backup = tokio::spawn(async move { backup_engine - .create_backup(ACCOUNT, "source", "snapshot") + .create(ACCOUNT, "source", "snapshot", JobPermit::detached()) .await }); loop { @@ -2670,7 +3179,12 @@ mod restore_tests { let request_engine = engine.clone(); let request = tokio::spawn(async move { request_engine - .create_backup(ACCOUNT, "source", "cancelled-request") + .create( + ACCOUNT, + "source", + "cancelled-request", + JobPermit::detached(), + ) .await }); tokio::time::timeout(std::time::Duration::from_secs(10), async { @@ -2824,7 +3338,7 @@ mod restore_tests { let (engine, path) = file_engine().await; bulk_indexed_table(&engine, "src", BULK_ITEMS).await; let backup = engine - .create_backup(ACCOUNT, "src", "bkp") + .create(ACCOUNT, "src", "bkp", JobPermit::detached()) .await .expect("backup"); assert!( @@ -2881,7 +3395,7 @@ mod restore_tests { let (engine, path) = file_engine().await; bulk_indexed_table(&engine, "src", BULK_ITEMS).await; let backup = engine - .create_backup(ACCOUNT, "src", "bkp") + .create(ACCOUNT, "src", "bkp", JobPermit::detached()) .await .expect("backup"); let request = restore_request(&engine, "dst", &backup.backup_arn) @@ -2889,7 +3403,7 @@ mod restore_tests { .expect("request"); let polled_once = engine - .restore_table_from_backup(ACCOUNT, request) + .restore(ACCOUNT, request, JobPermit::detached()) .now_or_never(); assert!( polled_once.is_none(), @@ -2933,7 +3447,7 @@ mod restore_tests { let (engine, path) = file_engine().await; bulk_indexed_table(&engine, "src", BULK_ITEMS).await; let backup = engine - .create_backup(ACCOUNT, "src", "bkp") + .create(ACCOUNT, "src", "bkp", JobPermit::detached()) .await .expect("backup"); let desc = start_restore(&engine, "dst", &backup.backup_arn) @@ -2941,7 +3455,7 @@ mod restore_tests { .expect("restore"); let err = engine - .delete_backup(ACCOUNT, &backup.backup_arn) + .delete(ACCOUNT, &backup.backup_arn) .await .expect_err("the source backup is in use while the copy runs"); assert!(matches!(err, StorageError::BackupInUse(_)), "{err:?}"); @@ -2975,7 +3489,7 @@ mod restore_tests { BULK_ITEMS ); engine - .delete_backup(ACCOUNT, &backup.backup_arn) + .delete(ACCOUNT, &backup.backup_arn) .await .expect("deletable once the restore is done"); delete_target() @@ -2985,19 +3499,24 @@ mod restore_tests { } /// A target registered but never copied (a process that exited between - /// registration and copy) is removed by the startup sweep, its restore - /// registration with it, which frees the source backup. + /// registration and copy) holds its backup in use. At startup every + /// RUNNING record's heartbeat is aged to the epoch, so the reconciler's + /// duties find the restore dead on their first run, fail it and remove + /// the target, its registration with it, which frees the source backup. #[tokio::test] - async fn startup_sweep_removes_a_registered_target_that_was_never_copied() { + async fn a_restart_ages_an_interrupted_restore_so_the_reconciler_removes_it() { + use extenddb_storage::BackupCatalog; + use extenddb_storage::backup_catalog::{RestoreTargetKey, UNOWNED_GENERATION}; let engine = engine().await; indexed_table(&engine, "src").await; let backup = engine - .create_backup(ACCOUNT, "src", "bkp") + .create(ACCOUNT, "src", "bkp", JobPermit::detached()) .await .expect("backup"); let request = restore_request(&engine, "dst", &backup.backup_arn) .await .expect("request"); + let generation = extenddb_storage::backup_catalog::new_generation(); let desc = engine .create_table_for_restore( ACCOUNT, @@ -3006,18 +3525,72 @@ mod restore_tests { backup_arn: &request.backup_arn, source_table_arn: &request.source_table_arn, restore_date_time: request.restore_date_time, + owner_generation: generation, }, ) .await .expect("register without a copy"); let err = engine - .delete_backup(ACCOUNT, &backup.backup_arn) + .delete(ACCOUNT, &backup.backup_arn) .await .expect_err("registered restore holds the backup"); assert!(matches!(err, StorageError::BackupInUse(_)), "{err:?}"); + let stale = Duration::from_secs(300); + assert!( + engine + .list_dead_restore_jobs(stale) + .await + .expect("list") + .is_empty(), + "a record registered a moment ago is not stale" + ); + + // The startup step: the previous process owned every RUNNING record. + let aged = engine + .expire_restore_job_heartbeats() + .await + .expect("expire"); + assert_eq!(aged, vec!["dst".to_owned()]); + let (heartbeat, record_generation): (String, i64) = sqlx::query_as( + "SELECT heartbeat_at, owner_generation FROM table_restores WHERE table_id = ?", + ) + .bind(&desc.table_id) + .fetch_one(&engine.pool) + .await + .expect("record"); + assert_eq!(heartbeat, "1970-01-01T00:00:00.000Z"); + assert_eq!( + record_generation, + i64::try_from(generation).unwrap(), + "aging touches the heartbeat alone" + ); + assert!( + engine + .expire_restore_job_heartbeats() + .await + .expect("expire again") + .is_empty(), + "a record is reported once" + ); - let removed = engine.sweep_abandoned_restores().await.expect("sweep"); - assert_eq!(removed, vec!["dst".to_owned()]); + // The reconciler's duties on the first cycle. + let dead = engine.list_dead_restore_jobs(stale).await.expect("list"); + assert_eq!(dead.len(), 1, "{dead:?}"); + let target = RestoreTargetKey { + account_id: ACCOUNT.to_owned(), + table_name: "dst".to_owned(), + table_id: desc.table_id.clone(), + }; + assert_eq!(dead[0].target, target); + assert_eq!(dead[0].heartbeat_at, 0.0); + assert_ne!(dead[0].owner_generation, UNOWNED_GENERATION); + assert!( + engine + .fail_restore_job(&target, "owner dead") + .await + .expect("fail") + ); + assert!(engine.remove_restore_target(&target).await.expect("remove")); assert_eq!(status_of(&engine, &desc.table_id).await, None); let registrations: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM table_restores WHERE table_id = ?") @@ -3027,9 +3600,9 @@ mod restore_tests { .expect("registrations"); assert_eq!(registrations, 0); engine - .delete_backup(ACCOUNT, &backup.backup_arn) + .delete(ACCOUNT, &backup.backup_arn) .await - .expect("the backup is free after the sweep"); + .expect("the backup is free once the target is gone"); } /// table_restores records the source table ARN and the restore time the @@ -3040,7 +3613,7 @@ mod restore_tests { let engine = engine().await; indexed_table(&engine, "src").await; let backup = engine - .create_backup(ACCOUNT, "src", "bkp") + .create(ACCOUNT, "src", "bkp", JobPermit::detached()) .await .expect("backup"); let mut request = restore_request(&engine, "dst", &backup.backup_arn) @@ -3051,7 +3624,7 @@ mod restore_tests { request.restore_date_time = 1_791_976_431.438; request.source_table_arn = "arn:aws:dynamodb:us-east-1:000000000000:table/elsewhere".into(); let desc = engine - .restore_table_from_backup(ACCOUNT, request) + .restore(ACCOUNT, request, JobPermit::detached()) .await .expect("restore"); assert!(wait_for_copy(&engine, &desc.table_id).await); @@ -3102,7 +3675,7 @@ mod restore_tests { let mut fractional = None; for i in 0..20 { let backup = engine - .create_backup(ACCOUNT, "src", &format!("bkp{i}")) + .create(ACCOUNT, "src", &format!("bkp{i}"), JobPermit::detached()) .await .expect("backup"); let id = backup.backup_arn.rsplit('/').next().expect("id").to_owned(); @@ -3116,12 +3689,12 @@ mod restore_tests { extenddb_storage::util::backup_arn("us-east-1", ACCOUNT, "src", &id) ); let described = engine - .describe_backup(ACCOUNT, &backup.backup_arn) + .describe(ACCOUNT, &backup.backup_arn) .await .expect("describe"); assert_eq!(described.backup_details.backup_creation_date_time, from_id); let source = engine - .backup_restore_source(ACCOUNT, &backup.backup_arn) + .restore_source(ACCOUNT, &backup.backup_arn) .await .expect("source"); assert_eq!(source.backup_creation_date_time, from_id); @@ -3132,10 +3705,7 @@ mod restore_tests { tokio::time::sleep(Duration::from_millis(1)).await; } let fractional = fractional.expect("some backup with a non-zero millisecond"); - let listed = engine - .list_backups(ACCOUNT, Some("src")) - .await - .expect("list"); + let listed = engine.list(ACCOUNT, Some("src")).await.expect("list"); let summary = listed .iter() .find(|b| b.backup_arn == fractional) @@ -3168,11 +3738,11 @@ mod restore_tests { .await .expect("stream"); let backup = engine - .create_backup(ACCOUNT, "src", "bkp") + .create(ACCOUNT, "src", "bkp", JobPermit::detached()) .await .expect("backup"); let source = engine - .backup_restore_source(ACCOUNT, &backup.backup_arn) + .restore_source(ACCOUNT, &backup.backup_arn) .await .expect("source"); let table = engine @@ -3213,7 +3783,7 @@ mod restore_tests { ), ] { let err = engine - .backup_restore_source(account, arn) + .restore_source(account, arn) .await .expect_err("not found"); assert!( @@ -3336,7 +3906,12 @@ mod restore_tests { let engine = engine.clone(); async move { engine - .create_backup(ACCOUNT, "source", &format!("flood{i}")) + .create( + ACCOUNT, + "source", + &format!("flood{i}"), + JobPermit::detached(), + ) .await } })) @@ -3393,11 +3968,11 @@ mod restore_tests { let engine = engine().await; indexed_table(&engine, "src").await; let b1 = engine - .create_backup(ACCOUNT, "src", "b1") + .create(ACCOUNT, "src", "b1", JobPermit::detached()) .await .expect("backup 1"); let b2 = engine - .create_backup(ACCOUNT, "src", "b2") + .create(ACCOUNT, "src", "b2", JobPermit::detached()) .await .expect("backup 2"); let desc = restore(&engine, "dst", &b1.backup_arn) @@ -3456,7 +4031,10 @@ mod restore_tests { .await .expect("request"); missing.backup_arn = flip_last_hex(&b2.backup_arn); - match engine.restore_table_from_backup(ACCOUNT, missing).await { + match engine + .restore(ACCOUNT, missing, JobPermit::detached()) + .await + { Err(StorageError::TableBeingRestored { table_name, backup_arn, @@ -3472,6 +4050,19 @@ mod restore_tests { .execute(&engine.pool) .await .expect("finished"); + // The in-use rule reads the target's status, not the registration: + // an ACTIVE former target with nothing else CREATING holds the backup + // no longer, for the rule's own answer and for a second restore. + let b1_key = extenddb_storage::backup_catalog::BackupKey::from_arn( + &extenddb_core::backup_format::BackupArn::parse(&b1.backup_arn).expect("arn"), + ); + assert_eq!( + extenddb_storage::BackupCatalog::backup_in_use(&engine, &b1_key) + .await + .expect("in use"), + None, + "an ACTIVE former target does not hold its backup in use" + ); restore(&engine, "dst2", &b1.backup_arn) .await .expect("a restore once the first target is ACTIVE"); @@ -3482,9 +4073,10 @@ mod restore_tests { ); } - /// The bound's arithmetic and its refusal, without a database. + /// The bound's arithmetic and its refusal, without a database: a full + /// bound refuses with the one job-refusal text and does not wait. #[tokio::test] - async fn backup_slots_follow_the_pool_and_refuse_after_the_wait() { + async fn backup_slots_follow_the_pool_and_refuse_without_waiting() { assert_eq!(backup_server_slot_count(2), 1); assert_eq!(backup_server_slot_count(3), 1); assert_eq!(backup_server_slot_count(4), 1); @@ -3493,18 +4085,752 @@ mod restore_tests { assert_eq!(backup_server_slot_count(34), 16); assert_eq!(backup_server_slot_count(1_000), 16); let slots = std::sync::Arc::new(tokio::sync::Semaphore::new(1)); - let held = acquire_backup_permit(slots.clone(), 1, Duration::from_millis(50)) - .await - .expect("first"); - match acquire_backup_permit(slots.clone(), 1, Duration::from_millis(50)).await { + let held = acquire_backup_permit(&slots, 1).expect("first"); + match acquire_backup_permit(&slots, 1) { Err(StorageError::LimitExceeded(m)) => { - assert!(m.contains("in progress on this server (at most 1"), "{m}"); + assert_eq!( + m, + limit_exceeded_message(JobBound::Counted { + name: BACKUP_SERVER_SLOTS_NAME, + limit: 1, + }) + ); + assert!( + m.contains( + "(at most 1 at once, the [storage.sqlite] pool_size snapshot-slot bound)" + ), + "{m}" + ); } other => panic!("a full bound refuses: {:?}", other.err()), } drop(held); - let _freed = acquire_backup_permit(slots, 1, Duration::from_millis(50)) + let _freed = acquire_backup_permit(&slots, 1).expect("freed"); + } + + /// With every server slot held, CreateBackup is refused at once, with the + /// `LimitExceededException` text the engine's `max_concurrent_jobs` permit + /// answers, rather than waiting for a slot to free up; the refusal writes + /// no row, and a freed slot admits the next create. + #[tokio::test] + async fn a_create_with_no_free_slot_is_refused_at_once_with_the_one_text() { + use std::time::Instant; + let engine = engine().await; + indexed_table(&engine, "src").await; + let held: Vec = (0..engine.backup_server_slot_count) + .map(|_| { + std::sync::Arc::clone(&engine.backup_server_slots) + .try_acquire_owned() + .expect("a free slot") + }) + .collect(); + assert_eq!(engine.backup_server_slots.available_permits(), 0); + + let started = Instant::now(); + let refused = tokio::time::timeout( + Duration::from_secs(5), + engine.create(ACCOUNT, "src", "bkp", JobPermit::detached()), + ) + .await + .expect("a create with no free slot is refused at once, not after a wait"); + match refused { + Err(StorageError::LimitExceeded(m)) => { + assert_eq!( + m, + limit_exceeded_message(JobBound::Counted { + name: BACKUP_SERVER_SLOTS_NAME, + limit: engine.backup_server_slot_count, + }) + ); + } + other => panic!("a full bound refuses: {:?}", other.err()), + } + assert!( + started.elapsed() < Duration::from_secs(1), + "refused in {:?}", + started.elapsed() + ); + let rows: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM backups") + .fetch_one(&engine.pool) + .await + .expect("rows"); + assert_eq!(rows, 0, "a refused create writes nothing"); + + drop(held); + engine + .create(ACCOUNT, "src", "bkp", JobPermit::detached()) .await - .expect("freed"); + .expect("admitted once a slot is free"); + } + + /// The catalog index: `backup_index` rows, their claim, the restore job + /// record and the migration's backfill. + mod catalog_index { + use extenddb_core::backup_format::BackupArn; + use extenddb_storage::BackupCatalog; + use extenddb_storage::backup_catalog::{ + BackupRow, BackupStatus, DeleteStart, Heartbeat, JobStatus, RestoreTargetKey, + StatusFields, StoreIdentity, StoreKind, TimeBounds, UNOWNED_GENERATION, WriteOutcome, + }; + use futures::TryStreamExt; + + use super::*; + + const REGION: &str = "us-east-1"; + const OTHER_ACCOUNT: &str = "111111111111"; + + fn store() -> StoreIdentity { + StoreIdentity { + store_kind: StoreKind::Filesystem, + store_root: "/var/lib/extenddb/backups".to_owned(), + store_id: "store-1".to_owned(), + } + } + + /// A row for a backup of `table` in `account` minted now, with the + /// manifest fixture's source table. + fn row(account: &str, table: &str, name: &str) -> BackupRow { + let backup_id = extenddb_storage::util::new_backup_id(); + let backup_arn = extenddb_storage::util::backup_arn(REGION, account, table, &backup_id); + let manifest: serde_json::Value = serde_json::from_str(include_str!( + "../../core/testdata/extenddb-manifest/v1.json" + )) + .expect("fixture"); + BackupRow { + backup_id: backup_id.clone(), + account_id: account.to_owned(), + table_name: table.to_owned(), + backup_arn: backup_arn.clone(), + backup_name: name.to_owned(), + status: BackupStatus::Creating, + store: store(), + size_bytes: 0, + item_count: 0, + created_at: extenddb_storage::util::backup_creation_date_time(&backup_id).unwrap(), + completed_at: None, + failure_reason: None, + source_table_details_json: manifest["source_table"].clone(), + owner_generation: 0, + heartbeat_at: 0.0, + } + } + + fn arn_of(row: &BackupRow) -> BackupArn { + BackupArn::parse(&row.backup_arn).unwrap() + } + + /// The claim: a second claim of the same backup is refused while the + /// first is held, the heartbeat confirms the claim and refreshes the + /// row, a status write is conditioned on the generation and the + /// `from` status, and the row is read back by account, backup id, + /// table and ARN. + #[tokio::test] + async fn claim_backup_excludes_a_second_owner_and_fences_its_writes() { + let engine = engine().await; + assert!( + uuid::Uuid::parse_str(&engine.catalog_id().await.expect("id")).is_ok(), + "the catalog has an id from the migration" + ); + assert!(!engine.check_maintenance_claim().await.expect("check")); + let row = row(ACCOUNT, "Music", "nightly"); + let claim = engine.claim_backup(&row).await.expect("claim"); + assert_ne!(claim.generation(), UNOWNED_GENERATION); + let second = engine.claim_backup(&row).await.err(); + assert!( + matches!(second, Some(StorageError::AlreadyExists(_))), + "{second:?}" + ); + let stored = engine + .read_backup_row(&arn_of(&row)) + .await + .expect("read") + .expect("the row exists"); + assert_eq!(stored.status, BackupStatus::Creating); + assert_eq!(stored.owner_generation, claim.generation()); + assert_eq!(stored.store, store()); + assert!( + (stored.created_at - row.created_at).abs() < 1e-6, + "{} is the id's instant {}", + stored.created_at, + row.created_at + ); + assert_eq!( + stored.source_table_details_json, + row.source_table_details_json + ); + let before = stored.heartbeat_at; + assert!(before > 0.0); + tokio::time::sleep(Duration::from_millis(20)).await; + assert_eq!( + claim.heartbeat().await.expect("heartbeat"), + Heartbeat::Confirmed + ); + let after = engine + .read_backup_row(&arn_of(&row)) + .await + .unwrap() + .unwrap() + .heartbeat_at; + assert!(after > before, "{after} > {before}"); + + // The same id under another account, table or region finds nothing. + let other_account = + BackupArn::new("aws", REGION, OTHER_ACCOUNT, "Music", &row.backup_id).unwrap(); + assert!( + engine + .read_backup_row(&other_account) + .await + .unwrap() + .is_none() + ); + let other_table = + BackupArn::new("aws", REGION, ACCOUNT, "Other", &row.backup_id).unwrap(); + assert!( + engine + .read_backup_row(&other_table) + .await + .unwrap() + .is_none() + ); + let other_region = + BackupArn::new("aws", "eu-west-1", ACCOUNT, "Music", &row.backup_id).unwrap(); + assert!( + engine + .read_backup_row(&other_region) + .await + .unwrap() + .is_none() + ); + + // A write from the wrong state matches nothing; the right one + // matches once, and the row then carries what it set. + let fields = StatusFields { + size_bytes: Some(1234), + item_count: Some(5), + failure_reason: None, + completed: true, + }; + assert_eq!( + claim + .set_status(JobStatus::Running, JobStatus::Available, &fields) + .await + .unwrap(), + WriteOutcome::NotMatched + ); + assert_eq!( + claim + .set_status(JobStatus::Creating, JobStatus::Available, &fields) + .await + .unwrap(), + WriteOutcome::Matched + ); + assert_eq!( + claim + .set_status(JobStatus::Creating, JobStatus::Available, &fields) + .await + .unwrap(), + WriteOutcome::NotMatched + ); + let read = claim.read().await.expect("read"); + assert_eq!(read.status, JobStatus::Available); + assert_eq!(read.owner_generation, claim.generation()); + assert!(read.heartbeat_at >= after); + let stored = engine + .read_backup_row(&arn_of(&row)) + .await + .unwrap() + .unwrap(); + assert_eq!(stored.status, BackupStatus::Available); + assert_eq!(stored.size_bytes, 1234); + assert_eq!(stored.item_count, 5); + assert!(stored.completed_at.is_some()); + + // A row another generation took over confirms nothing for this + // claim. + sqlx::query( + "UPDATE backup_index SET owner_generation = owner_generation + 1 \ + WHERE backup_id = ?", + ) + .bind(&row.backup_id) + .execute(&engine.pool) + .await + .expect("take over"); + assert_eq!(claim.heartbeat().await.expect("heartbeat"), Heartbeat::Lost); + assert_eq!( + claim + .set_status( + JobStatus::Available, + JobStatus::Failed, + &StatusFields::default() + ) + .await + .unwrap(), + WriteOutcome::NotMatched + ); + drop(claim); + // The slot went with the claim: the row can be claimed by a + // successor only through its insert, which the existing row + // refuses. + let third = engine.claim_backup(&row).await.err(); + assert!( + matches!(third, Some(StorageError::AlreadyExists(_))), + "{third:?}" + ); + } + + /// DeleteBackup's first step and the tombstone writes: only an + /// AVAILABLE row moves to DELETING; a CREATING row answers its status; + /// a backup a CREATING restore target reads answers the target's + /// name; DELETING moves to DELETED once; a DELETED row is removed + /// once; a missing row is "Backup not found". + #[tokio::test] + async fn start_backup_delete_checks_the_status_and_the_in_use_rule_under_the_lock() { + let engine = engine().await; + let row = row(ACCOUNT, "Music", "nightly"); + let key = row.key(); + assert!(matches!( + engine.start_backup_delete(&key).await, + Err(StorageError::Validation(m)) if m.contains("Backup not found") + )); + let claim = engine.claim_backup(&row).await.expect("claim"); + assert_eq!( + engine.start_backup_delete(&key).await.unwrap(), + DeleteStart::Status(BackupStatus::Creating) + ); + assert_eq!( + claim + .set_status( + JobStatus::Creating, + JobStatus::Available, + &StatusFields::default() + ) + .await + .unwrap(), + WriteOutcome::Matched + ); + assert_eq!(engine.backup_in_use(&key).await.unwrap(), None); + + // A CREATING restore target registered from the backup: in use, + // naming the target; an ACTIVE one is not. + let target_id = indexed_table(&engine, "del_target").await; + sqlx::query(&format!( + "INSERT INTO table_restores (table_id, source_backup_arn, source_table_arn, \ + restore_date_time, status, kind, owner_generation, heartbeat_at) \ + VALUES (?, ?, 'arn:aws:dynamodb:us-east-1:123456789012:table/Music', {NOW}, \ + 'RUNNING', 'manifest', 7, {NOW})", + NOW = crate::backup_catalog::NOW + )) + .bind(&target_id) + .bind(&row.backup_arn) + .execute(&engine.pool) + .await + .expect("register a target"); + sqlx::query( + "UPDATE tables SET table_status = 'CREATING', status_transition_at = NULL \ + WHERE table_id = ?", + ) + .bind(&target_id) + .execute(&engine.pool) + .await + .expect("make it CREATING"); + assert_eq!( + engine.backup_in_use(&key).await.unwrap().as_deref(), + Some("del_target") + ); + assert_eq!( + engine.start_backup_delete(&key).await.unwrap(), + DeleteStart::InUse("del_target".to_owned()) + ); + assert_eq!( + engine + .read_backup_row(&arn_of(&row)) + .await + .unwrap() + .unwrap() + .status, + BackupStatus::Available, + "a refused delete leaves the row AVAILABLE" + ); + sqlx::query("UPDATE tables SET table_status = 'ACTIVE' WHERE table_id = ?") + .bind(&target_id) + .execute(&engine.pool) + .await + .expect("make it ACTIVE"); + assert_eq!(engine.backup_in_use(&key).await.unwrap(), None); + + assert_eq!( + engine.start_backup_delete(&key).await.unwrap(), + DeleteStart::Deleting + ); + assert_eq!( + engine.start_backup_delete(&key).await.unwrap(), + DeleteStart::Status(BackupStatus::Deleting), + "a retry finds the row DELETING" + ); + assert!( + !engine.remove_backup_row(&key).await.unwrap(), + "DELETING is not a tombstone" + ); + assert!(engine.set_backup_deleted(&key).await.unwrap()); + assert!(!engine.set_backup_deleted(&key).await.unwrap()); + let stored = engine + .read_backup_row(&arn_of(&row)) + .await + .unwrap() + .unwrap(); + assert_eq!(stored.status, BackupStatus::Deleted); + assert!(stored.completed_at.is_some()); + assert!(engine.remove_backup_row(&key).await.unwrap()); + assert!(!engine.remove_backup_row(&key).await.unwrap()); + assert!( + engine + .read_backup_row(&arn_of(&row)) + .await + .unwrap() + .is_none() + ); + drop(claim); + } + + /// The listings: by account with an optional table and the time + /// bounds (lower inclusive, upper exclusive) in (created_at, ARN) + /// order, and by store and status. + #[tokio::test] + async fn the_listings_are_scoped_and_ordered() { + let engine = engine().await; + let mut rows = Vec::new(); + for (account, table, name) in [ + (ACCOUNT, "Music", "a"), + (ACCOUNT, "Music", "b"), + (ACCOUNT, "Other", "c"), + (OTHER_ACCOUNT, "Music", "d"), + ] { + tokio::time::sleep(Duration::from_millis(5)).await; + let row = row(account, table, name); + let claim = engine.claim_backup(&row).await.expect("claim"); + if name == "b" { + assert_eq!( + claim + .set_status( + JobStatus::Creating, + JobStatus::Failed, + &StatusFields::default() + ) + .await + .unwrap(), + WriteOutcome::Matched + ); + } + rows.push((row, claim)); + } + let names = |listed: Vec| -> Vec { + listed.into_iter().map(|r| r.backup_name).collect() + }; + let all: Vec = engine + .list_account_backup_rows(ACCOUNT, None, &TimeBounds::default()) + .try_collect() + .await + .unwrap(); + assert_eq!( + names(all.clone()), + ["a", "b", "c"], + "every status, in creation order" + ); + assert_eq!(all[1].status, BackupStatus::Failed); + let music: Vec = engine + .list_account_backup_rows(ACCOUNT, Some("Music"), &TimeBounds::default()) + .try_collect() + .await + .unwrap(); + assert_eq!(names(music), ["a", "b"]); + let bounds = TimeBounds { + lower: Some(rows[1].0.created_at), + upper: Some(rows[2].0.created_at), + }; + let bounded: Vec = engine + .list_account_backup_rows(ACCOUNT, None, &bounds) + .try_collect() + .await + .unwrap(); + assert_eq!(names(bounded), ["b"], "lower inclusive, upper exclusive"); + let other: Vec = engine + .list_account_backup_rows(OTHER_ACCOUNT, None, &TimeBounds::default()) + .try_collect() + .await + .unwrap(); + assert_eq!(names(other), ["d"]); + + let by_store: Vec = engine + .list_backup_rows(None, &store()) + .try_collect() + .await + .unwrap(); + assert_eq!(names(by_store), ["a", "b", "c", "d"]); + let failed: Vec = engine + .list_backup_rows(Some(BackupStatus::Failed), &store()) + .try_collect() + .await + .unwrap(); + assert_eq!(names(failed), ["b"]); + let elsewhere = StoreIdentity { + store_id: "store-2".to_owned(), + ..store() + }; + let none: Vec = engine + .list_backup_rows(None, &elsewhere) + .try_collect() + .await + .unwrap(); + assert!(none.is_empty(), "another store's rows are not listed"); + drop(rows); + } + + /// A catalog at the shape before migration 5 holding a finished + /// restore and a restore the stop interrupted (its target CREATING) + /// migrates with the registrations backfilled: the finished one DONE, + /// the interrupted one RUNNING with `heartbeat_at` at the epoch and + /// the unowned generation, so the reconciler's first pass fails it + /// and removes its target. + #[tokio::test] + async fn migration_5_backfills_the_registrations_and_the_reconciler_removes_the_interrupted_one() + { + let engine = engine().await; + indexed_table(&engine, "bf_src").await; + let backup = engine + .create(ACCOUNT, "bf_src", "b", JobPermit::detached()) + .await + .expect("backup"); + let finished = restore(&engine, "bf_done", &backup.backup_arn) + .await + .expect("restore"); + let interrupted = restore(&engine, "bf_cut", &backup.backup_arn) + .await + .expect("restore"); + let catalog_id = engine.catalog_id().await.expect("id"); + + // Back to the shape before migration 5: the job record's columns + // and index gone, the registrations as the old release wrote + // them, the interrupted target CREATING, and the migration + // forgotten. + let mut undo = String::from("DROP INDEX IF EXISTS idx_table_restores_status;"); + for column in [ + "status", + "kind", + "store_kind", + "store_root", + "store_id", + "owner_generation", + "heartbeat_at", + "spool_host", + "file_index", + "items_applied", + "failure_reason", + "completed_at", + ] { + undo.push_str(&format!( + " ALTER TABLE table_restores DROP COLUMN {column};" + )); + } + sqlx::raw_sql(&undo) + .execute(&engine.pool) + .await + .expect("drop the job record's columns"); + sqlx::query( + "UPDATE tables SET table_status = 'CREATING', status_transition_at = NULL \ + WHERE table_id = ?", + ) + .bind(&interrupted.table_id) + .execute(&engine.pool) + .await + .expect("interrupt the second restore"); + let on_pool = + crate::migrations::SqliteMigrations::on_pool(engine.pool.clone(), ":memory:") + .expect("one connection"); + let runner = extenddb_storage::migrations::Runner::new(&on_pool, "test"); + runner.forget(5).await.expect("forget migration 5"); + let report = runner.migrate().await.expect("migrate again"); + assert_eq!(report.applied.iter().map(|a| a.id).collect::>(), [5]); + + #[allow(clippy::type_complexity)] + let records: Vec<(String, String, String, i64, String, Option)> = sqlx::query_as( + "SELECT t.table_name, r.status, r.kind, r.owner_generation, r.heartbeat_at, \ + r.completed_at = t.creation_date_time \ + FROM table_restores r JOIN tables t ON t.table_id = r.table_id \ + ORDER BY t.table_name", + ) + .fetch_all(&engine.pool) + .await + .expect("read the records"); + assert_eq!(records.len(), 2, "{records:?}"); + let (name, status, kind, generation, heartbeat, completed) = &records[0]; + assert_eq!(name, "bf_cut"); + assert_eq!(status, "RUNNING"); + assert_eq!(kind, "legacy"); + assert_eq!(*generation, i64::try_from(UNOWNED_GENERATION).unwrap()); + assert_eq!(heartbeat, "1970-01-01T00:00:00.000Z", "the epoch, not NULL"); + assert_eq!(*completed, None); + let (name, status, kind, _, _, completed) = &records[1]; + assert_eq!(name, "bf_done"); + assert_eq!(status, "DONE"); + assert_eq!(kind, "legacy"); + assert_eq!( + *completed, + Some(true), + "completed when the table was created" + ); + assert_eq!( + engine.catalog_id().await.expect("id"), + catalog_id, + "a second run keeps the catalog id" + ); + + // The first reconciler pass: the interrupted restore is dead. + let dead = engine + .list_dead_restore_jobs(Duration::from_secs(300)) + .await + .expect("list"); + assert_eq!(dead.len(), 1, "{dead:?}"); + assert_eq!( + dead[0].target, + RestoreTargetKey { + account_id: ACCOUNT.to_owned(), + table_name: "bf_cut".to_owned(), + table_id: interrupted.table_id.clone(), + } + ); + assert_eq!(dead[0].owner_generation, UNOWNED_GENERATION); + assert_eq!(dead[0].heartbeat_at, 0.0); + assert!( + engine + .fail_restore_job(&dead[0].target, "owner dead") + .await + .unwrap() + ); + assert!(engine.remove_restore_target(&dead[0].target).await.unwrap()); + assert_eq!(status_of(&engine, &interrupted.table_id).await, None); + let exists: bool = sqlx::query_scalar( + "SELECT EXISTS(SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = ?)", + ) + .bind(crate::data::data_table_name(&interrupted.table_id)) + .fetch_one(&engine.pool) + .await + .expect("exists"); + assert!(!exists, "the interrupted target's data table is dropped"); + assert_eq!( + status_of(&engine, &finished.table_id).await.as_deref(), + Some("ACTIVE") + ); + } + + /// The legacy restore registers through the job record: RUNNING under + /// the claim's generation while the copy runs, DONE with the ACTIVE + /// flip, and never dead while it runs. + #[tokio::test] + async fn a_legacy_restore_registers_a_running_job_record_and_finishes_it_done() { + let (engine, path) = file_engine().await; + bulk_indexed_table(&engine, "jr_src", BULK_ITEMS).await; + let backup = engine + .create(ACCOUNT, "jr_src", "b", JobPermit::detached()) + .await + .expect("backup"); + let desc = start_restore(&engine, "jr_dst", &backup.backup_arn) + .await + .expect("registers"); + // At return the copy has not finished (several batches remain), + // so the record is read while the restore runs. + assert_eq!( + status_of(&engine, &desc.table_id).await.as_deref(), + Some("CREATING"), + "the restore must return before its copy completes" + ); + let record: (String, String, i64, f64) = sqlx::query_as( + "SELECT status, kind, owner_generation, \ + (julianday('now') - julianday(heartbeat_at)) * 86400.0 \ + FROM table_restores WHERE table_id = ?", + ) + .bind(&desc.table_id) + .fetch_one(&engine.pool) + .await + .expect("record"); + assert_eq!(record.0, "RUNNING"); + assert_eq!(record.1, "legacy"); + assert_ne!(record.2, i64::try_from(UNOWNED_GENERATION).unwrap()); + assert!(record.3 < 60.0, "a fresh heartbeat: {} s old", record.3); + assert!( + engine + .list_dead_restore_jobs(Duration::from_secs(300)) + .await + .unwrap() + .is_empty(), + "a running restore is not dead" + ); + assert!( + wait_for_copy(&engine, &desc.table_id).await, + "target removed" + ); + let record: (String, Option, Option) = sqlx::query_as( + "SELECT status, completed_at, items_applied FROM table_restores \ + WHERE table_id = ?", + ) + .bind(&desc.table_id) + .fetch_one(&engine.pool) + .await + .expect("record"); + assert_eq!(record.0, "DONE"); + assert!(record.1.is_some()); + assert_eq!(record.2, Some(BULK_ITEMS)); + close_file_engine(engine, &path).await; + } + + /// A copy that fails marks its record FAILED with the reason before + /// the target is removed; the removal cascades the record. + #[tokio::test] + async fn a_failed_legacy_restore_marks_its_record_failed_before_removing_the_target() { + let engine = engine().await; + indexed_table(&engine, "src").await; + let backup = engine + .create(ACCOUNT, "src", "bkp", JobPermit::detached()) + .await + .expect("backup"); + // Corrupt one backup row so the copy fails part-way. + sqlx::query( + "UPDATE backup_items SET item_data = '{\"pk\": {\"S\": \"x\"}}' \ + WHERE id = (SELECT MAX(id) FROM backup_items WHERE backup_arn = ?)", + ) + .bind(&backup.backup_arn) + .execute(&engine.pool) + .await + .expect("corrupt"); + let (logs_guard, logs) = crate::schema::log_capture::capture(); + let desc = start_restore(&engine, "dst", &backup.backup_arn) + .await + .expect("registration"); + assert!( + !wait_for_copy(&engine, &desc.table_id).await, + "an item without its sort key cannot restore" + ); + drop(logs_guard); + let registrations: i64 = + sqlx::query_scalar("SELECT COUNT(*) FROM table_restores WHERE table_id = ?") + .bind(&desc.table_id) + .fetch_one(&engine.pool) + .await + .expect("registrations"); + assert_eq!(registrations, 0, "the record went with the target"); + let logs = logs.lock().expect("logs").clone(); + assert!( + logs.iter() + .any(|l| l.contains("removing the partial table")), + "the FAILED write matched and the removal followed: {logs:#?}" + ); + assert!( + !logs.iter().any(|l| l.contains("left to its owner")), + "{logs:#?}" + ); + engine + .delete(ACCOUNT, &backup.backup_arn) + .await + .expect("the backup is free"); + } } } diff --git a/crates/storage-sqlite/src/backup_catalog.rs b/crates/storage-sqlite/src/backup_catalog.rs new file mode 100644 index 000000000..6f6db92a0 --- /dev/null +++ b/crates/storage-sqlite/src/backup_catalog.rs @@ -0,0 +1,931 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! The catalog index on SQLite: `backup_index`, the restore job fields of +//! `table_restores`, `backup_maintenance` and `settings.catalog_id`. +//! +//! One process serves a database file (the serve lock enforces it), so a +//! job's claim is an in-process slot ([`JobSlots`]): a key held in a set for +//! the job's life and released by a drop guard, so a task that ends by any +//! route, a panic included, frees it. A restart frees every slot at once, +//! which is the proof of death this backend needs: a CREATING in-catalog +//! backup row whose slot no task holds is dead, and every RUNNING restore +//! job record of the stopped process is aged at startup so the reconciler's +//! first cycle finds it ([`SqliteEngine::expire_restore_job_heartbeats`]). +//! +//! Every write a job makes to its row goes through its claim and carries the +//! claim's `owner_generation`, so a row a successor took over matches no +//! write. The lock on a row for DeleteBackup and a restore's registration is +//! the engine write lock with a `BEGIN IMMEDIATE` transaction, as for every +//! other writer (design decision D1). Times are RFC 3339 text on the +//! database's clock (`strftime('now')`), and staleness is measured there. +//! A conditional write never has an unknown outcome here (one process, one +//! file: a statement either ran or errored), so `WriteOutcome::Unknown` is +//! never returned. + +use std::collections::HashSet; +use std::sync::{Arc, Mutex}; +use std::time::Duration; + +use extenddb_core::backup_format::BackupArn; +use extenddb_storage::JobPermit; +use extenddb_storage::backup_catalog::{ + BackupCatalog, BackupKey, BackupRow, BackupStatus, DeleteStart, Heartbeat, JobClaim, JobRow, + JobStatus, RestoreJobRow, RestoreKind, RestoreTargetKey, StatusFields, StoreIdentity, + StoreKind, TimeBounds, WriteOutcome, new_generation, +}; +use extenddb_storage::error::StorageError; +use futures::future::BoxFuture; +use futures::stream::BoxStream; + +use crate::sqlite_util::{format_epoch_seconds, timestamp_epoch_seconds}; +use crate::store::SqliteEngine; + +/// The database's clock as a catalog timestamp, millisecond precision. +pub(crate) const NOW: &str = "strftime('%Y-%m-%dT%H:%M:%fZ', 'now')"; + +/// The in-process job slots of one engine: the key of every row a job of +/// this process owns. A key is held from before the row exists until the job +/// ends, so a row whose key is absent has no live owner. +#[derive(Debug, Default)] +pub(crate) struct JobSlots(Mutex>); + +impl JobSlots { + /// Take `key`, or `None` when a job of this process holds it. + pub(crate) fn take(slots: &Arc, key: String) -> Option { + let mut held = slots + .0 + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + held.insert(key.clone()).then(|| JobSlot { + slots: Arc::clone(slots), + key, + }) + } + + /// Whether a job of this process holds `key`. + pub(crate) fn holds(&self, key: &str) -> bool { + self.0 + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .contains(key) + } +} + +/// A held slot; dropping it frees the key. +#[derive(Debug)] +pub(crate) struct JobSlot { + slots: Arc, + key: String, +} + +impl Drop for JobSlot { + fn drop(&mut self) { + self.slots + .0 + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .remove(&self.key); + } +} + +/// The slot key of a backup job's claim on a `backup_index` row. +fn backup_slot_key(backup_id: &str) -> String { + format!("backup:{backup_id}") +} + +/// The slot key of a legacy create's hold on its CREATING `backups` row. +pub(crate) fn legacy_backup_slot_key(backup_arn: &str) -> String { + format!("legacy-backup:{backup_arn}") +} + +/// An `owner_generation` as the catalog stores it (`INTEGER`); generations +/// are minted below `i64::MAX`. +pub(crate) fn generation_column(generation: u64) -> i64 { + i64::try_from(generation).unwrap_or(i64::MAX) +} + +/// The row a claim writes through. +#[derive(Debug, Clone)] +pub(crate) enum JobRecord { + /// A `backup_index` row. + Backup(BackupKey), + /// The `table_restores` record of a restore target. + Restore { table_id: String }, +} + +/// A job's handle on its record: the in-process slot, the generation, and +/// the engine the conditional writes go through. Held by a backup job on its +/// `backup_index` row and by a legacy restore on its job record. +pub(crate) struct SqliteJobClaim { + engine: SqliteEngine, + _slot: Option, + _permit: Option, + generation: u64, + record: JobRecord, +} + +impl SqliteJobClaim { + /// A claim on a restore job record, born with the registration that + /// writes `generation` on the row. The permit is the job's slot in the + /// engine's `max_concurrent_jobs`, freed with the claim. + pub(crate) fn for_restore( + engine: &SqliteEngine, + table_id: &str, + generation: u64, + permit: JobPermit, + ) -> Self { + Self { + engine: engine.clone(), + _slot: None, + _permit: Some(permit), + generation, + record: JobRecord::Restore { + table_id: table_id.to_owned(), + }, + } + } + + /// Heartbeat the record every `period` until a heartbeat is not + /// confirmed; returns why. Never returns while the claim is confirmed. + pub(crate) async fn heartbeat_loop(&self, period: Duration) -> StorageError { + let mut tick = tokio::time::interval_at(tokio::time::Instant::now() + period, period); + tick.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); + loop { + tick.tick().await; + match self.heartbeat().await { + Ok(Heartbeat::Confirmed) => {} + Ok(Heartbeat::Lost) => { + return StorageError::Internal( + "the job record no longer carries this claim's generation".to_owned(), + ); + } + Err(e) => return e, + } + } + } +} + +impl JobClaim for SqliteJobClaim { + fn generation(&self) -> u64 { + self.generation + } + + fn heartbeat(&self) -> BoxFuture<'_, Result> { + Box::pin(async move { + let _writer = self.engine.write_lock.lock().await; + let affected = match &self.record { + JobRecord::Backup(key) => { + sqlx::query(&format!( + "UPDATE backup_index SET heartbeat_at = {NOW} \ + WHERE account_id = ? AND backup_id = ? AND owner_generation = ?" + )) + .bind(&key.account_id) + .bind(&key.backup_id) + .bind(generation_column(self.generation)) + .execute(&self.engine.pool) + .await + } + JobRecord::Restore { table_id } => { + sqlx::query(&format!( + "UPDATE table_restores SET heartbeat_at = {NOW} \ + WHERE table_id = ? AND status = 'RUNNING' AND owner_generation = ?" + )) + .bind(table_id) + .bind(generation_column(self.generation)) + .execute(&self.engine.pool) + .await + } + } + .map_err(db_err)? + .rows_affected(); + Ok(if affected == 1 { + Heartbeat::Confirmed + } else { + Heartbeat::Lost + }) + }) + } + + fn set_status<'a>( + &'a self, + from: JobStatus, + to: JobStatus, + fields: &'a StatusFields, + ) -> BoxFuture<'a, Result> { + Box::pin(async move { + let _writer = self.engine.write_lock.lock().await; + let affected = match &self.record { + JobRecord::Backup(key) => { + sqlx::query(&format!( + "UPDATE backup_index SET status = ?, \ + size_bytes = COALESCE(?, size_bytes), \ + item_count = COALESCE(?, item_count), \ + failure_reason = COALESCE(?, failure_reason), \ + completed_at = CASE WHEN ? THEN {NOW} ELSE completed_at END \ + WHERE account_id = ? AND backup_id = ? AND owner_generation = ? \ + AND status = ?" + )) + .bind(to.as_str()) + .bind(fields.size_bytes) + .bind(fields.item_count) + .bind(fields.failure_reason.as_deref()) + .bind(fields.completed) + .bind(&key.account_id) + .bind(&key.backup_id) + .bind(generation_column(self.generation)) + .bind(from.as_str()) + .execute(&self.engine.pool) + .await + } + JobRecord::Restore { table_id } => { + sqlx::query(&format!( + "UPDATE table_restores SET status = ?, \ + items_applied = COALESCE(?, items_applied), \ + failure_reason = COALESCE(?, failure_reason), \ + completed_at = CASE WHEN ? THEN {NOW} ELSE completed_at END \ + WHERE table_id = ? AND owner_generation = ? AND status = ?" + )) + .bind(to.as_str()) + .bind(fields.item_count) + .bind(fields.failure_reason.as_deref()) + .bind(fields.completed) + .bind(table_id) + .bind(generation_column(self.generation)) + .bind(from.as_str()) + .execute(&self.engine.pool) + .await + } + } + .map_err(db_err)? + .rows_affected(); + // One process and one file: a statement either ran or errored, + // so no outcome is ever unknown. + Ok(if affected == 1 { + WriteOutcome::Matched + } else { + WriteOutcome::NotMatched + }) + }) + } + + fn read(&self) -> BoxFuture<'_, Result> { + Box::pin(async move { + let row: Option<(String, i64, String, Option)> = match &self.record { + JobRecord::Backup(key) => { + sqlx::query_as( + "SELECT status, owner_generation, heartbeat_at, failure_reason \ + FROM backup_index WHERE account_id = ? AND backup_id = ?", + ) + .bind(&key.account_id) + .bind(&key.backup_id) + .fetch_optional(&self.engine.pool) + .await + } + JobRecord::Restore { table_id } => { + sqlx::query_as( + "SELECT status, owner_generation, heartbeat_at, failure_reason \ + FROM table_restores WHERE table_id = ?", + ) + .bind(table_id) + .fetch_optional(&self.engine.pool) + .await + } + } + .map_err(db_err)?; + let (status, generation, heartbeat_at, failure_reason) = + row.ok_or_else(|| match &self.record { + JobRecord::Backup(key) => not_found(&key.backup_arn), + JobRecord::Restore { table_id } => StorageError::Internal(format!( + "restore job record of table {table_id} is gone" + )), + })?; + Ok(JobRow { + status: JobStatus::parse(&status)?, + owner_generation: u64::try_from(generation).unwrap_or_default(), + heartbeat_at: timestamp_epoch_seconds(&heartbeat_at), + failure_reason, + }) + }) + } +} + +/// The name of a CREATING restore target registered from `backup_arn`, if +/// any: the one in-use rule DeleteBackup, a restore's registration and +/// `backup_in_use` share. Reads the target's status, not the job's. +pub(crate) async fn creating_restore_target<'e, E: sqlx::SqliteExecutor<'e>>( + executor: E, + backup_arn: &str, +) -> Result, StorageError> { + sqlx::query_scalar( + "SELECT t.table_name FROM table_restores r JOIN tables t ON t.table_id = r.table_id \ + WHERE r.source_backup_arn = ? AND t.table_status = 'CREATING' LIMIT 1", + ) + .bind(backup_arn) + .fetch_optional(executor) + .await + .map_err(db_err) +} + +/// The error for a backup a CREATING restore target still reads: the shared +/// text, [`extenddb_storage::backup_catalog::backup_in_use`]. +pub(crate) fn backup_in_use(table: &str, backup_arn: &str) -> StorageError { + extenddb_storage::backup_catalog::backup_in_use(table, backup_arn) +} + +fn db_err(e: sqlx::Error) -> StorageError { + StorageError::Internal(format!("Database error: {e}")) +} + +fn not_found(backup_arn: &str) -> StorageError { + extenddb_storage::backup_catalog::backup_not_found(backup_arn) +} + +/// The columns of one `backup_index` row, in the order every read selects +/// them. +const ROW_COLUMNS: &str = "backup_id, account_id, table_name, backup_arn, backup_name, status, \ + store_kind, store_root, store_id, size_bytes, item_count, created_at, completed_at, \ + failure_reason, source_table_details_json, owner_generation, heartbeat_at"; + +/// One `backup_index` row as the driver reads it. +#[derive(sqlx::FromRow)] +struct RowTuple { + backup_id: String, + account_id: String, + table_name: String, + backup_arn: String, + backup_name: String, + status: String, + store_kind: String, + store_root: String, + store_id: String, + size_bytes: i64, + item_count: i64, + created_at: String, + completed_at: Option, + failure_reason: Option, + source_table_details_json: String, + owner_generation: i64, + heartbeat_at: String, +} + +fn row_from_tuple(t: RowTuple) -> Result { + let RowTuple { + backup_id, + account_id, + table_name, + backup_arn, + backup_name, + status, + store_kind, + store_root, + store_id, + size_bytes, + item_count, + created_at, + completed_at, + failure_reason, + source_table_details_json, + owner_generation, + heartbeat_at, + } = t; + let store_kind = StoreKind::parse(&store_kind).ok_or_else(|| { + StorageError::Internal(format!( + "backup {backup_arn} carries an unknown store kind {store_kind:?}" + )) + })?; + let source_table_details_json = + serde_json::from_str(&source_table_details_json).map_err(|e| { + StorageError::Internal(format!( + "backup {backup_arn} carries a source table that is not JSON: {e}" + )) + })?; + Ok(BackupRow { + backup_id, + account_id, + table_name, + backup_arn, + backup_name, + status: BackupStatus::parse(&status)?, + store: StoreIdentity { + store_kind, + store_root, + store_id, + }, + size_bytes, + item_count, + created_at: timestamp_epoch_seconds(&created_at), + completed_at: completed_at.as_deref().map(timestamp_epoch_seconds), + failure_reason, + source_table_details_json, + owner_generation: u64::try_from(owner_generation).unwrap_or_default(), + heartbeat_at: timestamp_epoch_seconds(&heartbeat_at), + }) +} + +/// The rows `read` fetches, as a stream: the read runs once, on the first +/// poll, and a failed read is one `Err` item. +fn rows_stream<'a, F>(read: F) -> BoxStream<'a, Result> +where + F: std::future::Future, StorageError>> + Send + 'a, +{ + use futures::StreamExt; + Box::pin( + futures::stream::once(async move { + match read.await { + Ok(rows) => rows.into_iter().map(row_from_tuple).collect::>(), + Err(e) => vec![Err(e)], + } + }) + .flat_map(futures::stream::iter), + ) +} + +impl SqliteEngine { + /// Every RUNNING restore job record at startup was owned by the process + /// that stopped: one process serves a file, so a restart is proof of + /// death. Set each record's heartbeat to the epoch, so the reconciler's + /// first cycle fails it and removes its target without waiting the stale + /// window. Returns the names of the targets whose record was aged (one + /// already at the epoch is not reported again). Runs before the server + /// takes requests, so no restore of this process can be in flight. + pub(crate) async fn expire_restore_job_heartbeats(&self) -> Result, StorageError> { + let _writer = self.write_lock.lock().await; + let mut tx = self + .pool + .begin_with("BEGIN IMMEDIATE") + .await + .map_err(db_err)?; + let names: Vec = sqlx::query_scalar( + "UPDATE table_restores SET heartbeat_at = '1970-01-01T00:00:00.000Z' \ + WHERE status = 'RUNNING' AND heartbeat_at > '1970-01-01T00:00:00.000Z' \ + RETURNING COALESCE((SELECT table_name FROM tables t \ + WHERE t.table_id = table_restores.table_id), table_id)", + ) + .fetch_all(&mut *tx) + .await + .map_err(db_err)?; + tx.commit().await.map_err(db_err)?; + Ok(names) + } +} + +impl BackupCatalog for SqliteEngine { + fn catalog_id(&self) -> BoxFuture<'_, Result> { + Box::pin(async move { + let id: Option = + sqlx::query_scalar("SELECT value FROM settings WHERE key = 'catalog_id'") + .fetch_optional(&self.pool) + .await + .map_err(db_err)?; + id.ok_or_else(|| { + StorageError::Internal( + "the catalog has no catalog_id; run extenddb migrate --yes".to_owned(), + ) + }) + }) + } + + fn claim_backup<'a>( + &'a self, + row: &'a BackupRow, + ) -> BoxFuture<'a, Result, StorageError>> { + Box::pin(async move { + // The slot first, so no row ever exists unowned. + let Some(slot) = JobSlots::take(&self.job_slots, backup_slot_key(&row.backup_id)) + else { + return Err(StorageError::AlreadyExists(format!( + "a live job owns backup {}", + row.backup_arn + ))); + }; + let generation = new_generation(); + let source_table = serde_json::to_string(&row.source_table_details_json) + .map_err(|e| StorageError::Internal(format!("source table details: {e}")))?; + let _writer = self.write_lock.lock().await; + let mut tx = self + .pool + .begin_with("BEGIN IMMEDIATE") + .await + .map_err(db_err)?; + let inserted = sqlx::query(&format!( + "INSERT OR IGNORE INTO backup_index (backup_id, account_id, table_name, \ + backup_arn, backup_name, status, store_kind, store_root, store_id, size_bytes, \ + item_count, created_at, source_table_details_json, owner_generation, \ + heartbeat_at) \ + VALUES (?, ?, ?, ?, ?, 'CREATING', ?, ?, ?, ?, ?, ?, ?, ?, {NOW})" + )) + .bind(&row.backup_id) + .bind(&row.account_id) + .bind(&row.table_name) + .bind(&row.backup_arn) + .bind(&row.backup_name) + .bind(row.store.store_kind.as_str()) + .bind(&row.store.store_root) + .bind(&row.store.store_id) + .bind(row.size_bytes) + .bind(row.item_count) + .bind(format_epoch_seconds(row.created_at)) + .bind(&source_table) + .bind(generation_column(generation)) + .execute(&mut *tx) + .await + .map_err(db_err)? + .rows_affected(); + tx.commit().await.map_err(db_err)?; + if inserted == 0 { + drop(slot); + return Err(StorageError::AlreadyExists(format!( + "backup {} already has a row", + row.backup_arn + ))); + } + Ok(Box::new(SqliteJobClaim { + engine: self.clone(), + _slot: Some(slot), + _permit: None, + generation, + record: JobRecord::Backup(row.key()), + }) as Box) + }) + } + + fn list_backup_rows<'a>( + &'a self, + status: Option, + store: &'a StoreIdentity, + ) -> BoxStream<'a, Result> { + rows_stream(async move { + sqlx::query_as(&format!( + "SELECT {ROW_COLUMNS} FROM backup_index \ + WHERE store_kind = ?1 AND store_root = ?2 AND store_id = ?3 \ + AND (?4 IS NULL OR status = ?4) ORDER BY created_at, backup_arn" + )) + .bind(store.store_kind.as_str()) + .bind(&store.store_root) + .bind(&store.store_id) + .bind(status.map(BackupStatus::as_str)) + .fetch_all(&self.pool) + .await + .map_err(db_err) + }) + } + + fn list_account_backup_rows<'a>( + &'a self, + account_id: &'a str, + table_name: Option<&'a str>, + bounds: &'a TimeBounds, + ) -> BoxStream<'a, Result> { + rows_stream(async move { + // `created_at` is written by `format_epoch_seconds`, fixed width, + // so the bounds compare as text in time order. + sqlx::query_as(&format!( + "SELECT {ROW_COLUMNS} FROM backup_index WHERE account_id = ?1 \ + AND (?2 IS NULL OR table_name = ?2) \ + AND (?3 IS NULL OR created_at >= ?3) \ + AND (?4 IS NULL OR created_at < ?4) \ + ORDER BY created_at, backup_arn" + )) + .bind(account_id) + .bind(table_name) + .bind(bounds.lower.map(format_epoch_seconds)) + .bind(bounds.upper.map(format_epoch_seconds)) + .fetch_all(&self.pool) + .await + .map_err(db_err) + }) + } + + fn read_backup_row<'a>( + &'a self, + arn: &'a BackupArn, + ) -> BoxFuture<'a, Result, StorageError>> { + Box::pin(async move { + let row: Option = sqlx::query_as(&format!( + "SELECT {ROW_COLUMNS} FROM backup_index WHERE account_id = ? AND backup_id = ? \ + AND table_name = ? AND backup_arn = ?" + )) + .bind(arn.account_id()) + .bind(arn.backup_id()) + .bind(arn.table_name()) + .bind(arn.to_arn_string()) + .fetch_optional(&self.pool) + .await + .map_err(db_err)?; + row.map(row_from_tuple).transpose() + }) + } + + fn start_backup_delete<'a>( + &'a self, + key: &'a BackupKey, + ) -> BoxFuture<'a, Result> { + Box::pin(async move { + // Under the write lock, in one write transaction: a restore's + // registration runs its in-use check the same way, so the two + // order wholly one before the other. + let _writer = self.write_lock.lock().await; + let mut tx = self + .pool + .begin_with("BEGIN IMMEDIATE") + .await + .map_err(db_err)?; + let status: Option = sqlx::query_scalar( + "SELECT status FROM backup_index WHERE account_id = ? AND backup_id = ?", + ) + .bind(&key.account_id) + .bind(&key.backup_id) + .fetch_optional(&mut *tx) + .await + .map_err(db_err)?; + let status = BackupStatus::parse(&status.ok_or_else(|| not_found(&key.backup_arn))?)?; + if status != BackupStatus::Available { + return Ok(DeleteStart::Status(status)); + } + if let Some(table) = creating_restore_target(&mut *tx, &key.backup_arn).await? { + return Ok(DeleteStart::InUse(table)); + } + sqlx::query( + "UPDATE backup_index SET status = 'DELETING' \ + WHERE account_id = ? AND backup_id = ? AND status = 'AVAILABLE'", + ) + .bind(&key.account_id) + .bind(&key.backup_id) + .execute(&mut *tx) + .await + .map_err(db_err)?; + tx.commit().await.map_err(db_err)?; + Ok(DeleteStart::Deleting) + }) + } + + fn set_backup_deleted<'a>( + &'a self, + key: &'a BackupKey, + ) -> BoxFuture<'a, Result> { + Box::pin(async move { + let _writer = self.write_lock.lock().await; + let affected = sqlx::query(&format!( + "UPDATE backup_index SET status = 'DELETED', completed_at = {NOW} \ + WHERE account_id = ? AND backup_id = ? AND status = 'DELETING'" + )) + .bind(&key.account_id) + .bind(&key.backup_id) + .execute(&self.pool) + .await + .map_err(db_err)? + .rows_affected(); + Ok(affected == 1) + }) + } + + fn remove_backup_row<'a>( + &'a self, + key: &'a BackupKey, + ) -> BoxFuture<'a, Result> { + Box::pin(async move { + let _writer = self.write_lock.lock().await; + let affected = sqlx::query( + "DELETE FROM backup_index WHERE account_id = ? AND backup_id = ? \ + AND status IN ('DELETED', 'FAILED')", + ) + .bind(&key.account_id) + .bind(&key.backup_id) + .execute(&self.pool) + .await + .map_err(db_err)? + .rows_affected(); + Ok(affected == 1) + }) + } + + fn backup_in_use<'a>( + &'a self, + key: &'a BackupKey, + ) -> BoxFuture<'a, Result, StorageError>> { + Box::pin(async move { creating_restore_target(&self.pool, &key.backup_arn).await }) + } + + fn remove_restore_target<'a>( + &'a self, + key: &'a RestoreTargetKey, + ) -> BoxFuture<'a, Result> { + Box::pin(async move { + // The key must name a CREATING restore target with this table id; + // a table re-created under the name has another id and is not + // touched. `abort_restore` then claims it from CREATING in one + // statement under the write lock, so it and the ACTIVE flip + // exclude each other. + let is_target: bool = sqlx::query_scalar( + "SELECT EXISTS(SELECT 1 FROM tables t JOIN table_restores r \ + ON r.table_id = t.table_id WHERE t.table_id = ? AND t.account_id = ? \ + AND t.table_name = ? AND t.table_status = 'CREATING')", + ) + .bind(&key.table_id) + .bind(&key.account_id) + .bind(&key.table_name) + .fetch_one(&self.pool) + .await + .map_err(db_err)?; + if !is_target { + return Ok(false); + } + // Progress rows belong to a later release's restore path; a + // catalog without that table has none to delete. + let has_progress: bool = sqlx::query_scalar( + "SELECT EXISTS(SELECT 1 FROM sqlite_master \ + WHERE type = 'table' AND name = 'restore_progress')", + ) + .fetch_one(&self.pool) + .await + .map_err(db_err)?; + if has_progress { + let _writer = self.write_lock.lock().await; + sqlx::query("DELETE FROM restore_progress WHERE table_id = ?") + .bind(&key.table_id) + .execute(&self.pool) + .await + .map_err(db_err)?; + } + self.abort_restore(&key.table_id).await + }) + } + + fn list_dead_restore_jobs( + &self, + stale_after: Duration, + ) -> BoxFuture<'_, Result, StorageError>> { + Box::pin(async move { + // Two arms, as on PostgreSQL: a RUNNING record whose heartbeat is + // stale (a dead owner), and a FAILED record whose table is still + // a CREATING restore target once the record is stale (the FAILED + // write landed and the removal did not; `fail_restore_job` and + // the copy's own failure path stamp `completed_at`, and a record + // without one is aged by its heartbeat). The delay is what leaves + // a task still on its own failure path to finish it. Both ages + // are measured on the database's clock (`julianday('now')`). + #[allow(clippy::type_complexity)] + let rows: Vec<( + String, + String, + String, + String, + String, + String, + i64, + String, + )> = sqlx::query_as( + "SELECT r.table_id, t.account_id, t.table_name, r.source_backup_arn, \ + r.kind, r.status, r.owner_generation, r.heartbeat_at \ + FROM table_restores r JOIN tables t ON t.table_id = r.table_id \ + WHERE (r.status = 'RUNNING' \ + AND julianday(r.heartbeat_at) < julianday('now') - ?1 / 86400.0) \ + OR (r.status = 'FAILED' AND t.table_status = 'CREATING' \ + AND julianday(COALESCE(r.completed_at, r.heartbeat_at)) \ + < julianday('now') - ?1 / 86400.0) \ + ORDER BY t.account_id, t.table_name", + ) + .bind(stale_after.as_secs_f64()) + .fetch_all(&self.pool) + .await + .map_err(db_err)?; + rows.into_iter() + .map( + |( + table_id, + account_id, + table_name, + source_backup_arn, + kind, + status, + owner_generation, + heartbeat_at, + )| { + Ok(RestoreJobRow { + target: RestoreTargetKey { + account_id, + table_name, + table_id, + }, + source_backup_arn, + kind: RestoreKind::parse(&kind)?, + status: JobStatus::parse(&status)?, + owner_generation: u64::try_from(owner_generation).unwrap_or_default(), + heartbeat_at: timestamp_epoch_seconds(&heartbeat_at), + }) + }, + ) + .collect() + }) + } + + fn fail_restore_job<'a>( + &'a self, + key: &'a RestoreTargetKey, + reason: &'a str, + ) -> BoxFuture<'a, Result> { + Box::pin(async move { + let _writer = self.write_lock.lock().await; + let affected = sqlx::query(&format!( + "UPDATE table_restores SET status = 'FAILED', failure_reason = ?, \ + completed_at = {NOW} \ + WHERE table_id = ? AND status = 'RUNNING' AND EXISTS(\ + SELECT 1 FROM tables t WHERE t.table_id = table_restores.table_id \ + AND t.account_id = ? AND t.table_name = ?)" + )) + .bind(reason) + .bind(&key.table_id) + .bind(&key.account_id) + .bind(&key.table_name) + .execute(&self.pool) + .await + .map_err(db_err)? + .rows_affected(); + Ok(affected == 1) + }) + } + + fn check_maintenance_claim(&self) -> BoxFuture<'_, Result> { + Box::pin(async move { + sqlx::query_scalar("SELECT EXISTS(SELECT 1 FROM backup_maintenance)") + .fetch_one(&self.pool) + .await + .map_err(db_err) + }) + } +} + +/// The `backup_catalog` module's unit tests that need no database. +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_row_tuple_with_an_unknown_store_kind_status_or_source_table_is_internal() { + let good = || RowTuple { + backup_id: "01757500000123-0badcafe".to_owned(), + account_id: "123456789012".to_owned(), + table_name: "Music".to_owned(), + backup_arn: "arn".to_owned(), + backup_name: "b".to_owned(), + status: "AVAILABLE".to_owned(), + store_kind: "s3".to_owned(), + store_root: "s3://bucket/prefix".to_owned(), + store_id: "store-1".to_owned(), + size_bytes: 1, + item_count: 2, + created_at: "2025-09-10T10:26:40.123000Z".to_owned(), + completed_at: None, + failure_reason: None, + source_table_details_json: "{}".to_owned(), + owner_generation: 7, + heartbeat_at: "2025-09-10T10:26:41.000Z".to_owned(), + }; + let row = row_from_tuple(good()).unwrap(); + assert_eq!(row.store.store_kind, StoreKind::S3); + assert_eq!(row.owner_generation, 7); + assert!((row.created_at - 1_757_500_000.123).abs() < 1e-6); + assert_eq!(row.heartbeat_at, 1_757_500_001.0); + let mut bad_kind = good(); + bad_kind.store_kind = "gcs".to_owned(); + assert!(matches!( + row_from_tuple(bad_kind), + Err(StorageError::Internal(_)) + )); + let mut bad_status = good(); + bad_status.status = "ACTIVE".to_owned(); + assert!(matches!( + row_from_tuple(bad_status), + Err(StorageError::Internal(_)) + )); + let mut bad_source = good(); + bad_source.source_table_details_json = "not json".to_owned(); + assert!(matches!( + row_from_tuple(bad_source), + Err(StorageError::Internal(_)) + )); + } + + #[test] + fn a_slot_is_held_once_and_freed_by_its_guard() { + let slots = Arc::new(JobSlots::default()); + let held = JobSlots::take(&slots, "backup:1".to_owned()).expect("free"); + assert!(slots.holds("backup:1")); + assert!(JobSlots::take(&slots, "backup:1".to_owned()).is_none()); + assert!(JobSlots::take(&slots, "backup:2".to_owned()).is_some()); + drop(held); + assert!(!slots.holds("backup:1")); + assert!(JobSlots::take(&slots, "backup:1".to_owned()).is_some()); + } + + #[test] + fn a_generation_fits_the_integer_column() { + assert_eq!(generation_column(7), 7); + assert_eq!(generation_column(u64::MAX), i64::MAX); + assert_ne!( + generation_column(new_generation()), + generation_column(extenddb_storage::backup_catalog::UNOWNED_GENERATION) + ); + } +} diff --git a/crates/storage-sqlite/src/config.rs b/crates/storage-sqlite/src/config.rs index 01f69c0e2..6ba092de0 100644 --- a/crates/storage-sqlite/src/config.rs +++ b/crates/storage-sqlite/src/config.rs @@ -21,14 +21,14 @@ use serde::Deserialize; /// of 10, and 1 for every `pool_size` of 5 or less; this is the one statement /// of the formula, `backup::backup_server_slot_count` computes it), shared by /// backups and table snapshots: each running backup and each open snapshot -/// stream (file-backed or in-memory) holds one. A snapshot that finds no slot -/// free is refused at once; a `CreateBackup` waits 30 s for one and then -/// answers `LimitExceededException`, with no backup running if snapshots hold -/// every slot. With one slot, one held snapshot blocks every `CreateBackup` -/// for as long as it is open. Running backups hold one connection each for -/// their read snapshot plus one shared writer (`n` backups hold `n + 1`), so -/// from `pool_size` 4 up a flood of backups leaves at least half the pool to -/// other requests; with 2 or 3 the one admitted backup holds two of them. +/// stream (file-backed or in-memory) holds one. A snapshot or a `CreateBackup` +/// that finds no slot free is refused at once with `LimitExceededException`, +/// so no backup runs while snapshots hold every slot. With one slot, one held +/// snapshot refuses every `CreateBackup` for as long as it is open. Running +/// backups hold one connection each for their read snapshot plus one shared +/// writer (`n` backups hold `n + 1`), so from `pool_size` 4 up a flood of +/// backups leaves at least half the pool to other requests; with 2 or 3 the +/// one admitted backup holds two of them. #[derive(Debug, Clone, Deserialize)] #[serde(deny_unknown_fields)] pub struct SqliteConfig { diff --git a/crates/storage-sqlite/src/create_table.rs b/crates/storage-sqlite/src/create_table.rs index ce068463a..509e73086 100644 --- a/crates/storage-sqlite/src/create_table.rs +++ b/crates/storage-sqlite/src/create_table.rs @@ -22,12 +22,17 @@ use extenddb_storage::util::{index_arn, stream_arn, table_arn}; use crate::sqlite_util::{format_epoch_seconds, format_timestamp, is_unique_violation}; use crate::store::SqliteEngine; -/// What a restore records about itself when its target is created. +/// What a restore records about itself when its target is created: the +/// registration and the restore job record it is born with (`RUNNING`, +/// `kind = legacy`, the claim's generation, a fresh heartbeat). pub(crate) struct RestoreRegistration<'a> { pub backup_arn: &'a str, pub source_table_arn: &'a str, /// Epoch seconds: the source backup's creation time. pub restore_date_time: f64, + /// The fencing token of the restore's claim; every later write by the + /// restore to its record carries it. + pub owner_generation: u64, } impl SqliteEngine { @@ -189,18 +194,10 @@ impl SqliteEngine { // predicate as DeleteBackup's, in the same kind of write // transaction under the write lock, so two simultaneous restores // cannot both pass. - let restoring: Option = sqlx::query_scalar( - "SELECT t.table_name FROM table_restores r JOIN tables t ON t.table_id = r.table_id \ - WHERE r.source_backup_arn = ? AND t.table_status = 'CREATING' LIMIT 1", - ) - .bind(backup_arn) - .fetch_optional(&mut *tx) - .await - .map_err(|e| StorageError::Internal(e.to_string()))?; - if let Some(table) = restoring { - return Err(StorageError::BackupInUse(format!( - "Backup is being used to restore table {table}: {backup_arn}" - ))); + if let Some(table) = + crate::backup_catalog::creating_restore_target(&mut *tx, backup_arn).await? + { + return Err(crate::backup_catalog::backup_in_use(&table, backup_arn)); } } @@ -262,15 +259,20 @@ impl SqliteEngine { } if let Some(registration) = restore { - sqlx::query( + sqlx::query(&format!( "INSERT INTO table_restores \ - (table_id, source_backup_arn, source_table_arn, restore_date_time) \ - VALUES (?, ?, ?, ?)", - ) + (table_id, source_backup_arn, source_table_arn, restore_date_time, \ + status, kind, owner_generation, heartbeat_at) \ + VALUES (?, ?, ?, ?, 'RUNNING', 'legacy', ?, {})", + crate::backup_catalog::NOW + )) .bind(&table_id) .bind(registration.backup_arn) .bind(registration.source_table_arn) .bind(format_epoch_seconds(registration.restore_date_time)) + .bind(crate::backup_catalog::generation_column( + registration.owner_generation, + )) .execute(&mut *tx) .await .map_err(|e| StorageError::Internal(e.to_string()))?; diff --git a/crates/storage-sqlite/src/lib.rs b/crates/storage-sqlite/src/lib.rs index 7e9b278cb..1e3c9a94e 100644 --- a/crates/storage-sqlite/src/lib.rs +++ b/crates/storage-sqlite/src/lib.rs @@ -23,6 +23,7 @@ mod admin_store; mod authorization_store; mod backup; +mod backup_catalog; mod bootstrapper; mod catalog_store; pub mod config; @@ -286,31 +287,24 @@ fn sqlite_server_components_factory( Err(e) => tracing::error!("Failed to recover control plane transitions: {e}"), } - // Remove any restore target a prior crash left mid-copy. Before the - // server takes requests, so no restore can be in flight. - match engine.sweep_abandoned_restores().await { + // A restore the previous run left mid-copy: one process serves this + // file, so its owner is gone. Age every RUNNING job record so the + // engine's reconciler fails it and removes its target on its first + // cycle rather than after the stale window. Before the server takes + // requests, so no restore of this process can be in flight. A backup + // the previous run left CREATING is reported to the reconciler by + // `abandoned_creating_backups` without a step here: no task of this + // process holds its slot. + match engine.expire_restore_job_heartbeats().await { Ok(names) => { for name in &names { tracing::warn!( - "removed table '{name}': its restore did not finish before the last \ - shutdown" + "table '{name}': its restore did not finish before the last shutdown; \ + the reconciler removes it" ); } } - Err(e) => tracing::error!("Failed to sweep abandoned restores: {e}"), - } - - // Likewise a backup a prior crash left CREATING: its items were only - // partly copied, and nothing will finish them. - match engine.sweep_incomplete_backups().await { - Ok(arns) => { - for arn in &arns { - tracing::warn!( - "removed backup {arn}: its copy did not finish before the last shutdown" - ); - } - } - Err(e) => tracing::error!("Failed to sweep incomplete backups: {e}"), + Err(e) => tracing::error!("Failed to mark interrupted restores: {e}"), } // Rebuild any GSI left mid-backfill (status CREATING) by a prior crash. diff --git a/crates/storage-sqlite/src/migrations/golden_tests.rs b/crates/storage-sqlite/src/migrations/golden_tests.rs index d8b1e3a78..6a3e4dc44 100644 --- a/crates/storage-sqlite/src/migrations/golden_tests.rs +++ b/crates/storage-sqlite/src/migrations/golden_tests.rs @@ -1,7 +1,7 @@ // Copyright 2026 ExtendDB contributors // SPDX-License-Identifier: Apache-2.0 -//! Golden tests for SQLite migrations 2 to 4, each run on a released catalog +//! Golden tests for SQLite migrations 2 to 5, each run on a released catalog //! with the damage or the content its body handles, and migration 1's //! accepted and converged pre-release shapes. @@ -10,8 +10,8 @@ use std::path::Path; use extenddb_storage::migrations::{CatalogStatus, MigrationError, exit}; use super::tests::{ - DEV_0_1_7, DEV_0_1_14, copies, count, exec, load_dump, plain_pool, run, scalar, status, - temp_db, version_row, + DEV_0_1_7, DEV_0_1_14, copies, count, exec, load_dump, plain_pool, reconverge, run, scalar, + status, temp_db, version_row, }; /// Rows of a query, one tab-separated line each: text as it is (JSON with @@ -549,3 +549,106 @@ async fn migration_4_adds_the_base_key_index_to_index_tables_created_before_v0_1 text.push_str(&dump(&db, "SELECT notes FROM extenddb_migrations WHERE id = 4").await); golden("0004_index_table_base_key", &text); } + +/// Migration 5 on a released catalog holding two restore registrations: one +/// whose table is ACTIVE (a finished restore) and one whose table is CREATING +/// (a restore the stop interrupted). The job record is added and backfilled, +/// the interrupted one RUNNING with the epoch heartbeat and generation 0, the +/// index and maintenance tables exist, the catalog id is written once, and a +/// second run changes nothing. +#[tokio::test] +async fn migration_5_adds_the_catalog_index_and_backfills_the_registrations() { + let db = temp_db("m5"); + load_dump(&db, DEV_0_1_14).await; + // The dump predates `table_restores`; create it as that release did. + exec( + &db, + "CREATE TABLE table_restores (\ + table_id TEXT PRIMARY KEY REFERENCES tables(table_id) ON DELETE CASCADE, \ + source_backup_arn TEXT NOT NULL, \ + source_table_arn TEXT, \ + restore_date_time TEXT NOT NULL); \ + CREATE TEMP TABLE interrupted AS SELECT * FROM tables; \ + UPDATE interrupted SET table_id = '4e1c0d2b-7a9f-4c3e-8b6d-2f5a1c9e7d30', \ + table_name = 'interrupted', table_status = 'CREATING', status_transition_at = NULL; \ + INSERT INTO tables SELECT * FROM interrupted; \ + INSERT INTO table_restores (table_id, source_backup_arn, source_table_arn, restore_date_time) \ + SELECT table_id, 'arn:aws:dynamodb:us-east-1:000000000000:table/cu_orders/backup/01700000000000-0000000a', \ + 'arn:aws:dynamodb:us-east-1:000000000000:table/cu_orders', '2026-01-01T00:00:00.000Z' \ + FROM tables WHERE table_name IN ('cu_orders', 'interrupted');", + ) + .await; + run(&db).await.expect("migrate"); + let catalog_id = scalar(&db, "SELECT value FROM settings WHERE key = 'catalog_id'") + .await + .expect("a catalog id"); + assert!(uuid::Uuid::parse_str(&catalog_id).is_ok(), "{catalog_id}"); + assert_eq!(count(&db, "SELECT COUNT(*) FROM backup_index").await, 0); + assert_eq!( + count(&db, "SELECT COUNT(*) FROM backup_maintenance").await, + 0 + ); + let notes = scalar(&db, "SELECT notes FROM extenddb_migrations WHERE id = 5") + .await + .expect("row"); + for expected in [ + "added table_restores.status", + "added table_restores.completed_at", + "backfilled 2 restore registration(s) as legacy restores, 1 of them RUNNING", + "wrote settings.catalog_id", + ] { + assert!(notes.contains(expected), "{expected} not in {notes}"); + } + let mut text = dump( + &db, + "SELECT t.table_name, t.table_status, r.status, r.kind, r.owner_generation, \ + r.heartbeat_at, r.store_kind, r.completed_at = t.creation_date_time \ + FROM table_restores r JOIN tables t ON t.table_id = r.table_id ORDER BY t.table_name", + ) + .await; + text.push_str( + &dump( + &db, + "SELECT name FROM pragma_table_info('table_restores') ORDER BY cid", + ) + .await, + ); + text.push_str( + &dump( + &db, + "SELECT name, type FROM sqlite_master WHERE name LIKE 'backup_%' OR name LIKE \ + 'idx_backup_%' OR name = 'idx_table_restores_status' ORDER BY name", + ) + .await, + ); + text.push_str(&dump(&db, "SELECT notes FROM extenddb_migrations WHERE id = 5").await); + golden("0005_backup_catalog_index", &text); + + // A second run (through reconverge, which runs every convergent body + // again) adds nothing and keeps the catalog id. + reconverge(&db).await.expect("reconverge"); + assert_eq!( + scalar(&db, "SELECT value FROM settings WHERE key = 'catalog_id'") + .await + .as_deref(), + Some(catalog_id.as_str()) + ); + assert_eq!( + count( + &db, + "SELECT COUNT(*) FROM pragma_table_info('table_restores')" + ) + .await, + 16 + ); + assert_eq!( + scalar( + &db, + "SELECT status FROM table_restores r JOIN tables t ON t.table_id = r.table_id \ + WHERE t.table_name = 'interrupted'" + ) + .await + .as_deref(), + Some("RUNNING") + ); +} diff --git a/crates/storage-sqlite/src/migrations/m0005_backup_catalog_index.rs b/crates/storage-sqlite/src/migrations/m0005_backup_catalog_index.rs new file mode 100644 index 000000000..7f0b08de7 --- /dev/null +++ b/crates/storage-sqlite/src/migrations/m0005_backup_catalog_index.rs @@ -0,0 +1,185 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! Migration 5, `backup_catalog_index`, SQLite: the catalog index of backups +//! kept in a backup store (`backup_index`), the restore job fields on the +//! restore registrations (`table_restores`), the maintenance claim record +//! (`backup_maintenance`) and the catalog id (`settings.catalog_id`). +//! +//! SQLite has no `ADD COLUMN IF NOT EXISTS`, so the body reads the columns of +//! `table_restores` first and adds each one only when it is missing; the +//! tables and indexes are created `IF NOT EXISTS`; the backfill touches only +//! registrations whose `status` is still NULL; and the catalog id is written +//! only when the row is absent. A second run changes nothing, which makes +//! the migration convergent. +//! +//! The backfill: every registration that exists was written by a release +//! without the job record, so each is a legacy restore. One whose table is +//! not CREATING finished (DONE). One whose table is CREATING is a restore the +//! upgrade's stop interrupted: RUNNING, with a heartbeat at the epoch (not +//! NULL, so the staleness test selects it) and an owner generation no job +//! holds (0), so the first reconciler pass fails it and removes its target. +//! +//! This file's checksum is pinned once the migration is released. It is +//! excluded from rustfmt and clippy so neither can require an edit, and it +//! keeps its own column check instead of calling the crate's helpers, which +//! may change. + +use async_trait::async_trait; +use extenddb_storage::migrations::{CodeBody, Done, MigrationError}; +use sqlx::SqlitePool; + +use super::SqliteStores; + +/// This file, whose SHA-256 the checksum test pins. +pub(crate) const SOURCE: &str = include_str!("m0005_backup_catalog_index.rs"); + +/// Recorded as this migration's checksum. Changes only with a new migration. +const FINGERPRINT: &str = "0005_backup_catalog_index/sqlite/1"; + +/// The columns the job record adds to `table_restores`, with their SQLite +/// declarations. None carries NOT NULL: `ADD COLUMN` on a populated table +/// cannot, and the application writes every one of them. +const RESTORE_JOB_COLUMNS: [(&str, &str); 12] = [ + ("status", "TEXT"), + ("kind", "TEXT"), + ("store_kind", "TEXT"), + ("store_root", "TEXT"), + ("store_id", "TEXT"), + ("owner_generation", "INTEGER"), + ("heartbeat_at", "TEXT"), + ("spool_host", "TEXT"), + ("file_index", "INTEGER"), + ("items_applied", "INTEGER"), + ("failure_reason", "TEXT"), + ("completed_at", "TEXT"), +]; + +/// The statements that need no column check. +const GUARDED_SQL: &str = "\ +CREATE TABLE IF NOT EXISTS backup_index ( + account_id TEXT NOT NULL, + backup_id TEXT NOT NULL, + table_name TEXT NOT NULL, + backup_arn TEXT NOT NULL, + backup_name TEXT NOT NULL, + status TEXT NOT NULL, + store_kind TEXT NOT NULL, + store_root TEXT NOT NULL, + store_id TEXT NOT NULL, + size_bytes INTEGER NOT NULL DEFAULT 0, + item_count INTEGER NOT NULL DEFAULT 0, + created_at TEXT NOT NULL, + completed_at TEXT, + failure_reason TEXT, + source_table_details_json TEXT NOT NULL, + owner_generation INTEGER NOT NULL, + heartbeat_at TEXT NOT NULL, + PRIMARY KEY (account_id, backup_id) +); +CREATE UNIQUE INDEX IF NOT EXISTS idx_backup_index_backup_id ON backup_index (backup_id); +CREATE INDEX IF NOT EXISTS idx_backup_index_account_table ON backup_index (account_id, table_name); +CREATE INDEX IF NOT EXISTS idx_backup_index_store_status + ON backup_index (store_kind, store_root, store_id, status); +CREATE TABLE IF NOT EXISTS backup_maintenance ( + singleton INTEGER PRIMARY KEY CHECK (singleton = 1), + holder TEXT NOT NULL, + owner_generation INTEGER NOT NULL, + claimed_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')), + heartbeat_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')) +); +"; + +/// The backfill of the registrations that exist, scoped to rows the job +/// record is still empty on. +const BACKFILL_SQL: &str = "\ +UPDATE table_restores SET + kind = 'legacy', + owner_generation = 0, + heartbeat_at = '1970-01-01T00:00:00.000Z', + status = CASE WHEN (SELECT table_status FROM tables t WHERE t.table_id = table_restores.table_id) = 'CREATING' + THEN 'RUNNING' ELSE 'DONE' END, + completed_at = CASE WHEN (SELECT table_status FROM tables t WHERE t.table_id = table_restores.table_id) = 'CREATING' + THEN NULL + ELSE (SELECT creation_date_time FROM tables t WHERE t.table_id = table_restores.table_id) END +WHERE status IS NULL; +CREATE INDEX IF NOT EXISTS idx_table_restores_status ON table_restores (status, heartbeat_at); +"; + +pub(crate) struct BackupCatalogIndex; + +#[async_trait] +impl CodeBody for BackupCatalogIndex { + fn fingerprint(&self) -> &'static str { + FINGERPRINT + } + + /// Nothing this migration finds stops it. + async fn detect(&self, _stores: &SqliteStores) -> Result, MigrationError> { + Ok(Vec::new()) + } + + async fn run(&self, stores: &SqliteStores) -> Result { + let pool = &stores.pool; + sqlx::raw_sql(GUARDED_SQL) + .execute(pool) + .await + .map_err(|e| MigrationError::store("create backup_index and backup_maintenance", e))?; + let mut notes = Vec::new(); + let present = columns_of(pool, "table_restores").await?; + for (name, declaration) in RESTORE_JOB_COLUMNS { + if present.iter().any(|c| c == name) { + continue; + } + sqlx::query(&format!("ALTER TABLE table_restores ADD COLUMN {name} {declaration}")) + .execute(pool) + .await + .map_err(|e| MigrationError::store(&format!("add table_restores.{name}"), e))?; + notes.push(format!("added table_restores.{name}")); + } + let running: i64 = sqlx::query_scalar( + "SELECT COUNT(*) FROM table_restores r JOIN tables t ON t.table_id = r.table_id \ + WHERE r.status IS NULL AND t.table_status = 'CREATING'", + ) + .fetch_one(pool) + .await + .map_err(|e| MigrationError::store("count interrupted restores", e))?; + let backfilled: i64 = + sqlx::query_scalar("SELECT COUNT(*) FROM table_restores WHERE status IS NULL") + .fetch_one(pool) + .await + .map_err(|e| MigrationError::store("count registrations", e))?; + sqlx::raw_sql(BACKFILL_SQL) + .execute(pool) + .await + .map_err(|e| MigrationError::store("backfill table_restores", e))?; + if backfilled > 0 { + notes.push(format!( + "backfilled {backfilled} restore registration(s) as legacy restores, {running} of \ + them RUNNING with a dead owner for the reconciler to remove" + )); + } + let wrote_id = sqlx::query( + "INSERT INTO settings (key, value) SELECT 'catalog_id', ? \ + WHERE NOT EXISTS (SELECT 1 FROM settings WHERE key = 'catalog_id')", + ) + .bind(uuid::Uuid::new_v4().to_string()) + .execute(pool) + .await + .map_err(|e| MigrationError::store("write catalog_id", e))? + .rows_affected(); + if wrote_id > 0 { + notes.push("wrote settings.catalog_id".to_owned()); + } + Ok(Done { notes }) + } +} + +/// The column names of `table`, from `pragma_table_info`. +async fn columns_of(pool: &SqlitePool, table: &str) -> Result, MigrationError> { + sqlx::query_scalar("SELECT name FROM pragma_table_info(?)") + .bind(table) + .fetch_all(pool) + .await + .map_err(|e| MigrationError::store(&format!("read the columns of {table}"), e)) +} diff --git a/crates/storage-sqlite/src/migrations/mod.rs b/crates/storage-sqlite/src/migrations/mod.rs index 37f3176ff..48d5ccfa9 100644 --- a/crates/storage-sqlite/src/migrations/mod.rs +++ b/crates/storage-sqlite/src/migrations/mod.rs @@ -29,6 +29,9 @@ mod m0003_sort_key_definitions; #[rustfmt::skip] #[allow(clippy::all, deprecated)] mod m0004_index_table_base_key; +#[rustfmt::skip] +#[allow(clippy::all, deprecated)] +mod m0005_backup_catalog_index; mod postcheck; use std::time::Duration; @@ -93,6 +96,15 @@ pub fn migration(id: MigrationId) -> Migration { m0004_index_table_base_key::SOURCE, )], }, + MigrationId::BackupCatalogIndex => Migration::Applies { + kind: Kind::Breaking, + convergent: true, + bodies: vec![StoreBody::code( + StoreKind::Catalog, + &m0005_backup_catalog_index::BackupCatalogIndex, + m0005_backup_catalog_index::SOURCE, + )], + }, } } diff --git a/crates/storage-sqlite/src/migrations/released_tests.rs b/crates/storage-sqlite/src/migrations/released_tests.rs index ce64c5f83..870453895 100644 --- a/crates/storage-sqlite/src/migrations/released_tests.rs +++ b/crates/storage-sqlite/src/migrations/released_tests.rs @@ -307,7 +307,16 @@ async fn every_released_catalog_upgrades_to_the_fresh_schema() { for (key, value) in state.settings { expected_settings.insert((*key).to_owned(), (*value).to_owned()); } - assert_eq!(settings(&db).await, expected_settings, "{name}: settings"); + // Migration 5 writes a random catalog id on every catalog. + let mut actual_settings = settings(&db).await; + let catalog_id = actual_settings + .remove("catalog_id") + .unwrap_or_else(|| panic!("{name}: no catalog_id")); + assert!( + uuid::Uuid::parse_str(&catalog_id).is_ok(), + "{name}: {catalog_id}" + ); + assert_eq!(actual_settings, expected_settings, "{name}: settings"); // No post-check note: no released state has a pre-release shape. let notes = scalar(&db, "SELECT notes FROM extenddb_migrations WHERE id = 1").await; diff --git a/crates/storage-sqlite/src/migrations/tests.rs b/crates/storage-sqlite/src/migrations/tests.rs index baff616a2..db8aa1557 100644 --- a/crates/storage-sqlite/src/migrations/tests.rs +++ b/crates/storage-sqlite/src/migrations/tests.rs @@ -275,9 +275,13 @@ async fn a_fresh_database_is_current_after_one_run_and_seeds_nothing() { assert_eq!(version_row(&db).await, Some(LATEST.to_string())); assert_eq!( count(&db, "SELECT COUNT(*) FROM settings").await, - 1, - "only the version row" + 2, + "the version row and the catalog id migration 5 writes" ); + let catalog_id = scalar(&db, "SELECT value FROM settings WHERE key = 'catalog_id'") + .await + .expect("a catalog id"); + assert!(uuid::Uuid::parse_str(&catalog_id).is_ok(), "{catalog_id}"); assert!(run(&db).await.expect("again").applied.is_empty()); } diff --git a/crates/storage-sqlite/src/snapshot_engine.rs b/crates/storage-sqlite/src/snapshot_engine.rs index e8f12a53e..e79fc6da0 100644 --- a/crates/storage-sqlite/src/snapshot_engine.rs +++ b/crates/storage-sqlite/src/snapshot_engine.rs @@ -30,10 +30,10 @@ //! pool free for requests counts snapshots too; when no slot is free the //! open returns `LimitExceeded` at once rather than queueing. The slot count //! is stated once, in `SqliteConfig`'s doc (`config.rs`). The other side of -//! sharing: a `CreateBackup` waits 30 s for a slot, so snapshots holding -//! every slot make every `CreateBackup` wait and then fail with the shared -//! refusal (`backup::slots_exhausted`), which names table snapshots for that -//! reason. +//! sharing: a `CreateBackup` tries the same slots without waiting +//! (`backup::acquire_backup_permit`), so snapshots holding every slot make +//! every `CreateBackup` fail at once with the one `LimitExceededException` +//! text, with no backup running. //! //! In-memory databases are pinned to a single connection (a second connection //! would open a different database), so a stream that held its transaction @@ -120,9 +120,10 @@ impl SqliteEngine { // would hold every connection and every writer would wait out the // pool's acquire timeout under the engine write lock. Refuse rather // than queue; the caller bounds concurrency. - let permit = std::sync::Arc::clone(&self.backup_server_slots) - .try_acquire_owned() - .map_err(|_| crate::backup::slots_exhausted(self.backup_server_slot_count))?; + let permit = crate::backup::acquire_backup_permit( + &self.backup_server_slots, + self.backup_server_slot_count, + )?; let mut tx: Transaction<'static, Sqlite> = self.pool.begin().await.map_err(|e| db_err(&e))?; @@ -696,19 +697,21 @@ mod tests { .await .expect_err("no slot left"); assert!(matches!(err, StorageError::LimitExceeded(_)), "{err}"); - // CreateBackup's admission on the same slots, with its 30 s wait - // shortened: with snapshots holding every slot and no backup running, - // the refusal must name snapshots. + // CreateBackup's admission on the same slots: with snapshots holding + // every slot and no backup running, it is refused at once with the + // one text. match crate::backup::acquire_backup_permit( - std::sync::Arc::clone(&engine.backup_server_slots), + &engine.backup_server_slots, engine.backup_server_slot_count, - std::time::Duration::from_millis(50), - ) - .await - { - Err(StorageError::LimitExceeded(m)) => assert!( - m.contains("backups or table snapshots") && m.contains(&format!("at most {slots}")), - "{m}" + ) { + Err(StorageError::LimitExceeded(m)) => assert_eq!( + m, + extenddb_storage::backup_catalog::limit_exceeded_message( + extenddb_storage::backup_catalog::JobBound::Counted { + name: crate::backup::BACKUP_SERVER_SLOTS_NAME, + limit: slots, + } + ) ), other => panic!( "CreateBackup is refused while snapshots hold every slot: {:?}", diff --git a/crates/storage-sqlite/src/store.rs b/crates/storage-sqlite/src/store.rs index 4c61482c0..e5daac6de 100644 --- a/crates/storage-sqlite/src/store.rs +++ b/crates/storage-sqlite/src/store.rs @@ -86,6 +86,9 @@ pub struct SqliteEngine { /// it; see [`crate::backup::backup_server_slot_count`]. pub(crate) backup_server_slot_count: usize, pub(crate) backup_server_slots: Arc, + /// The in-process claims of the backup and restore jobs this process + /// runs; see [`crate::backup_catalog::JobSlots`]. + pub(crate) job_slots: Arc, } impl SqliteEngine { @@ -177,6 +180,7 @@ impl SqliteEngine { serve_lock: None, backup_server_slot_count, backup_server_slots: Arc::new(tokio::sync::Semaphore::new(backup_server_slot_count)), + job_slots: Arc::default(), }) } @@ -560,20 +564,26 @@ mod d1_write_lock_tests { #[tokio::test] async fn delete_backup_waits_for_the_write_lock() { - use extenddb_storage::BackupEngine; + use extenddb_storage::{JobPermit, LegacyBackupReader}; let engine = engine().await; create_table(&engine, "d1-bkp-t").await; // The backup must exist before the lock window: delete_backup resolves // it with a read first and returns early when it is missing, which // would complete without ever reaching the writes under test. - let details = BackupEngine::create_backup(&engine, "000000000000", "d1-bkp-t", "b") - .await - .expect("backup"); + let details = LegacyBackupReader::create( + &engine, + "000000000000", + "d1-bkp-t", + "b", + JobPermit::detached(), + ) + .await + .expect("backup"); let e = engine.clone(); assert_serialized( &engine, async move { - BackupEngine::delete_backup(&e, "000000000000", &details.backup_arn) + LegacyBackupReader::delete(&e, "000000000000", &details.backup_arn) .await .expect("delete backup"); }, @@ -584,7 +594,7 @@ mod d1_write_lock_tests { #[tokio::test] async fn update_continuous_backups_waits_for_the_write_lock() { - use extenddb_storage::BackupEngine; + use extenddb_storage::LegacyBackupReader; let engine = engine().await; // The table must exist before the lock window: the pre-lock existence // check returns TableNotFound otherwise, completing without reaching @@ -594,9 +604,14 @@ mod d1_write_lock_tests { assert_serialized( &engine, async move { - BackupEngine::update_continuous_backups(&e, "000000000000", "d1-pitr-t", true) - .await - .expect("update continuous backups"); + LegacyBackupReader::update_continuous_backups( + &e, + "000000000000", + "d1-pitr-t", + true, + ) + .await + .expect("update continuous backups"); }, "update_continuous_backups", ) diff --git a/crates/storage-sqlite/tests/fixtures/golden/0005_backup_catalog_index.txt b/crates/storage-sqlite/tests/fixtures/golden/0005_backup_catalog_index.txt new file mode 100644 index 000000000..63734b9c1 --- /dev/null +++ b/crates/storage-sqlite/tests/fixtures/golden/0005_backup_catalog_index.txt @@ -0,0 +1,47 @@ +-- SELECT t.table_name, t.table_status, r.status, r.kind, r.owner_generation, r.heartbeat_at, r.store_kind, r.completed_at = t.creation_date_time FROM table_restores r JOIN tables t ON t.table_id = r.table_id ORDER BY t.table_name +cu_orders ACTIVE DONE legacy 0 1970-01-01T00:00:00.000Z NULL 1 +interrupted CREATING RUNNING legacy 0 1970-01-01T00:00:00.000Z NULL NULL +-- SELECT name FROM pragma_table_info('table_restores') ORDER BY cid +table_id +source_backup_arn +source_table_arn +restore_date_time +status +kind +store_kind +store_root +store_id +owner_generation +heartbeat_at +spool_host +file_index +items_applied +failure_reason +completed_at +-- SELECT name, type FROM sqlite_master WHERE name LIKE 'backup_%' OR name LIKE 'idx_backup_%' OR name = 'idx_table_restores_status' ORDER BY name +backup_definitions table +backup_index table +backup_items table +backup_maintenance table +backups table +idx_backup_index_account_table index +idx_backup_index_backup_id index +idx_backup_index_store_status index +idx_backup_items_arn index +idx_backups_table index +idx_table_restores_status index +-- SELECT notes FROM extenddb_migrations WHERE id = 5 +added table_restores.status +added table_restores.kind +added table_restores.store_kind +added table_restores.store_root +added table_restores.store_id +added table_restores.owner_generation +added table_restores.heartbeat_at +added table_restores.spool_host +added table_restores.file_index +added table_restores.items_applied +added table_restores.failure_reason +added table_restores.completed_at +backfilled 2 restore registration(s) as legacy restores, 1 of them RUNNING with a dead owner for the reconciler to remove +wrote settings.catalog_id diff --git a/crates/storage/src/backup_catalog.rs b/crates/storage/src/backup_catalog.rs new file mode 100644 index 000000000..d2168aced --- /dev/null +++ b/crates/storage/src/backup_catalog.rs @@ -0,0 +1,1254 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! The catalog index of backups and restore jobs, and the frozen reader of +//! the in-catalog backups earlier releases wrote. +//! +//! The engine (`crates/engine/src/backup.rs`) is backend-agnostic, and every +//! write to a catalog row (the claim, the heartbeat, the conditional status +//! writes, the store-scoped listings, the restore job record) has to be +//! written by the backend that holds the row. [`BackupCatalog`] is how the +//! engine reaches the index; [`JobClaim`] is the handle a job holds on its +//! row; [`LegacyBackupReader`] is the frozen path to backups stored in the +//! catalog itself (`backups` and `backup_items`), which stays until that +//! format is retired. +//! +//! Rows are addressed by [`BackupKey`] (account, backup id, ARN), built from +//! a parsed [`BackupArn`]; no method takes a bare backup id. A restore target +//! is addressed by [`RestoreTargetKey`] (account, name, table id), the table +//! id guarding against a table re-created under the name. +//! +//! Status mapping to the API ([`BackupStatus::reported`]): CREATING and +//! AVAILABLE as themselves; DELETING is reported as DELETED by DescribeBackup +//! and DeleteBackup and is not listed; DELETED and FAILED rows are tombstones +//! that are neither described nor listed. + +use std::sync::Arc; +use std::time::Duration; + +use extenddb_core::backup_format::{BackupArn, SourceTableSchema}; +use extenddb_core::types::{ + BackupDescription, BackupDetails, BackupSummary, BillingMode, GlobalSecondaryIndexInfo, + LocalSecondaryIndexInfo, ProvisionedThroughput, SourceTableDetails, SourceTableFeatureDetails, + TableDescription, +}; +use futures::future::BoxFuture; +use futures::stream::BoxStream; + +use crate::backup_definition::{BackupRestoreSource, RestoreRequest}; +use crate::error::StorageError; + +/// `BackupType` of every backup this server creates. +pub const BACKUP_TYPE_USER: &str = "USER"; + +/// The sentinel `owner_generation` the migration writes on a restore +/// registration that existed before the job record had one; no job ever +/// holds it. +pub const UNOWNED_GENERATION: u64 = 0; + +/// A fresh `owner_generation`: random, never [`UNOWNED_GENERATION`], and +/// below `i64::MAX` so every backend's integer column holds it. +#[must_use] +pub fn new_generation() -> u64 { + loop { + let g = rand::random::() >> 1; + if g != UNOWNED_GENERATION { + return g; + } + } +} + +/// Connections a `[storage.postgres] pool_size` must have for +/// `max_concurrent_jobs` jobs: twice the jobs, so a backend that admits at +/// most half its pool as open snapshots (#395) still admits every job the +/// permit allows, plus ten for requests. The caller takes the larger of this +/// and its own minimum pool size. A job's claim connection is outside the +/// pool and not counted here. +#[must_use] +pub fn pool_size_for_jobs(max_concurrent_jobs: usize) -> u32 { + let jobs = u32::try_from(max_concurrent_jobs).unwrap_or(u32::MAX); + jobs.saturating_mul(2).saturating_add(10) +} + +/// How often a running job refreshes the heartbeat of the record it owns, +/// on every backend, until a later release wires `[backup] +/// heartbeat_interval_secs` to the jobs. `BackupConfig::validate` floors +/// `stale_job_after_secs` on this period, so a configured interval shorter +/// than it cannot make the reconciler declare a live job dead between two +/// heartbeats. +pub const JOB_HEARTBEAT_INTERVAL: Duration = Duration::from_secs(15); + +/// The bound a job refusal names, so the text says which one was met +/// (an operator reading the engine's `max_concurrent_jobs` refusal and +/// SQLite's slot refusal, which differ in number, can tell them apart). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum JobBound<'a> { + /// A counted bound of this instance: `name` (the config key or the + /// backend's slot formula) admits `limit` jobs at once. + Counted { name: &'a str, limit: usize }, + /// The database server refused the job's dedicated connection + /// (PostgreSQL's `too_many_connections`): the bound is the server's own + /// connection limit, which this process does not know. + ServerConnections, +} + +/// The text of the `LimitExceededException` a CreateBackup or +/// RestoreTableFromBackup answers when a bound on running jobs is met. One +/// implementation: the engine's `max_concurrent_jobs` permit, a backend's own +/// admission (SQLite's snapshot slots) and PostgreSQL's claim connection +/// answer the same wire refusal, each naming its bound. +#[must_use] +pub fn limit_exceeded_message(bound: JobBound<'_>) -> String { + let detail = match bound { + JobBound::Counted { name, limit } => format!("at most {limit} at once, the {name} bound"), + JobBound::ServerConnections => { + "the database server has no connection slot left for the job".to_owned() + } + }; + format!("Too many backup and restore jobs are running on this server ({detail}); retry later") +} + +/// Where a backup's bytes live. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum StoreKind { + Filesystem, + S3, +} + +impl StoreKind { + /// The value stored in the `store_kind` column. + #[must_use] + pub const fn as_str(self) -> &'static str { + match self { + Self::Filesystem => "filesystem", + Self::S3 => "s3", + } + } + + /// The kind a `store_kind` column holds. + #[must_use] + pub fn parse(s: &str) -> Option { + match s { + "filesystem" => Some(Self::Filesystem), + "s3" => Some(Self::S3), + _ => None, + } + } +} + +impl std::fmt::Display for StoreKind { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(self.as_str()) + } +} + +/// The store a row belongs to: kind, the configured root at creation time, +/// and the id in the root's `extenddb-store.json`. Every duty that reads or +/// writes a backup row acts only on rows of the configured store. +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct StoreIdentity { + pub store_kind: StoreKind, + pub store_root: String, + pub store_id: String, +} + +/// The status column of a backup row. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum BackupStatus { + Creating, + Available, + Deleting, + Deleted, + Failed, +} + +impl BackupStatus { + /// The value stored in the `status` column. + #[must_use] + pub const fn as_str(self) -> &'static str { + match self { + Self::Creating => "CREATING", + Self::Available => "AVAILABLE", + Self::Deleting => "DELETING", + Self::Deleted => "DELETED", + Self::Failed => "FAILED", + } + } + + /// The status a column holds. + /// + /// # Errors + /// + /// `Internal` for a value no release writes. + pub fn parse(s: &str) -> Result { + match s { + "CREATING" => Ok(Self::Creating), + "AVAILABLE" => Ok(Self::Available), + "DELETING" => Ok(Self::Deleting), + "DELETED" => Ok(Self::Deleted), + "FAILED" => Ok(Self::Failed), + other => Err(StorageError::Internal(format!( + "backup row carries an unknown status {other:?}" + ))), + } + } + + /// The `BackupStatus` the API reports for a row of this status: DELETING + /// is reported as DELETED, so a client retrying DeleteBackup sees the + /// terminal status. + #[must_use] + pub const fn reported(self) -> &'static str { + match self { + Self::Deleting => "DELETED", + other => other.as_str(), + } + } + + /// Whether DescribeBackup answers for a row of this status (CREATING, + /// AVAILABLE, and DELETING as DELETED). DELETED and FAILED rows are + /// `BackupNotFoundException`. + #[must_use] + pub const fn is_described(self) -> bool { + matches!(self, Self::Creating | Self::Available | Self::Deleting) + } + + /// Whether ListBackups lists a row of this status (CREATING and + /// AVAILABLE only). + #[must_use] + pub const fn is_listed(self) -> bool { + matches!(self, Self::Creating | Self::Available) + } +} + +impl std::fmt::Display for BackupStatus { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(self.as_str()) + } +} + +/// One backup row's address: the account, the backup id and the ARN as the +/// row stores it. Built from a parsed [`BackupArn`] or from a row read. +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct BackupKey { + pub account_id: String, + pub backup_id: String, + pub backup_arn: String, +} + +impl BackupKey { + /// The key of the row `arn` names. + #[must_use] + pub fn from_arn(arn: &BackupArn) -> Self { + Self { + account_id: arn.account_id().to_owned(), + backup_id: arn.backup_id().to_owned(), + backup_arn: arn.to_arn_string(), + } + } +} + +/// One restore target's address. The table id guards against a table +/// re-created under the same name after the target was removed. +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct RestoreTargetKey { + pub account_id: String, + pub table_name: String, + pub table_id: String, +} + +/// The `TimeRangeLowerBound` (inclusive) and `TimeRangeUpperBound` +/// (exclusive) of ListBackups, epoch seconds, applied to `created_at`. +#[derive(Debug, Clone, Copy, Default, PartialEq)] +pub struct TimeBounds { + pub lower: Option, + pub upper: Option, +} + +impl TimeBounds { + /// Whether `created_at` (epoch seconds) is within the bounds. + #[must_use] + pub fn contains(&self, created_at: f64) -> bool { + self.lower.is_none_or(|lower| created_at >= lower) + && self.upper.is_none_or(|upper| created_at < upper) + } +} + +/// A row of `backup_index`. Times are epoch seconds; `created_at` is the +/// id's instant, so `BackupCreationDateTime`, the ARN and the ListBackups +/// cursor agree. +#[derive(Debug, Clone, PartialEq)] +pub struct BackupRow { + pub backup_id: String, + pub account_id: String, + pub table_name: String, + pub backup_arn: String, + pub backup_name: String, + pub status: BackupStatus, + pub store: StoreIdentity, + pub size_bytes: i64, + pub item_count: i64, + pub created_at: f64, + pub completed_at: Option, + pub failure_reason: Option, + /// The manifest's `source_table` object, verbatim + /// ([`SourceTableSchema`]); the API's source-table shapes derive from it. + pub source_table_details_json: serde_json::Value, + pub owner_generation: u64, + /// The last heartbeat by the owning job, on the database's clock. + pub heartbeat_at: f64, +} + +impl BackupRow { + /// The row's address. + #[must_use] + pub fn key(&self) -> BackupKey { + BackupKey { + account_id: self.account_id.clone(), + backup_id: self.backup_id.clone(), + backup_arn: self.backup_arn.clone(), + } + } + + /// The source table the row recorded. + /// + /// # Errors + /// + /// `Internal` when `source_table_details_json` is not a manifest + /// `source_table` object. + pub fn source_table(&self) -> Result { + serde_json::from_value(self.source_table_details_json.clone()).map_err(|e| { + StorageError::Internal(format!( + "backup {} carries a source table this build cannot read: {e}", + self.backup_arn + )) + }) + } + + /// `BackupDetails` as DescribeBackup and DeleteBackup report them, with + /// the status mapped by [`BackupStatus::reported`]. + #[must_use] + pub fn details(&self) -> BackupDetails { + BackupDetails { + backup_arn: self.backup_arn.clone(), + backup_name: self.backup_name.clone(), + backup_status: self.status.reported().to_owned(), + backup_type: BACKUP_TYPE_USER.to_owned(), + backup_size_bytes: self.size_bytes, + backup_creation_date_time: self.created_at, + } + } + + /// The DescribeBackup description: the details, `SourceTableDetails` and + /// `SourceTableFeatureDetails` derived from the recorded source table. + /// `SourceTableDetails.ProvisionedThroughput` is always set (`{0, 0}` for + /// an on-demand table, as the service reports it); a GSI's throughput is + /// `{0, 0}` on an on-demand table too. Feature members the source + /// recorded nothing for are omitted. + /// + /// # Errors + /// + /// As [`Self::source_table`]. + pub fn description(&self) -> Result { + let source = self.source_table()?; + let zero = ProvisionedThroughput { + read_capacity_units: 0, + write_capacity_units: 0, + }; + let on_demand = source.billing_mode == BillingMode::PayPerRequest; + let provisioned_throughput = if on_demand { + Some(zero.clone()) + } else { + source + .provisioned_throughput + .clone() + .or_else(|| Some(zero.clone())) + }; + let features = SourceTableFeatureDetails { + local_secondary_indexes: (!source.local_secondary_indexes.is_empty()).then(|| { + source + .local_secondary_indexes + .iter() + .map(|i| LocalSecondaryIndexInfo { + index_name: i.index_name.clone(), + key_schema: i.key_schema.clone(), + projection: i.projection.clone(), + }) + .collect() + }), + global_secondary_indexes: (!source.global_secondary_indexes.is_empty()).then(|| { + source + .global_secondary_indexes + .iter() + .map(|i| GlobalSecondaryIndexInfo { + index_name: i.index_name.clone(), + key_schema: i.key_schema.clone(), + projection: i.projection.clone(), + provisioned_throughput: if on_demand { + Some(zero.clone()) + } else { + i.provisioned_throughput.clone() + }, + }) + .collect() + }), + stream_description: source.stream_specification.clone(), + time_to_live_description: source.time_to_live.clone(), + sse_description: source.sse_description.clone(), + }; + Ok(BackupDescription { + backup_details: self.details(), + source_table_details: SourceTableDetails { + table_name: source.table_name.clone(), + table_id: source.table_id.clone(), + table_arn: source.table_arn.clone(), + key_schema: source.key_schema.clone(), + item_count: source.item_count, + table_size_bytes: source.table_size_bytes, + billing_mode: Some(billing_mode_name(source.billing_mode).to_owned()), + table_creation_date_time: source.table_creation_date_time, + provisioned_throughput, + on_demand_throughput: source.on_demand_throughput.clone(), + }, + source_table_feature_details: (!features.is_empty()).then_some(features), + }) + } + + /// The ListBackups summary. It needs only `table_id` and `table_arn` of + /// the recorded source table, and reads those two members directly, so a + /// row whose `source_table_details_json` a later build extended with a + /// member this build requires still lists; DescribeBackup of the same + /// row is the internal error [`Self::description`] reports. + /// + /// # Errors + /// + /// `Internal` when the recorded source carries no `table_id` or + /// `table_arn` string. + pub fn summary(&self) -> Result { + let member = |name: &str| -> Result { + self.source_table_details_json + .get(name) + .and_then(serde_json::Value::as_str) + .map(str::to_owned) + .ok_or_else(|| { + StorageError::Internal(format!( + "backup {} carries a source table without a {name} this build can read", + self.backup_arn + )) + }) + }; + Ok(BackupSummary { + backup_arn: self.backup_arn.clone(), + backup_name: self.backup_name.clone(), + table_name: self.table_name.clone(), + table_id: member("table_id")?, + table_arn: member("table_arn")?, + backup_status: self.status.reported().to_owned(), + backup_type: BACKUP_TYPE_USER.to_owned(), + backup_size_bytes: self.size_bytes, + backup_creation_date_time: self.created_at, + }) + } +} + +/// The wire name of a billing mode. +#[must_use] +pub const fn billing_mode_name(mode: BillingMode) -> &'static str { + match mode { + BillingMode::Provisioned => "PROVISIONED", + BillingMode::PayPerRequest => "PAY_PER_REQUEST", + } +} + +/// What `start_backup_delete` found under the row's lock. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum DeleteStart { + /// The row was AVAILABLE and is now DELETING. + Deleting, + /// A restore target registered from the backup is CREATING; its name. + InUse(String), + /// The row was not AVAILABLE; its status. + Status(BackupStatus), +} + +/// What a heartbeat found. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Heartbeat { + /// The row still carries the claim's generation and the claim is held. + Confirmed, + /// The row is gone, carries another generation, or the claim connection + /// no longer holds the claim. The caller treats an `Err` the same way. + Lost, +} + +/// The status a job record moves through: CREATING, AVAILABLE and FAILED for +/// a backup row; RUNNING, DONE and FAILED for a restore job record. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum JobStatus { + Creating, + Available, + Running, + Done, + Failed, +} + +impl JobStatus { + /// The value stored in the record's status column. + #[must_use] + pub const fn as_str(self) -> &'static str { + match self { + Self::Creating => "CREATING", + Self::Available => "AVAILABLE", + Self::Running => "RUNNING", + Self::Done => "DONE", + Self::Failed => "FAILED", + } + } + + /// The status a column holds. + /// + /// # Errors + /// + /// `Internal` for a value no release writes. + pub fn parse(s: &str) -> Result { + match s { + "CREATING" => Ok(Self::Creating), + "AVAILABLE" => Ok(Self::Available), + "RUNNING" => Ok(Self::Running), + "DONE" => Ok(Self::Done), + "FAILED" => Ok(Self::Failed), + other => Err(StorageError::Internal(format!( + "job record carries an unknown status {other:?}" + ))), + } + } +} + +impl std::fmt::Display for JobStatus { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(self.as_str()) + } +} + +/// The fields a status write sets beside the status. A `None` leaves the +/// column as it is; `completed` stamps `completed_at` with the database's +/// `now()`. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct StatusFields { + pub size_bytes: Option, + pub item_count: Option, + pub failure_reason: Option, + pub completed: bool, +} + +/// What a conditional write did. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum WriteOutcome { + /// Exactly one row matched the condition and was written. + Matched, + /// No row matched: the record is gone, carries another generation, or + /// is not in the `from` status. + NotMatched, + /// The statement may or may not have been applied (the connection failed + /// after it was sent). The caller re-reads the record ([`JobClaim::read`]) + /// rather than treating this as a failure. + Unknown, +} + +/// A job record as [`JobClaim::read`] returns it. +#[derive(Debug, Clone, PartialEq)] +pub struct JobRow { + pub status: JobStatus, + pub owner_generation: u64, + /// Epoch seconds, on the database's clock. + pub heartbeat_at: f64, + pub failure_reason: Option, +} + +/// How a restore job reads its backup. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum RestoreKind { + /// Through the [`LegacyBackupReader`]: an in-catalog backup. + Legacy, + /// From a manifest in the store. + Manifest, +} + +impl RestoreKind { + /// The value stored in the `kind` column. + #[must_use] + pub const fn as_str(self) -> &'static str { + match self { + Self::Legacy => "legacy", + Self::Manifest => "manifest", + } + } + + /// The kind a column holds. + /// + /// # Errors + /// + /// `Internal` for a value no release writes. + pub fn parse(s: &str) -> Result { + match s { + "legacy" => Ok(Self::Legacy), + "manifest" => Ok(Self::Manifest), + other => Err(StorageError::Internal(format!( + "restore job carries an unknown kind {other:?}" + ))), + } + } +} + +/// A restore job record the reconciler acts on: RUNNING with a dead owner, +/// or FAILED with its target still CREATING ([`BackupCatalog::list_dead_restore_jobs`]). +#[derive(Debug, Clone, PartialEq)] +pub struct RestoreJobRow { + pub target: RestoreTargetKey, + pub source_backup_arn: String, + pub kind: RestoreKind, + /// RUNNING or FAILED; the reconciler reads it to decide whether the + /// FAILED write is still to be made. + pub status: JobStatus, + pub owner_generation: u64, + /// Epoch seconds, on the database's clock. + pub heartbeat_at: f64, +} + +/// A legacy CREATING backup row whose create died. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct AbandonedLegacyBackup { + pub account_id: String, + pub backup_arn: String, +} + +/// One of the `max_concurrent_jobs` slots of this instance, held by a backup +/// or restore job for its life. The engine takes it last among a request's +/// checks and hands it to the backend with the job; dropping it frees the +/// slot. +#[derive(Debug)] +pub struct JobPermit(#[allow(dead_code)] tokio::sync::OwnedSemaphorePermit); + +impl JobPermit { + /// Take a free slot of `slots`, or `None` when every slot is held. The + /// request never waits: a queued job would hold a snapshot or a + /// transaction while it waited. + #[must_use] + pub fn try_acquire(slots: &Arc) -> Option { + Arc::clone(slots).try_acquire_owned().ok().map(Self) + } + + /// A permit of a slot nobody else can take: for a caller that runs a + /// job outside the engine's slots (a tool, a backend's own tests). + #[must_use] + pub fn detached() -> Self { + Self::try_acquire(&Arc::new(tokio::sync::Semaphore::new(1))).expect("a fresh slot") + } +} + +/// The text every backend and the engine answer for a backup a CREATING +/// restore target still reads (`BackupInUseException` on the wire). One +/// implementation: the refusal is wire-visible and was spelled in each +/// backend before. +#[must_use] +pub fn backup_in_use_message(table: &str, backup_arn: &str) -> String { + format!("Backup is being used to restore table {table}: {backup_arn}") +} + +/// [`backup_in_use_message`] as the storage error the engine maps. +#[must_use] +pub fn backup_in_use(table: &str, backup_arn: &str) -> StorageError { + StorageError::BackupInUse(backup_in_use_message(table, backup_arn)) +} + +/// The one signal of a missing backup every backend emits inside a +/// `Validation` error, which the engine turns into `BackupNotFoundException` +/// ([`is_backup_not_found`]). +pub const BACKUP_NOT_FOUND: &str = "Backup not found"; + +/// The error for a backup the catalog has no row for, or one of another +/// account, or (on the legacy reader) one that is not AVAILABLE. +#[must_use] +pub fn backup_not_found(backup_arn: &str) -> StorageError { + StorageError::Validation(format!("{BACKUP_NOT_FOUND}: {backup_arn}")) +} + +/// Whether a backend error says the backup is not there. +#[must_use] +pub fn is_backup_not_found(e: &StorageError) -> bool { + matches!(e, StorageError::Validation(m) if m.contains(BACKUP_NOT_FOUND)) +} + +/// The catalog index of backups and restore jobs, implemented by each +/// backend. The engine reaches every row through it; the per-backend +/// mechanics (the lock on a row, the claim, the clock) are the backend's. +pub trait BackupCatalog: Send + Sync { + /// The catalog's random id, written at `init` and by the migration that + /// created the index for an existing catalog. + fn catalog_id(&self) -> BoxFuture<'_, Result>; + + /// Insert `row` with status CREATING under a new claim, and return the + /// claim. The claim precedes the insert or is the insert itself, so no + /// row ever exists unowned. + /// + /// # Errors + /// + /// `AlreadyExists` when a live owner holds the row's claim or the row + /// exists. + fn claim_backup<'a>( + &'a self, + row: &'a BackupRow, + ) -> BoxFuture<'a, Result, StorageError>>; + + /// Every row of `store` with `status`, or of every status for `None`. + fn list_backup_rows<'a>( + &'a self, + status: Option, + store: &'a StoreIdentity, + ) -> BoxStream<'a, Result>; + + /// The account's rows, of `table_name` when given, created within + /// `bounds`, of every status: the caller decides which statuses are + /// listed ([`BackupStatus::is_listed`]). + fn list_account_backup_rows<'a>( + &'a self, + account_id: &'a str, + table_name: Option<&'a str>, + bounds: &'a TimeBounds, + ) -> BoxStream<'a, Result>; + + /// The row `arn` names, read by the ARN's account and backup id and + /// returned only when the row's `account_id`, `table_name` and + /// `backup_arn` equal the ARN's; `None` otherwise, of whatever status. + fn read_backup_row<'a>( + &'a self, + arn: &'a BackupArn, + ) -> BoxFuture<'a, Result, StorageError>>; + + /// DeleteBackup's first step: under the row's lock, the in-use check and + /// the AVAILABLE-to-DELETING write in one transaction. + /// + /// # Errors + /// + /// `Validation("Backup not found: ")` when there is no such row. + fn start_backup_delete<'a>( + &'a self, + key: &'a BackupKey, + ) -> BoxFuture<'a, Result>; + + /// DELETING to DELETED, conditional on the row being DELETING; `false` + /// when it was not. + fn set_backup_deleted<'a>( + &'a self, + key: &'a BackupKey, + ) -> BoxFuture<'a, Result>; + + /// Remove a DELETED or FAILED tombstone; `false` when the row is not one. + fn remove_backup_row<'a>( + &'a self, + key: &'a BackupKey, + ) -> BoxFuture<'a, Result>; + + /// The name of a CREATING restore target registered from the backup, if + /// any: the rule under which DeleteBackup and a second restore of the + /// backup are refused with `BackupInUseException`. Reads the target's + /// status, not the job's. + fn backup_in_use<'a>( + &'a self, + key: &'a BackupKey, + ) -> BoxFuture<'a, Result, StorageError>>; + + /// Remove a CREATING restore target, its job record and its progress + /// rows, conditional on the table still being a CREATING restore target + /// with this table id, so this and the ACTIVE flip exclude each other. + /// `false` when the table is no longer one. Not DeleteTable, which + /// refuses a restore target. Also resumes, from the drop, the removal of + /// a FAILED record's target that an earlier call left DELETING with its + /// registration (the state [`Self::list_dead_restore_jobs`] names). + fn remove_restore_target<'a>( + &'a self, + key: &'a RestoreTargetKey, + ) -> BoxFuture<'a, Result>; + + /// The restore job records the reconciler acts on, two kinds: RUNNING + /// records whose `heartbeat_at` is older than `stale_after` on the + /// database's clock (the owner is dead by every backend's rule; the + /// reconciler fails or resumes them), and FAILED records whose table is + /// still a CREATING restore target (the FAILED write landed and the + /// target's removal did not; the reconciler removes the target), listed + /// once the record's `completed_at`, or its `heartbeat_at` where that is + /// unset, is older than `stale_after`, so a job still on its own failure + /// path is left to finish it. On a backend whose removal is more than one + /// write (Cassandra: a claim CREATING to DELETING, then the drop), a + /// FAILED record whose target is DELETING with its registration is + /// returned once stale on the same rule, and + /// [`Self::remove_restore_target`] resumes the removal; PostgreSQL and + /// SQLite remove in one transaction and never leave that state, and + /// MongoDB deletes the document first, so neither does it. + fn list_dead_restore_jobs( + &self, + stale_after: Duration, + ) -> BoxFuture<'_, Result, StorageError>>; + + /// RUNNING to FAILED with `reason` on the target's job record, conditional + /// on RUNNING; `false` when the record was not RUNNING or is gone. The + /// reconciler's write for a dead legacy restore, which is never resumed. + fn fail_restore_job<'a>( + &'a self, + key: &'a RestoreTargetKey, + reason: &'a str, + ) -> BoxFuture<'a, Result>; + + /// Whether the catalog-level maintenance claim is held. Jobs check it + /// after their own claim and row exist and refuse while it is held; the + /// reconciler runs only the duties the holder waits on. + fn check_maintenance_claim(&self) -> BoxFuture<'_, Result>; +} + +/// A job's handle on its row: the claim that excludes a second owner and +/// lets a successor detect a dead one, and the writes the job makes through +/// it, each conditioned on the claim's generation. +pub trait JobClaim: Send + Sync { + /// The fencing token the claim was taken with. + fn generation(&self) -> u64; + + /// Refresh `heartbeat_at` on the database's clock and confirm the claim + /// is still held. The caller treats an `Err` as [`Heartbeat::Lost`]. + fn heartbeat(&self) -> BoxFuture<'_, Result>; + + /// Move the record from `from` to `to`, conditioned on the claim's + /// generation and on the record being in `from`. + fn set_status<'a>( + &'a self, + from: JobStatus, + to: JobStatus, + fields: &'a StatusFields, + ) -> BoxFuture<'a, Result>; + + /// Re-read the record, after a write whose outcome is unknown. + fn read(&self) -> BoxFuture<'_, Result>; +} + +/// The frozen path to backups stored in the catalog itself: the create, +/// describe, list, delete, restore-source and restore paths of the +/// in-catalog format, which each backend keeps as a copy of its own code for +/// reading and decoding that shape. The engine consults the catalog index +/// first and falls back to this reader; `create` is the CreateBackup path +/// while no `[backup.store]` is configured. +/// +/// The one part that is not frozen is the restore's registration: a restore +/// registers through the restore job fields (`kind = legacy`, the claim), so +/// the reconciler finds a restore whose owner died as it finds any other. +pub trait LegacyBackupReader: Send + Sync { + /// Create an in-catalog backup of a table, snapshotting all items. The + /// permit is the job's slot, released when the create ends. + fn create( + &self, + account_id: &str, + table_name: &str, + backup_name: &str, + permit: JobPermit, + ) -> BoxFuture<'_, Result>; + + /// Describe a backup by ARN, scoped to the owning account. + /// + /// Backups belong to an account, so implementations must match on both + /// `account_id` and `backup_arn` and report a missing backup when the ARN + /// belongs to another account. + fn describe( + &self, + account_id: &str, + backup_arn: &str, + ) -> BoxFuture<'_, Result>; + + /// List the account's in-catalog backups, optionally only those of + /// `table_name`, in any order. Each summary carries the source table's + /// `TableId`. + fn list( + &self, + account_id: &str, + table_name: Option<&str>, + ) -> BoxFuture<'_, Result, StorageError>>; + + /// Delete a backup by ARN, scoped to the owning account; an ARN owned by + /// another account is reported as missing. `BackupInUse` while a restore + /// target of the backup is CREATING. + fn delete( + &self, + account_id: &str, + backup_arn: &str, + ) -> BoxFuture<'_, Result>; + + /// The backup's recorded source, for an AVAILABLE backup owned by + /// `account_id`. + /// + /// The engine resolves the restore target and runs every refusal from + /// this, and fills DescribeBackup's `SourceTableFeatureDetails` and + /// `SourceTableDetails.ProvisionedThroughput` from it. `definition` is the + /// stored [`crate::backup_definition::BackupTableDefinition`] (with + /// `time_to_live` and `stream_specification` as captured at + /// CreateBackup), or `None` for a backup recorded without one. + /// + /// # Errors + /// + /// `StorageError::Validation("Backup not found: ")` if the backup is + /// missing, owned by another account, or not AVAILABLE. + /// `StorageError::Unsupported` for a stored definition this build cannot + /// read. + fn restore_source( + &self, + account_id: &str, + backup_arn: &str, + ) -> BoxFuture<'_, Result>; + + /// Restore `request.input.table_name` from `request.backup_arn`. + /// + /// `request.input` is resolved and validated by the engine; create exactly + /// that table, CREATING. In the same atomic step that re-checks the backup + /// is still AVAILABLE and not in use, register the restore (the job + /// record, `kind = legacy`, with the claim) so DescribeTable can report + /// `RestoreSummary`, DeleteBackup refuses the source, and DeleteTable + /// refuses the target while it is being restored. Then return the CREATING + /// description at once. `permit` is the job's slot, released when the + /// copy ends. + /// + /// The item copy runs in a detached task that owns its own clones, so + /// the request future being dropped does not stop it. The task flips the + /// target ACTIVE when the copy completes, or removes the target on + /// failure. + /// + /// # Errors + /// + /// Returned synchronously, before the copy starts: + /// `StorageError::Validation("Backup not found: ")` if the backup is + /// not AVAILABLE at registration; `StorageError::BackupInUse` if a + /// target of it is CREATING; `StorageError::TableAlreadyExists(name)` if + /// the name is taken; `StorageError::TableBeingRestored` if it is a + /// CREATING restore target. + fn restore( + &self, + account_id: &str, + request: RestoreRequest, + permit: JobPermit, + ) -> BoxFuture<'_, Result>; + + /// In-catalog CREATING backup rows whose create is dead: on a backend + /// whose legacy create commits a CREATING row before the copy, those + /// whose heartbeat is older than `stale_after`, or that no running create + /// holds. The reconciler removes each through [`Self::delete`]. Empty on + /// a backend whose legacy create commits its row AVAILABLE with the copy. + fn abandoned_creating_backups( + &self, + stale_after: Duration, + ) -> BoxFuture<'_, Result, StorageError>>; + + /// Describe continuous backups / PITR status for a table. + fn describe_continuous_backups( + &self, + account_id: &str, + table_name: &str, + ) -> BoxFuture<'_, Result>; + + /// Update continuous backups (enable/disable PITR). + fn update_continuous_backups( + &self, + account_id: &str, + table_name: &str, + pitr_enabled: bool, + ) -> BoxFuture<'_, Result>; +} + +#[cfg(test)] +mod tests { + use extenddb_core::types::{KeyType, ProjectionType}; + + use super::*; + + /// The `source_table` object of the manifest fixture + /// `crates/core/testdata/extenddb-manifest/v1.json`. + fn fixture_source_table() -> serde_json::Value { + let manifest: serde_json::Value = serde_json::from_str(include_str!( + "../../core/testdata/extenddb-manifest/v1.json" + )) + .expect("fixture parses"); + manifest["source_table"].clone() + } + + fn row(status: BackupStatus) -> BackupRow { + BackupRow { + backup_id: "01757500000123-0badcafe".to_owned(), + account_id: "123456789012".to_owned(), + table_name: "Music".to_owned(), + backup_arn: + "arn:aws:dynamodb:us-east-1:123456789012:table/Music/backup/01757500000123-0badcafe" + .to_owned(), + backup_name: "music-nightly".to_owned(), + status, + store: StoreIdentity { + store_kind: StoreKind::Filesystem, + store_root: "/var/lib/extenddb/backups".to_owned(), + store_id: "store-1".to_owned(), + }, + size_bytes: 61828, + item_count: 100_000, + created_at: 1_757_500_000.123, + completed_at: None, + failure_reason: None, + source_table_details_json: fixture_source_table(), + owner_generation: 7, + heartbeat_at: 1_757_500_001.0, + } + } + + #[test] + fn a_description_derives_from_the_manifest_source_table() { + let desc = row(BackupStatus::Available).description().unwrap(); + let d = &desc.backup_details; + assert_eq!(d.backup_status, "AVAILABLE"); + assert_eq!(d.backup_type, "USER"); + assert_eq!(d.backup_size_bytes, 61828); + assert_eq!(d.backup_creation_date_time, 1_757_500_000.123); + let s = &desc.source_table_details; + assert_eq!(s.table_name, "Music"); + assert_eq!(s.table_id, "12345a12-abcd-123a-ab12-1234abc12345"); + assert_eq!( + s.table_arn, + "arn:aws:dynamodb:us-east-1:123456789012:table/Music" + ); + assert_eq!(s.key_schema.len(), 2); + assert_eq!(s.key_schema[1].key_type, KeyType::Range); + assert_eq!(s.item_count, 100_000); + assert_eq!(s.table_size_bytes, 123_456); + assert_eq!(s.billing_mode.as_deref(), Some("PROVISIONED")); + assert_eq!(s.table_creation_date_time, 1_757_400_000.0); + assert_eq!( + s.provisioned_throughput, + Some(ProvisionedThroughput { + read_capacity_units: 5, + write_capacity_units: 5, + }) + ); + assert_eq!( + s.on_demand_throughput + .as_ref() + .and_then(|o| o.max_read_request_units), + Some(100) + ); + let f = desc.source_table_feature_details.expect("features"); + let gsi = &f.global_secondary_indexes.unwrap()[0]; + assert_eq!(gsi.index_name, "GenreIndex"); + assert_eq!(gsi.projection.projection_type, ProjectionType::Include); + assert_eq!( + gsi.provisioned_throughput, + Some(ProvisionedThroughput { + read_capacity_units: 1, + write_capacity_units: 1, + }) + ); + let lsi = &f.local_secondary_indexes.unwrap()[0]; + assert_eq!(lsi.index_name, "TitleIndex"); + assert_eq!(lsi.projection.projection_type, ProjectionType::KeysOnly); + assert!(f.stream_description.unwrap().stream_enabled); + assert_eq!( + f.time_to_live_description + .unwrap() + .attribute_name + .as_deref(), + Some("expires_at") + ); + let sse = f.sse_description.unwrap(); + assert_eq!(sse.status, "ENABLED"); + assert_eq!(sse.sse_type, Some(extenddb_core::types::SseType::KMS)); + } + + #[test] + fn an_on_demand_source_reports_zero_throughput_on_the_table_and_its_indexes() { + let mut row = row(BackupStatus::Available); + row.source_table_details_json["billing_mode"] = "PAY_PER_REQUEST".into(); + let desc = row.description().unwrap(); + let zero = ProvisionedThroughput { + read_capacity_units: 0, + write_capacity_units: 0, + }; + assert_eq!( + desc.source_table_details.billing_mode.as_deref(), + Some("PAY_PER_REQUEST") + ); + assert_eq!( + desc.source_table_details.provisioned_throughput, + Some(zero.clone()) + ); + let f = desc.source_table_feature_details.unwrap(); + assert_eq!( + f.global_secondary_indexes.unwrap()[0].provisioned_throughput, + Some(zero) + ); + } + + #[test] + fn a_source_without_features_omits_the_feature_details() { + let mut row = row(BackupStatus::Available); + let s = &mut row.source_table_details_json; + s["global_secondary_indexes"] = serde_json::json!([]); + s["local_secondary_indexes"] = serde_json::json!([]); + s["stream_specification"] = serde_json::Value::Null; + s["time_to_live"] = serde_json::Value::Null; + s["sse_description"] = serde_json::Value::Null; + let desc = row.description().unwrap(); + assert!(desc.source_table_feature_details.is_none()); + } + + #[test] + fn a_summary_carries_the_source_table_id_and_arn() { + let s = row(BackupStatus::Creating).summary().unwrap(); + assert_eq!(s.table_id, "12345a12-abcd-123a-ab12-1234abc12345"); + assert_eq!( + s.table_arn, + "arn:aws:dynamodb:us-east-1:123456789012:table/Music" + ); + assert_eq!(s.table_name, "Music"); + assert_eq!(s.backup_status, "CREATING"); + assert_eq!(s.backup_creation_date_time, 1_757_500_000.123); + } + + #[test] + fn deleting_is_reported_as_deleted_and_tombstones_are_hidden() { + assert_eq!( + row(BackupStatus::Deleting).details().backup_status, + "DELETED" + ); + assert!(BackupStatus::Deleting.is_described()); + assert!(!BackupStatus::Deleting.is_listed()); + for status in [BackupStatus::Deleted, BackupStatus::Failed] { + assert!(!status.is_described(), "{status}"); + assert!(!status.is_listed(), "{status}"); + } + for status in [BackupStatus::Creating, BackupStatus::Available] { + assert!(status.is_described(), "{status}"); + assert!(status.is_listed(), "{status}"); + assert_eq!(status.reported(), status.as_str()); + } + } + + #[test] + fn a_source_table_this_build_cannot_read_is_internal() { + let mut row = row(BackupStatus::Available); + row.source_table_details_json = serde_json::json!({"table_name": "x"}); + assert!(matches!(row.description(), Err(StorageError::Internal(_)))); + assert!(matches!(row.summary(), Err(StorageError::Internal(_)))); + } + + /// A source table a later build extended with a member this build + /// requires (here: every member but the two the summary reads is gone) + /// still summarizes; only the description needs the whole object. + #[test] + fn a_summary_needs_only_the_source_table_id_and_arn() { + let mut row = row(BackupStatus::Available); + row.source_table_details_json = serde_json::json!({ + "table_id": "12345a12-abcd-123a-ab12-1234abc12345", + "table_arn": "arn:aws:dynamodb:us-east-1:123456789012:table/Music", + "member_this_build_does_not_know": true, + }); + assert!(matches!(row.description(), Err(StorageError::Internal(_)))); + let s = row.summary().unwrap(); + assert_eq!(s.table_id, "12345a12-abcd-123a-ab12-1234abc12345"); + assert_eq!( + s.table_arn, + "arn:aws:dynamodb:us-east-1:123456789012:table/Music" + ); + row.source_table_details_json["table_arn"] = serde_json::json!(7); + let e = row.summary().unwrap_err(); + assert!(e.to_string().contains("table_arn"), "{e}"); + } + + #[test] + fn statuses_round_trip_through_their_column_values() { + for status in [ + BackupStatus::Creating, + BackupStatus::Available, + BackupStatus::Deleting, + BackupStatus::Deleted, + BackupStatus::Failed, + ] { + assert_eq!(BackupStatus::parse(status.as_str()).unwrap(), status); + } + assert!(BackupStatus::parse("ACTIVE").is_err()); + for status in [ + JobStatus::Creating, + JobStatus::Available, + JobStatus::Running, + JobStatus::Done, + JobStatus::Failed, + ] { + assert_eq!(JobStatus::parse(status.as_str()).unwrap(), status); + } + for kind in [RestoreKind::Legacy, RestoreKind::Manifest] { + assert_eq!(RestoreKind::parse(kind.as_str()).unwrap(), kind); + } + for kind in [StoreKind::Filesystem, StoreKind::S3] { + assert_eq!(StoreKind::parse(kind.as_str()), Some(kind)); + } + assert_eq!(StoreKind::parse("gcs"), None); + } + + #[test] + fn a_key_is_built_from_the_parsed_arn() { + let arn = BackupArn::parse( + "arn:aws:dynamodb:us-east-1:123456789012:table/Music/backup/01757500000123-0badcafe", + ) + .unwrap(); + let key = BackupKey::from_arn(&arn); + assert_eq!(key.account_id, "123456789012"); + assert_eq!(key.backup_id, "01757500000123-0badcafe"); + assert_eq!(key.backup_arn, arn.to_arn_string()); + assert_eq!(row(BackupStatus::Available).key(), key); + } + + #[test] + fn time_bounds_are_lower_inclusive_and_upper_exclusive() { + let bounds = TimeBounds { + lower: Some(10.0), + upper: Some(20.0), + }; + assert!(bounds.contains(10.0)); + assert!(bounds.contains(19.999)); + assert!(!bounds.contains(20.0)); + assert!(!bounds.contains(9.999)); + assert!(TimeBounds::default().contains(0.0)); + } + + #[test] + fn the_pool_rule_is_twice_the_jobs_plus_ten() { + assert_eq!(pool_size_for_jobs(4), 18); + assert_eq!(pool_size_for_jobs(0), 10); + assert_eq!(pool_size_for_jobs(16), 42); + } + + #[tokio::test] + async fn a_permit_is_taken_without_waiting_and_freed_on_drop() { + let slots = Arc::new(tokio::sync::Semaphore::new(1)); + let held = JobPermit::try_acquire(&slots).expect("a free slot"); + assert!(JobPermit::try_acquire(&slots).is_none()); + drop(held); + assert!(JobPermit::try_acquire(&slots).is_some()); + } + + /// The one refusal text, by bound: a counted bound names its count and + /// its name in one parenthesis; the server's connection slots name no + /// count. Both carry the sentence a client matches on. + #[test] + fn limit_exceeded_message_names_the_bound() { + assert_eq!( + limit_exceeded_message(JobBound::Counted { + name: "[backup] max_concurrent_jobs", + limit: 4, + }), + "Too many backup and restore jobs are running on this server (at most 4 at once, \ + the [backup] max_concurrent_jobs bound); retry later" + ); + assert_eq!( + limit_exceeded_message(JobBound::ServerConnections), + "Too many backup and restore jobs are running on this server (the database server \ + has no connection slot left for the job); retry later" + ); + } +} diff --git a/crates/storage/src/backup_definition.rs b/crates/storage/src/backup_definition.rs index 411b22ab6..b8f014f09 100644 --- a/crates/storage/src/backup_definition.rs +++ b/crates/storage/src/backup_definition.rs @@ -334,7 +334,7 @@ const LEGACY_DEFAULT_THROUGHPUT: i64 = 5; /// A backup as the engine needs it to restore it and to describe it. /// -/// Returned by `BackupEngine::backup_restore_source` for an AVAILABLE backup. +/// Returned by `LegacyBackupReader::restore_source` for an AVAILABLE backup. #[derive(Debug, Clone, PartialEq)] pub struct BackupRestoreSource { pub backup_arn: String, diff --git a/crates/storage/src/config.rs b/crates/storage/src/config.rs index c9be8335c..64ffba9f5 100644 --- a/crates/storage/src/config.rs +++ b/crates/storage/src/config.rs @@ -88,38 +88,39 @@ impl Clone for Box { } /// The warning for a `server.region` outside `[a-z0-9-]`, or `None` when the -/// region is in that class. +/// region is in that class (the `BackupArn` parser's predicate, +/// [`extenddb_core::backup_format::is_region`], so the warning and the parser +/// cannot disagree). /// /// The region is baked into every backup ARN the server issues /// (`arn:aws:dynamodb:::table//backup/`), and the /// `BackupArn` parser accepts a region in `[a-z0-9-]` only, as the service -/// does. A deployment configured outside that class therefore issues ARNs -/// it refuses on `DescribeBackup`, `DeleteBackup`, and -/// `RestoreTableFromBackup` with `ValidationException` "Invalid Request: -/// BackupArn is not valid". The server starts anyway (the setting is -/// pre-existing and the data plane does not depend on it); the operator is -/// told at startup. +/// does. The engine checks the region before forming a backup ARN, so on a +/// deployment configured outside that class `CreateBackup` is refused with a +/// `ValidationException` naming `server.region`; backups taken before the +/// upgrade under such a region keep their stored ARN, which `DescribeBackup`, +/// `DeleteBackup` and `RestoreTableFromBackup` refuse as "Invalid Request: +/// BackupArn is not valid" as before. The server starts anyway (the setting +/// is pre-existing and the data plane does not depend on it); the operator +/// is told at startup. #[must_use] pub fn server_region_warning(region: &str) -> Option { - let well_formed = !region.is_empty() - && region - .bytes() - .all(|b| b.is_ascii_lowercase() || b.is_ascii_digit() || b == b'-'); - if well_formed { + if extenddb_core::backup_format::is_region(region) { return None; } Some(StartupWarning { title: format!( - "server.region {region:?} is outside [a-z0-9-]; the backup ARNs this server \ - issues will not resolve." + "server.region {region:?} is outside [a-z0-9-]; CreateBackup is refused on this \ + server." ), details: vec![ "Every backup ARN carries the configured region, and BackupArn is accepted \ - only with a region in [a-z0-9-], so DescribeBackup, DeleteBackup, and \ - RestoreTableFromBackup answer ValidationException \"Invalid Request: \ - BackupArn is not valid\" for every backup this server creates, and \ - ListBackups refuses the cursor it hands out, so a listing stops at its \ - first page." + only with a region in [a-z0-9-], so CreateBackup answers ValidationException \ + naming server.region (no backup ARN can be formed), and the backups created \ + under this region before the upgrade keep an ARN that DescribeBackup, \ + DeleteBackup, and RestoreTableFromBackup refuse with ValidationException \ + \"Invalid Request: BackupArn is not valid\"; ListBackups refuses a cursor that is \ + such an ARN (a listing of backups created under a valid region pages)." .to_owned(), "Set server.region to a value in [a-z0-9-] (for example us-east-1). Backups \ already created keep the old region in their stored ARN; see the upgrade \ @@ -160,6 +161,15 @@ mod tests { "{region:?}: {text}" ); assert!(text.contains("DescribeBackup"), "{region:?}: {text}"); + assert!( + text.contains("ListBackups refuses a cursor that is such an ARN"), + "{region:?}: {text}" + ); + assert!(!text.contains("first page"), "{region:?}: {text}"); + assert!( + text.contains("CreateBackup answers ValidationException naming server.region"), + "{region:?}: {text}" + ); // The warning agrees with the parser it speaks for. let arn = crate::util::backup_arn(region, "123456789012", "Music", "01489602797149-73d8d5bc"); diff --git a/crates/storage/src/error.rs b/crates/storage/src/error.rs index c936675a5..848a25f00 100755 --- a/crates/storage/src/error.rs +++ b/crates/storage/src/error.rs @@ -63,6 +63,10 @@ pub enum StorageError { /// `BackupInUseException`. #[error("{0}")] BackupInUse(String), + /// A catalog row that already exists, or whose claim a live owner holds + /// (`BackupCatalog::claim_backup`). + #[error("Already exists: {0}")] + AlreadyExists(String), /// A failure that is expected to succeed on retry: I/O errors, pool /// timeouts, SQLITE_BUSY / SQLITE_LOCKED. Exists so queue workers can tell /// "this row can never be applied" (drop it, or the whole queue stalls) diff --git a/crates/storage/src/lib.rs b/crates/storage/src/lib.rs index b3c68fb98..beca72727 100755 --- a/crates/storage/src/lib.rs +++ b/crates/storage/src/lib.rs @@ -9,6 +9,7 @@ pub mod authorization_store; pub mod backend; +pub mod backup_catalog; pub mod backup_definition; pub mod bootstrapper; pub mod config; @@ -35,6 +36,7 @@ pub mod vector_catalog; pub mod vector_lifecycle; pub use backend::{Backend, BackendAlreadySet, backend_name, set_backend, try_backend}; +pub use backup_catalog::{BackupCatalog, JobClaim, JobPermit, LegacyBackupReader}; pub use backup_definition::{BackupRestoreSource, RestoreRequest, RestoreTableOverrides}; pub use snapshot::{SnapshotEngine, SnapshotInfo, TableSnapshot}; @@ -705,116 +707,6 @@ pub trait WorkerStore: Send + Sync { ) -> BoxFuture<'_, Result, StorageError>>; } -/// Backup and point-in-time recovery operations. -pub trait BackupEngine: Send + Sync { - /// Create a backup of a table, snapshotting all items. - fn create_backup( - &self, - account_id: &str, - table_name: &str, - backup_name: &str, - ) -> BoxFuture<'_, Result>; - - /// Describe a backup by ARN, scoped to the owning account. - /// - /// Backups belong to an account, so implementations must match on both - /// `account_id` and `backup_arn` and report a missing backup when the ARN - /// belongs to another account — consistent with `list_backups`, which is - /// already account-scoped. - fn describe_backup( - &self, - account_id: &str, - backup_arn: &str, - ) -> BoxFuture<'_, Result>; - - /// List the account's backups, optionally only those of `table_name`. - /// - /// Returns every matching backup in any order; the engine sorts, applies - /// `Limit` and `ExclusiveStartBackupArn`, and sets - /// `LastEvaluatedBackupArn`. Each summary carries the source table's - /// `TableId`. - fn list_backups( - &self, - account_id: &str, - table_name: Option<&str>, - ) -> BoxFuture<'_, Result, StorageError>>; - - /// Delete a backup by ARN, scoped to the owning account. - /// - /// Same scoping requirement as `describe_backup`: an ARN owned by another - /// account must be reported as missing rather than deleted. - fn delete_backup( - &self, - account_id: &str, - backup_arn: &str, - ) -> BoxFuture<'_, Result>; - - /// The backup's recorded source, for an AVAILABLE backup owned by - /// `account_id`. - /// - /// The engine resolves the restore target and runs every refusal from - /// this, and fills DescribeBackup's `SourceTableFeatureDetails` and - /// `SourceTableDetails.ProvisionedThroughput` from it. `definition` is the - /// stored [`backup_definition::BackupTableDefinition`] (with - /// `time_to_live` and `stream_specification` as captured at - /// CreateBackup), or `None` for a backup recorded without one. - /// - /// # Errors - /// - /// `StorageError::Validation("Backup not found: ")` if the backup is - /// missing, owned by another account, or not AVAILABLE. - /// `StorageError::Unsupported` for a stored definition this build cannot - /// read ([`backup_definition::BackupTableDefinition::from_json`]). - fn backup_restore_source( - &self, - account_id: &str, - backup_arn: &str, - ) -> BoxFuture<'_, Result>; - - /// Restore `request.input.table_name` from `request.backup_arn`. - /// - /// `request.input` is resolved and validated by the engine; create exactly - /// that table, CREATING. In the same atomic step that re-checks the backup - /// is still AVAILABLE, register the restore (source backup ARN, source - /// table ARN, `restore_date_time`) so DescribeTable can report - /// `RestoreSummary`, DeleteBackup refuses the source, and DeleteTable - /// refuses the target while it is being restored. Then return the CREATING - /// description at once. - /// - /// The item copy runs in a detached task (`tokio::spawn`) that owns its - /// own clones, so the request future being dropped (a client timeout) does - /// not stop it. The task flips the target ACTIVE when the copy completes, - /// or removes the target on failure. - /// - /// # Errors - /// - /// Returned synchronously, before the copy starts: - /// `StorageError::Validation("Backup not found: ")` if the backup is - /// not AVAILABLE at registration; `StorageError::TableAlreadyExists(name)` - /// if the name is taken; `StorageError::LimitExceeded` if the backend's - /// restore capacity is exhausted. - fn restore_table_from_backup( - &self, - account_id: &str, - request: RestoreRequest, - ) -> BoxFuture<'_, Result>; - - /// Describe continuous backups / PITR status for a table. - fn describe_continuous_backups( - &self, - account_id: &str, - table_name: &str, - ) -> BoxFuture<'_, Result>; - - /// Update continuous backups (enable/disable PITR). - fn update_continuous_backups( - &self, - account_id: &str, - table_name: &str, - pitr_enabled: bool, - ) -> BoxFuture<'_, Result>; -} - /// Supertrait combining all `DynamoDB` operation traits. /// /// All storage backends must implement this to provide a complete @@ -825,7 +717,8 @@ pub trait StorageEngine: + DataEngine + MetadataEngine + StreamEngine - + BackupEngine + + BackupCatalog + + LegacyBackupReader + SnapshotEngine + WorkerStore + Send @@ -833,13 +726,14 @@ pub trait StorageEngine: { } -// Blanket implementation: any type implementing all 7 traits is a StorageEngine +// Blanket implementation: any type implementing all 8 traits is a StorageEngine impl StorageEngine for T where T: TableEngine + DataEngine + MetadataEngine + StreamEngine - + BackupEngine + + BackupCatalog + + LegacyBackupReader + SnapshotEngine + WorkerStore + Send diff --git a/crates/storage/src/migrations/registry.rs b/crates/storage/src/migrations/registry.rs index 99d581339..21261634c 100644 --- a/crates/storage/src/migrations/registry.rs +++ b/crates/storage/src/migrations/registry.rs @@ -60,6 +60,12 @@ migration_ids! { /// Gives every index table its base key index. IndexTableBaseKey = 4, "index_table_base_key", "give every secondary index table its base key index"; + /// Creates the catalog index of backups (`backup_index`), the restore + /// job fields on the restore registrations, the maintenance claim record + /// and the catalog id. + BackupCatalogIndex = 5, "backup_catalog_index", + "create the backup catalog index, the restore job fields, the maintenance claim \ + record and the catalog id"; } /// Highest id that has shipped in a release. Bodies at or below it are frozen: diff --git a/crates/storage/src/server_components.rs b/crates/storage/src/server_components.rs index f9115db15..bb07b8924 100644 --- a/crates/storage/src/server_components.rs +++ b/crates/storage/src/server_components.rs @@ -96,6 +96,11 @@ pub struct ServerComponentsOptions { /// The ExtendDB release that is starting, recorded in the ledger rows of /// migrations the factory runs (a dev build, an in-memory database). pub release: &'static str, + /// `backup.max_concurrent_jobs`: the backup and restore jobs the + /// instance runs at once. A backend whose pool has to admit every job + /// sizes it from this ([`crate::backup_catalog::pool_size_for_jobs`]); + /// `0` (the default outside a server) asks for no job capacity. + pub max_concurrent_jobs: usize, } /// Factory function type for creating server components. @@ -125,7 +130,7 @@ pub type ServerComponentsFactory = /// /// A `region` the `BackupArn` parser would refuse is logged at warn first /// ([`crate::config::server_region_warning`]): the server still starts, but -/// the operator is told its own backup ARNs will not resolve. +/// the operator is told `CreateBackup` is refused under it. pub async fn create_server_components( config: &(dyn StorageConfig + 'static), region: &str, diff --git a/crates/storage/src/util/catalog_version.rs b/crates/storage/src/util/catalog_version.rs index 252bc7abf..999738b40 100644 --- a/crates/storage/src/util/catalog_version.rs +++ b/crates/storage/src/util/catalog_version.rs @@ -2,10 +2,13 @@ // SPDX-License-Identifier: Apache-2.0 //! The rule every catalog writer applies before stamping its compiled -//! version over the stored one. +//! version over the stored one, and the rule `serve` applies to a stored +//! version string on the backends without a migration ledger. use extenddb_core::version::CatalogVersion; +use crate::migrations::{CatalogStatus, MigrationError}; + /// Refuse to stamp `compiled` over a stored catalog version unless doing so /// cannot hide a newer schema. /// @@ -45,12 +48,92 @@ pub fn check_stored_catalog_version( } } +/// The rule `serve` applies on a backend whose catalog carries a version +/// string in `settings` (MongoDB, Cassandra) rather than a migration ledger: +/// the stored `catalog_version` must be the one this binary writes, else the +/// server does not start. +/// +/// The PostgreSQL and SQLite factories read the ledgers' status instead and +/// refuse a catalog that is behind or ahead; this is the same refusal in the +/// one form those two backends can express. A catalog behind the binary is +/// refused with the command that brings it forward; one ahead of it, or one +/// whose version this binary cannot read, with the advice to run the binary +/// that wrote it. Without the guard a server of this release on an older +/// catalog served table and item operations and failed every backup +/// operation on the collection or column the migration adds, so an operator +/// learned of the pending migration from a client's error. +/// +/// `backend` names the backend in the message. The error carries the exit +/// code `serve` ends with: `Uninitialized` (1, with `extenddb init` advice) +/// when there is no version row, else a refusal (1) with the message. +/// +/// # Errors +/// +/// A [`MigrationError`] for `serve` to print through its catalog-refusal path. +pub fn require_served_catalog_version( + stored: Option<&str>, + expected: &str, + backend: &str, +) -> Result<(), MigrationError> { + let Some(stored) = stored else { + return Err(MigrationError::Status(CatalogStatus::Uninitialized)); + }; + if stored == expected { + return Ok(()); + } + let expected_v = expected.parse::().map_err(|e| { + MigrationError::Refused(format!( + "this binary's expected {backend} catalog version {expected:?} is invalid: {e}" + )) + })?; + match stored.parse::() { + Ok(stored_v) if stored_v < expected_v => Err(MigrationError::Refused(format!( + "The {backend} catalog is at version {stored}; this release serves version \ + {expected}. Stop every server built before this release, run \ + `extenddb migrate --yes` once, then start the server." + ))), + Ok(_) => Err(MigrationError::Refused(format!( + "The {backend} catalog is at version {stored}, newer than this binary's {expected}. \ + Run a binary that expects {stored} or later." + ))), + Err(e) => Err(MigrationError::Refused(format!( + "The stored {backend} catalog version {stored:?} is not a version this binary \ + ({expected}) can read ({e}). Inspect the 'catalog_version' row in the catalog's \ + settings: it was written by a binary this one cannot compare itself with, or \ + edited by hand." + ))), + } +} + #[cfg(test)] mod tests { use super::*; const V4: CatalogVersion = CatalogVersion::new(0, 0, 4); + #[test] + fn serve_accepts_only_the_version_it_writes() { + assert!(require_served_catalog_version(Some("0.0.5"), "0.0.5", "MongoDB").is_ok()); + let behind = require_served_catalog_version(Some("0.0.2"), "0.0.5", "MongoDB") + .unwrap_err() + .to_string(); + assert!(behind.contains("extenddb migrate --yes"), "{behind}"); + assert!(behind.contains("at version 0.0.2"), "{behind}"); + assert!(behind.contains("MongoDB"), "{behind}"); + let ahead = require_served_catalog_version(Some("0.0.6"), "0.0.5", "Cassandra") + .unwrap_err() + .to_string(); + assert!(ahead.contains("newer than this binary"), "{ahead}"); + let unreadable = require_served_catalog_version(Some("five"), "0.0.5", "Cassandra") + .unwrap_err() + .to_string(); + assert!(unreadable.contains("cannot compare"), "{unreadable}"); + assert!(matches!( + require_served_catalog_version(None, "0.0.5", "MongoDB"), + Err(MigrationError::Status(CatalogStatus::Uninitialized)) + )); + } + #[test] fn absent_older_and_equal_proceed() { assert!(check_stored_catalog_version(None, V4).is_ok()); diff --git a/crates/storage/src/util/mod.rs b/crates/storage/src/util/mod.rs index bb2493827..ff4d2442b 100755 --- a/crates/storage/src/util/mod.rs +++ b/crates/storage/src/util/mod.rs @@ -22,7 +22,7 @@ pub use backup_id::{ backup_id_epoch_millis, check_backup_arn, format_backup_id, is_backup_id, is_backup_resource_arn, new_backup_id, parse_backup_arn, }; -pub use catalog_version::check_stored_catalog_version; +pub use catalog_version::{check_stored_catalog_version, require_served_catalog_version}; pub use key::SortKeyValue; pub use key::{ composite_pk_to_text, effective_attribute_definitions, encode_netstring_composite, diff --git a/docs/design/02-high-level-design.md b/docs/design/02-high-level-design.md index 3a680e615..a24811a0c 100755 --- a/docs/design/02-high-level-design.md +++ b/docs/design/02-high-level-design.md @@ -376,12 +376,15 @@ Total connection footprint on the PostgreSQL server: pool because its ownership lock is session-scoped - one connection while a schema migration runs, pinned outside the pools so its transaction-scoped lock cannot move between backends - -So the total is `2 * pool_size + catalog_pool_size`, plus builds and migrations. -There is no separate poller pool: the log-level poller borrows from the management -pool like every other background poller. `max_connections` is per cluster rather -than per database, so both `pool_size` pools draw on the same budget: the defaults -need 60 of PostgreSQL's default 100. +- one connection per running backup or restore job, at most `[backup] + max_concurrent_jobs` (default 4), opened outside every pool because the job's + claim is a session-level advisory lock + +So the total is `2 * pool_size + catalog_pool_size + max_concurrent_jobs`, plus +builds and migrations. There is no separate poller pool: the log-level poller +borrows from the management pool like every other background poller. +`max_connections` is per cluster rather than per database, so both `pool_size` +pools draw on the same budget: the defaults need 64 of PostgreSQL's default 100. ### 6.3 Row-Level Locking diff --git a/docs/design/13-storage-mongodb.md b/docs/design/13-storage-mongodb.md index 5498e6cc0..a80f17689 100644 --- a/docs/design/13-storage-mongodb.md +++ b/docs/design/13-storage-mongodb.md @@ -835,9 +835,13 @@ construction. ### 5.10 Catalog version check `read_catalog_version` returns the `settings.catalog_version` value; -`expected_catalog_version` returns the compiled-in `0.0.2`. The bin -layer compares them on `extenddb serve` startup — same pattern as the -PostgreSQL backend. +`expected_catalog_version` returns the compiled-in `CATALOG_VERSION` +(`0.0.5`, one constant in `bootstrapper.rs`, which the last migration +step writes). `extenddb migrate` compares them and runs the pending +steps, and the server component factory refuses to serve a catalog at +another version (`require_served_catalog`, naming `extenddb migrate +--yes`), the same refusal the PostgreSQL backend reads from its +migration ledger. ## 6. Crate Structure diff --git a/docs/design/15-backup-restore.md b/docs/design/15-backup-restore.md index ebaf3318c..477e344c1 100644 --- a/docs/design/15-backup-restore.md +++ b/docs/design/15-backup-restore.md @@ -98,7 +98,7 @@ The invariants and state machines every later PR honours, by lettered part; the (d) Identities. The ARN encodes partition, region, account, table, and backup id, and the key prefix is derived from account, table and id deterministically by `BackupArn::store_prefix`, under the configured store's prefix, and no row stores it, so restore, sync, purge and the reconciler derive it the same way and `describe_backup(arn)` works from the store alone after a sync. The engine, the writer and sync all parse and format ARNs through one implementation, `BackupArn` in `extenddb_core::backup_format` (#394); the engine keeps #384's region handling: it resolves a backup under the ARN as written and under the ARN re-spelled in its own region, and refuses a partition other than its own. `backup_creation_date_time` and `created_at` are the id's instant, so the ARN, the manifest, the API's reported time and the ListBackups cursor agree by construction. One row is addressed by `BackupKey` (`account_id`, `backup_id`, `backup_arn`): PostgreSQL, SQLite and MongoDB key `backup_index` by (`account_id`, `backup_id`), Cassandra keys `backups_by_arn` by (`account_id`, `backup_arn`); no method takes a bare backup id, and the engine treats a row whose `account_id`, `table_name` or `backup_arn` (after that re-spelling) differs from the request's as absent. One target is addressed by `RestoreTargetKey` (`account_id`, `table_name`, `table_id`), the table id guarding against a table re-created under the name. -(e) The claim. There is no single-instance lease on PostgreSQL or MongoDB, so each job holds a claim that both excludes a second owner and lets a successor detect a dead one. Every backend sets `owner_generation` (a fresh random value) at claim time, and every catalog write by the job is `WHERE owner_generation = $gen`, so a job that lost its claim cannot publish over a successor's work. No row ever exists unowned: on every backend the claim precedes the row's insert or is the insert itself (5.1 step 2); what the claim is per backend is PR 6's section. Who is dead: on SQLite every CREATING row at startup, and at runtime a row whose in-process slot no task holds; on PostgreSQL a row whose advisory lock is free; on MongoDB and Cassandra a row whose `heartbeat_at` is older than `stale_job_after_secs` (default 300). Staleness is always measured on the database's clock, never on an application host's. The reconciler takes the same claim as a worker, with a fresh `owner_generation` that fences the dead job's generation out, leaves `heartbeat_at` untouched (a refreshed heartbeat would restart the stale window the prefix deletion waits for), writes AVAILABLE or FAILED at once, both conditioned on `status = CREATING`, and never continues the upload. A job whose conditional write matches no row, or whose claim connection errors, performs no further store or table operation, deletes of a prefix or of a restore target included (5.1 step 5, 5.3 step 7); on PostgreSQL a restore target is resumed or deleted only once the row's `heartbeat_at` is older than `stale_job_after_secs`, since a free advisory lock shows that the owner's catalog session ended, not that its process stopped (5.3 step 7). +(e) The claim. There is no single-instance lease on PostgreSQL or MongoDB, so each job holds a claim that both excludes a second owner and lets a successor detect a dead one. Every backend sets `owner_generation` (a fresh random value) at claim time, and every catalog write by the job is `WHERE owner_generation = $gen`, so a job that lost its claim cannot publish over a successor's work. No row ever exists unowned: on every backend the claim precedes the row's insert or is the insert itself (5.1 step 2); what the claim is per backend is PR 6's section. Who is dead: on SQLite every CREATING row at startup, and at runtime a row whose in-process slot no task holds; on PostgreSQL, MongoDB and Cassandra a row whose `heartbeat_at` is older than `stale_job_after_secs` (default 300), the one liveness rule the three share. On PostgreSQL the advisory lock is the claim's proof against a second owner, not the liveness test: a free lock shows that the owner's catalog session ended, not that its process stopped (a session a managed server dropped leaves a process whose data writes carry no generation), and the owner itself aborts at its next heartbeat inside the stale window, so the reconciler reads the heartbeat alone and never probes the lock. Staleness is always measured on the database's clock, never on an application host's. The reconciler takes the same claim as a worker, with a fresh `owner_generation` that fences the dead job's generation out, leaves `heartbeat_at` untouched (a refreshed heartbeat would restart the stale window the prefix deletion waits for), writes AVAILABLE or FAILED at once, both conditioned on `status = CREATING`, and never continues the upload. A job whose conditional write matches no row, or whose conditional write errors with an unknown outcome, performs no further store or table operation, deletes of a prefix or of a restore target included (5.1 step 5, 5.3 step 7). A FAILED `restore_jobs` row whose target table is still CREATING (the FAILED write landed and the target's removal did not, whether the job's own or the reconciler's) is the FAILED-target duty: the reconciler lists it once the row's `completed_at` (or, where none was stamped, its `heartbeat_at`) is older than `stale_job_after_secs`, so a writer still on its own failure path finishes it, and removes the target through `remove_restore_target` with no second FAILED write; PR 6's reconciler runs it for every kind of record, since on this rule the kind does not matter (PR 8 adds the `target_not_empty` exception). On Cassandra the removal is itself two writes, a lightweight transaction claiming the target CREATING to DELETING and then the drop, so the duty also lists a FAILED row whose target is DELETING with its registration still present, on the same staleness rule, and `remove_restore_target` resumes the removal from the drop; PostgreSQL and SQLite remove in one transaction and MongoDB deletes the document first, so neither leaves that state. (f) The maintenance claim. `backup purge`, `backup remove-target` and `backup sync` with `--move-store`, `--forget-store`, `--forget-backup`, `--prune-missing` or `--adopt-store` change rows or bytes that jobs on any instance may be using, and a job permit (5.1 step 1) is per instance, so these commands take a catalog-level maintenance claim first (PR 10's section: the claim per backend and the SQLite serve lock). Every backup and restore job checks the claim after its own claim and row exist (5.1 step 2, 5.3 step 4) and refuses while it is held; while a claim is held the reconciler runs only the duties that resolve the states the command waits on (DELETING rows, rows whose owner is dead, the FAILED-target duty) and skips every other duty; the command waits until no CREATING backup row with a live owner, no DELETING row of the configured store and no RUNNING `restore_jobs` row remains, and refuses and releases the claim if they have not gone within `2 * stale_job_after_secs`, which it prints; a second maintenance command while one holds the claim fails at once. Because the job checks after its row exists and the command checks rows after its claim exists, one of the two always sees the other. @@ -106,8 +106,8 @@ The invariants and state machines every later PR honours, by lettered part; the (h) Trust boundary of sync: a principal that can write under an account's prefix can forge a backup for that account. Checksums do not help, they are forgeable by the same principal. The deployment guide states that write access to an account's prefix is backup-administrator authority for that account, and gives per-account IAM prefix policies. A server-side MAC over manifests is not in v1; if hostile writers with bucket access are in scope for a deployment, the operator restricts write access rather than relying on ExtendDB to detect forgeries. -(i) Authority: the catalog migration PR 6 adds (PostgreSQL 005, SQLite schema, the MongoDB `backup_index` collection, `tables` fields and version step, and Cassandra V007, each writing -catalog 0.0.6) +(i) Authority: the catalog migration PR 6 adds (migration id 5 on PostgreSQL and SQLite; Cassandra V006 and MongoDB's step `002_backup_catalog_index`, both writing +catalog 0.0.5) the `BackupCatalog` trait of section 3 that each backend implements, and the engine's `backup` module (`crates/engine/src/backup/`), which binds it; this section is the specification they implement. The MongoDB version step and the mixed-version rule are PR 6's section. ### 5. Operations @@ -116,10 +116,10 @@ Per operation, the contract: the request checks in their order, the error each r #### 5.1 CreateBackup -1. Validate the request as #384 does and in its order, every check before the permit: `TableName` resolved (a table ARN is accepted), the table exists and is ACTIVE for the account (`TableNotFoundException`, "Table not found: ", otherwise, whether or not a permit is free), the table name fits the store prefix (`BackupArn`: a name over 255 bytes is refused with a ValidationException naming the limit). #384's `refuse_backup_of_vector_table` is not among the off-host path's checks (the manifest records vector indexes, section 1); the engine keeps the refusal in front of `LegacyBackupReader::create`, the path of every backend without `[backup.store]` and of Cassandra always (section 7), and the tests that change are PR 8's and PR 9a's sections. A second CreateBackup on a table that already has a CREATING backup is allowed and takes its own snapshot; the `BackupInUseException` the service models on CreateBackup is not raised. Then reserve a job slot: `max_concurrent_jobs` is a per-instance semaphore the request `try_acquire`s (the maintenance claim of section 4 is the cross-instance exclusion), and the permit lives with the job; if none is free return `LimitExceededException` before opening anything, because a queued snapshot would hold a PostgreSQL transaction or a MongoDB snapshot session while it waited; RestoreTableFromBackup and the legacy create take a permit from the same semaphore and return the same error (5.3 step 1, section 3). The default is 4 (section 7), so one running restore does not turn every other backup operation on the instance into `LimitExceededException`. +1. Validate the request as #384 does and in its order, every check before the permit: `TableName` resolved (a table ARN is accepted), the table exists and is ACTIVE for the account (`TableNotFoundException`, "Table not found: ", otherwise, whether or not a permit is free), the server's region and the table name fit a backup ARN (`check_backup_arn_fits`, each refusal naming its cause: a `server.region` outside `[a-z0-9-]` is a ValidationException naming the setting, since no backup ARN can be formed for any table under it; a name over 255 bytes or made only of dots, the store-prefix rules, is one naming the rule, applied before the ARN is formed so the limit is named whatever the name's length). #384's `refuse_backup_of_vector_table` is not among the off-host path's checks (the manifest records vector indexes, section 1); the engine keeps the refusal in front of `LegacyBackupReader::create`, the path of every backend without `[backup.store]` and of Cassandra always (section 7), and the tests that change are PR 8's and PR 9a's sections. A second CreateBackup on a table that already has a CREATING backup is allowed and takes its own snapshot; the `BackupInUseException` the service models on CreateBackup is not raised. Then reserve a job slot: `max_concurrent_jobs` is a per-instance semaphore the request `try_acquire`s (the maintenance claim of section 4 is the cross-instance exclusion), and the permit lives with the job; if none is free return `LimitExceededException` before opening anything, because a queued snapshot would hold a PostgreSQL transaction or a MongoDB snapshot session while it waited; RestoreTableFromBackup and the legacy create take a permit from the same semaphore and return the same error (5.3 step 1, section 3). The default is 4 (section 7), so one running restore does not turn every other backup operation on the instance into `LimitExceededException`. 2. Open the table snapshot (`SnapshotEngine::open_table_snapshot`) in the request. The pin happens during the request, so the backup holds every write acknowledged before the request was received, which is stricter than the service's one-minute window. Mint the backup id from the millisecond the open returned, just after the pin (`format_backup_id`), and the owner generation, then, through `BackupCatalog::claim_backup` (section 3), insert the catalog row with status CREATING, `created_at` = the id's instant, `owner_generation`, `heartbeat_at` = the database's `now()`, and `source_table_details_json` taken from the snapshot's description; then check the maintenance claim (section 4): if it is held, set the row FAILED with reason `maintenance`, release the snapshot, the lock and the permit, and return `LimitExceededException`; otherwise return `BackupDetails` with BackupStatus CREATING. Everything from the open to the hand-off in step 3 runs in a detached task whose result the request awaits, so a client that disconnects mid-request cannot leave a row whose owner was never spawned; if the row insert fails the snapshot is dropped, the lock released, the permit returned, and nothing was written. 3. The claim: section 4(e) states the rules; what the claim is on each backend, the claim connection's settings and the hand-off to the worker are PR 6's and PR 8's sections. -4. The job streams items from the snapshot into gzip data files through `DataFileEncoder`; a file closes once its `uncompressed_bytes` reaches `data_file_target_bytes` (default 64 MiB), so a file exceeds the knob by less than one line, at most `MAX_LINE_BYTES` (32 MiB, section 1), which is also the knob's minimum. Every item is checked, before it is encoded, with the item validator PR 8 adds to `extenddb-core` (`extenddb_core::validation::validate_restorable_item`, with this instance's `max_item_size_bytes`; the restore binds the same function, 5.3 step 5), so no backup that no restore can load is reported AVAILABLE; that is a regression from main for a table that already holds a value #618 let in (Drawbacks), and the reason reaches the operator only (`backup list --failed` and the log); the client sees CREATING and then BackupNotFound. The heartbeat is written on a timer (`heartbeat_interval_secs`, default 15) for as long as the job task is alive, independent of upload progress; before each put the job checks that its latest heartbeat confirmed the claim within `heartbeat_interval_secs` and starts no put otherwise; a heartbeat that does not confirm the claim, or that has not returned within `heartbeat_interval_secs`, aborts the job without further store writes, deletes included. `stale_job_after_secs` must exceed several heartbeat intervals; startup refuses a value under `3 * heartbeat_interval_secs`, which is what makes that detection precede a reconciler's deletion. After the last file: write `manifest-files.json`, `manifest-summary.json`, `extenddb-manifest.sha256` (the order #394's `BackupDocuments::build` doc gives); re-check the claim; then write `extenddb-manifest.json` and update the row to AVAILABLE with size and count (`JobClaim::set_status(CREATING, AVAILABLE)`, conditional on the generation and on `status = CREATING`). An update whose outcome is unknown is followed by a re-read of the row, not treated as a failure; one that matches zero rows ends the job with no store access. Store writes are not fenced by the generation; the re-check narrows the window in which a fenced-out job can still write a manifest, and the FAILED tombstone (section 4) closes what remains: sync never adopts a prefix whose row is FAILED, and the reconciler never deletes a prefix while the row's `heartbeat_at` is younger than `stale_job_after_secs` on the catalog database's clock, so a late manifest from a fenced-out job lands under a tombstone that sync will not adopt. +4. The job streams items from the snapshot into gzip data files through `DataFileEncoder`; a file closes once its `uncompressed_bytes` reaches `data_file_target_bytes` (default 64 MiB), so a file exceeds the knob by less than one line, at most `MAX_LINE_BYTES` (32 MiB, section 1), which is also the knob's minimum. Every item is checked, before it is encoded, with the item validator PR 8 adds to `extenddb-core` (`extenddb_core::validation::validate_restorable_item`, with this instance's `max_item_size_bytes`; the restore binds the same function, 5.3 step 5), so no backup that no restore can load is reported AVAILABLE; that is a regression from main for a table that already holds a value #618 let in (Drawbacks), and the reason reaches the operator only (`backup list --failed` and the log); the client sees CREATING and then BackupNotFound. The heartbeat is written on a timer (`heartbeat_interval_secs`, default 15) for as long as the job task is alive, independent of upload progress; before each put the job checks that its latest heartbeat confirmed the claim within `heartbeat_interval_secs` and starts no put otherwise; a heartbeat that does not confirm the claim, or that has not returned within `heartbeat_interval_secs`, aborts the job without further store writes, deletes included. `stale_job_after_secs` must exceed several heartbeat periods; startup refuses a value under three times the period the jobs run at, which until the knob reaches the jobs is the larger of `heartbeat_interval_secs` and the 15 s constant `extenddb_storage::backup_catalog::JOB_HEARTBEAT_INTERVAL` every backend's heartbeat loop reads (so 45 s at the least), which is what makes that detection precede a reconciler's deletion. After the last file: write `manifest-files.json`, `manifest-summary.json`, `extenddb-manifest.sha256` (the order #394's `BackupDocuments::build` doc gives); re-check the claim; then write `extenddb-manifest.json` and update the row to AVAILABLE with size and count (`JobClaim::set_status(CREATING, AVAILABLE)`, conditional on the generation and on `status = CREATING`). An update whose outcome is unknown is followed by a re-read of the row, not treated as a failure; one that matches zero rows ends the job with no store access. Store writes are not fenced by the generation; the re-check narrows the window in which a fenced-out job can still write a manifest, and the FAILED tombstone (section 4) closes what remains: sync never adopts a prefix whose row is FAILED, and the reconciler never deletes a prefix while the row's `heartbeat_at` is younger than `stale_job_after_secs` on the catalog database's clock, so a late manifest from a fenced-out job lands under a tombstone that sync will not adopt. 5. On any failure, including `SnapshotExpired`: set the row FAILED with the reason, conditional on the generation and on `status = CREATING`, so a job whose AVAILABLE update committed with its response lost cannot turn a backup a client has seen AVAILABLE into a deleted one. Only if that write reports exactly one row matched does the job stop its uploads, await its in-flight puts and `delete_prefix` the partial backup (which on S3 also aborts any upload still open under it); a write that matches zero rows, or any error on the claim connection, ends the job with no further store access of any kind, because the holder of the claim owns the prefix. If the delete fails the reconciler retries it (section 4). A backup never resumes from a cursor position, so a lost snapshot means a new CreateBackup. DescribeBackup returns BackupNotFound; ListBackups omits it. 6. Shutdown: the snapshot lives in the process and dies with it, so a cancelled job can never complete. On cancellation the worker runs step 5 with reason `shutdown`, and the server's shutdown awaits each job's failure path within the drain window before the runtime stops: runtime shutdown sends no abort for a put still in flight (section 2), so step 5's `delete_prefix` is what sweeps its upload. If the catalog is unreachable it leaves the row CREATING for the reconciler; if only the store is unreachable the row is FAILED and the reconciler deletes the prefix later. Crash: on startup and every reconciler cycle, a CREATING row is resolved when its owner is dead, by the test and the claim of section 4(e); the prefix of a FAILED row is deleted only once `heartbeat_at` is older than `stale_job_after_secs`, the delay step 4 relies on. A restarted process cannot finish a CREATING backup; AVAILABLE from an existing manifest or FAILED are the only two outcomes. @@ -130,7 +130,7 @@ DeleteBackup on a CREATING row returns `BackupInUseException`, as on the service #### 5.3 RestoreTableFromBackup -The request runs steps 1, 2 and 4 and returns CREATING in milliseconds plus the manifest parse, as DynamoDB does; the job runs steps 3, 5 and 6. The request's own awaits are the checks of step 1 in their order, the permit, the manifest read and the validation of step 2; the step 4 transaction and the spawn of the job run together in a detached task whose result the request awaits, so a client disconnect cannot commit the table and the RUNNING row without an owner. The table exists from the response onwards, so no second restore or CreateTable can take the name while files download, but it accepts no reads or writes until every file has verified and loaded. +The request runs steps 1, 2 and 4 and returns CREATING in milliseconds plus the manifest parse, as DynamoDB does; the job runs steps 3, 5 and 6. On MongoDB the request's wait for step 4 is bounded (`RESTORE_REGISTRATION_TIMEOUT`, 60 s, against a connection pool that has stopped handing out connections): past it the request answers `InternalServerError` while the detached task goes on, so the target can appear after the error, holding the backup in use and the name until the copy ends, and DescribeTable is where the client finds it (the differences doc's row on how `RestoreTableFromBackup` runs states it for operators). The request's own awaits are the checks of step 1 in their order, the permit, the manifest read and the validation of step 2; the step 4 transaction and the spawn of the job run together in a detached task whose result the request awaits, so a client disconnect cannot commit the table and the RUNNING row without an owner. The table exists from the response onwards, so no second restore or CreateTable can take the name while files download, but it accepts no reads or writes until every file has verified and loaded. 1. Check the request in the order main uses and the differences doc records, every refusal that needs no store access before the permit: the overrides' shape (below); the ARN with `BackupArn` (the account must equal the caller's before any store access); the target name (`TableAlreadyExistsException` when a table of that name exists, `TableInUseException` when it is a CREATING restore target); the backup row, read by the ARN's account and backup id and accepted only when its `account_id`, `table_name` and `backup_arn` (after the region re-spelling of section 4) equal the request's, or, if there is no such row, resolved through the `LegacyBackupReader` (section 3): an ARN with neither is `BackupNotFoundException`, so an ARN of account B that carries the id of account A's backup is BackupNotFound and never A's row, a CREATING row returns `BackupInUseException`, a backup in use (the rule of 5.2) returns `BackupInUseException` naming that table, a DELETING, DELETED or FAILED row returns `BackupNotFoundException`, a row of another store returns the ValidationException of section 4 and a row of an unhealthy configured store the one of section 7, none of them reading the store; then, last, the job permit (`LimitExceededException` when none is free). Only then read `extenddb-manifest.sha256` and `extenddb-manifest.json`, verify the sidecar against the manifest bytes, then `format_version`, the ARN and its agreement with `source_table`, and the data-file list rules (`ExtenddbManifest::from_json_bytes`). A manifest missing behind an AVAILABLE row returns `BackupNotFoundException` and logs the row for `backup sync --prune-missing`. 2. Validate that the target backend supports every feature in `source_table` that a restore reproduces (a non-empty `vector_indexes` into a backend without vector support is the one case, section 1); refuse before creating anything otherwise. Unlike #384 (`ensure_single_part_base_key`), this path restores multi-part-key tables (as `unordered` backups, section 1): `bulk_load` keys each row and derives its index rows with the functions the backend's item write path uses (section 3), so a restored multi-part-key table is laid out as a natively created one. @@ -138,7 +138,7 @@ The request runs steps 1, 2 and 4 and returns CREATING in milliseconds plus the 4. Create the target table and the `restore_jobs` row (RUNNING, with the job's claim, `kind = manifest` and the store the backup row names; section 4) in one catalog transaction (PostgreSQL and SQLite), one session transaction (MongoDB) or, on Cassandra, one `INSERT ... IF NOT EXISTS` of the `tables` row, whose columns are the job row, so no crash leaves a CREATING table without a job row. If another CreateTable or restore took the name between steps 1 and 4, the request returns `TableAlreadyExistsException` (or `TableInUseException` when the winner is itself a CREATING restore target) and the job row insert rolls back with it; CreateTable keeps `ResourceInUseException`. The registration (`BackupCatalog::claim_restore`) re-checks inside its transaction, under the backup row's lock that DeleteBackup's `start_backup_delete` takes (5.2), that the backup row is still AVAILABLE and that no restore target registered from it is CREATING: a row that is no longer AVAILABLE returns `BackupNotFoundException`, a backup in use returns `BackupInUseException`, and no target is left. After the transaction commits the job checks the maintenance claim (section 4); if it is held, the job runs step 7 with reason `maintenance` and the request returns `LimitExceededException`. Otherwise return `TableDescription` with TableStatus CREATING. Overrides are applied here (below). A restore the reconciler resumes is a job and takes a `max_concurrent_jobs` permit like a new one; when none is free the row is skipped until the next cycle. 5. (Job, after step 3.) Per data file: read the verified file through #394's `ExtenddbManifest::reader_prehashed`, apply the per-file key-range check of section 1 when the backup is key-ordered, validate every item with PR 8's item validator with the target's limits and fail the restore on the first invalid item, then write the file through `bulk_load(target_table_id, claim, file_index, start_at, items)` (section 3), with `start_at` the recorded `items_applied` of a file a resume stopped in and 0 otherwise. Each backend writes base rows and index rows together and records progress so that a crash cannot leave an item applied twice: in one transaction with the data on PostgreSQL and SQLite, and by idempotent re-application of the one unrecorded file or batch on MongoDB and Cassandra (PR 9a's section: the batch sizes and progress records). The restored table has no streams and no TTL, so there is nothing to bypass. 6. (Job.) After all files, and for an unordered backup after the count of section 1 matched, the job flips the target to ACTIVE conditional on the table still being CREATING as a restore target and on the job's `owner_generation`; on PostgreSQL and SQLite the table and every index of the target flip in one catalog transaction, which also marks the `restore_jobs` row DONE; on Cassandra the index rows are created ACTIVE with the table and the lightweight transaction on the `tables` row flips the table and marks the job row DONE. A Query on each GSI, and SearchVectors where the target supports it, therefore sees every restored item as soon as the target reports ACTIVE. MongoDB's sequence differs, because the existing backfill worker, not the job, promotes the indexes: the job marks the `restore_jobs` row DONE (conditioned on its generation and on RUNNING) and sets `restore_backfill_pending` in one session transaction, and the job task ends there; the worker promotes each index and then flips the table to ACTIVE, conditioned on `restore_backfill_pending` and on CREATING, not on the job's generation; and the reconciler never treats a row with `restore_backfill_pending` as a dead RUNNING job (the row is DONE). DeleteTable on a restore target is refused with `ResourceInUseException`, as #384 does and as the service does, which is why the job and the reconciler remove a target through `remove_restore_target` (step 7); there is no user-facing way to cancel a running restore. UpdateTable on a CREATING table is refused as today. -7. Failure (verification or load): the job marks the `restore_jobs` row FAILED with the reason, conditional on its generation. Only if that write matched exactly one row does the job remove the partially created table, through `BackupCatalog::remove_restore_target` (section 3), keyed by the target's account, name and table id and conditional on the table still being a CREATING restore target, which also deletes its progress rows, never through the DeleteTable API; a write that matches zero rows ends the job with only its spool removed, because the claim holder owns the table. DescribeTable then returns ResourceNotFound. Crash mid-restore: the reconciler claims RUNNING `restore_jobs` rows whose owner is dead (section 4(e)) and resumes from the recorded progress, reading the store the job row records if it is the configured store, failing the restore and naming `backup sync --move-store` if it is not, and resuming nothing while the configured store is unhealthy; on PostgreSQL only once the row's `heartbeat_at` is older than `stale_job_after_secs`, the delay the prefix deletion already waits, since a free advisory lock shows that the owner's catalog session ended, not that its process stopped, and the restore's data writes carry no generation; the owner aborts at its next heartbeat, inside that window. A legacy RUNNING restore whose owner is dead is marked FAILED and its target removed, never resumed (section 3). Shutdown: a restore job stops at its next batch without writing a status, so step 7 does not run; its row stays RUNNING with its spool and its progress, for a successor once the owner is dead by its backend's rule. +7. Failure (verification or load): the job marks the `restore_jobs` row FAILED with the reason, conditional on its generation. Only if that write matched exactly one row does the job remove the partially created table, through `BackupCatalog::remove_restore_target` (section 3), keyed by the target's account, name and table id and conditional on the table still being a CREATING restore target, which also deletes its progress rows, never through the DeleteTable API; a write that matches zero rows ends the job with only its spool removed, because the claim holder owns the table. DescribeTable then returns ResourceNotFound. Crash mid-restore: the reconciler claims RUNNING `restore_jobs` rows whose owner is dead (section 4(e)) and resumes from the recorded progress, reading the store the job row records if it is the configured store, failing the restore and naming `backup sync --move-store` if it is not, and resuming nothing while the configured store is unhealthy; on PostgreSQL, as on MongoDB and Cassandra, once the row's `heartbeat_at` is older than `stale_job_after_secs`, the delay the prefix deletion already waits (4(e): the advisory lock is the claim's exclusion, not the liveness rule; the owner aborts at its next heartbeat, inside that window). A legacy RUNNING restore whose owner is dead is marked FAILED and its target removed, never resumed (section 3). Shutdown: a restore job stops at its next batch without writing a status, so step 7 does not run; its row stays RUNNING with its spool and its progress, for a successor once the owner is dead by its backend's rule. Restore overrides. #384 applies every override the operation models as the service applies it (`crates/engine/src/backup/overrides.rs`) and refuses only `VectorIndexOverride`, which the botocore model this project targets does not carry, with a typed ValidationException. The v1 contract keeps exactly that: the new path applies the overrides at step 4 to the manifest's `source_table` before the target is created, through the same `overrides` module, and `VectorIndexOverride` keeps the refusal. The differences and limits docs' override rows stay as they are. @@ -160,10 +160,10 @@ Export gains the store as a target and uses `open_table_snapshot`, which fixes t [backup] data_file_target_bytes = 67108864 # uncompressed bytes per data file; a file closes once it reaches this, so it exceeds it by less than one line (5.1 step 4); minimum 32 MiB, the line cap heartbeat_interval_secs = 15 # job heartbeat timer (5.1 step 4); #384's restore heartbeat interval -stale_job_after_secs = 300 # heartbeat age after which a MongoDB or Cassandra job is dead, and the delay before a reconciler deletes a dead job's prefix on every backend; a new value (#384's restore grace is 60 s) +stale_job_after_secs = 300 # heartbeat age after which a PostgreSQL, MongoDB or Cassandra job is dead (4(e)), and the delay before a reconciler deletes a dead job's prefix on every backend; at least three times the period the jobs run at, max(heartbeat_interval_secs, JOB_HEARTBEAT_INTERVAL), so 45 at the least until the knob reaches the jobs (5.1 step 4); a new value (#384's restore grace is 60 s) failed_row_retention_secs = 604800 # FAILED tombstones kept for the operator, 7 days reconcile_interval_secs = 60 # reconciler cycle (section 4) -max_concurrent_jobs = 4 # backup and restore jobs running at once on this instance, legacy creates and restores included; a further CreateBackup or RestoreTableFromBackup is LimitExceeded +max_concurrent_jobs = 4 # backup and restore jobs running at once on this instance, legacy creates and restores included; a further CreateBackup or RestoreTableFromBackup is LimitExceeded, the text naming the bound it met; at most 1024; on PostgreSQL, pool_size and catalog_pool_size are raised to at least 2 * max_concurrent_jobs + 10 (below) spool_path = "" # restore download area for an S3 store; defaults beside the database file on SQLite, required on the other backends ``` @@ -171,7 +171,7 @@ spool_path = "" # restore download area for an S3 store; d On SQLite (file-backed) a missing `[backup]` section is equivalent to `kind = "filesystem"` with `path` = `backups/` beside the database file; a `validate` failure of that defaulted root is a logged warning that leaves the store unconfigured, not a startup error, so an upgrade never stops a server that never configured backups, while an explicitly configured root that fails `validate` is still a startup error. On PostgreSQL and MongoDB a missing `[backup.store]` leaves the off-host path unconfigured and startup logs that; `kind = "disabled"` leaves it unconfigured on purpose on any backend. Without a configured store, CreateBackup on every backend uses the legacy in-catalog create (`LegacyBackupReader::create`, section 3) and logs a deprecation warning naming `[backup.store]` on each call, until PR 14 removes that path; after it CreateBackup without a configured store returns a ValidationException naming `[backup.store]`. Legacy backups stay readable, listable, restorable and deletable either way. `sqlite-memory` (any `:memory:` database path) has no database file and so no default root: CreateBackup there takes the off-host path only with `[backup.store]` set and the legacy create without it. Cassandra has no `SnapshotEngine`, so it uses the legacy create even with a store configured (section 3); a configured store serves restores of off-host backups into Cassandra and nothing else there. -Memory per running job, which operators size hosts from: a backup holds the upload window (`upload_part_size_bytes * max_concurrent_uploads` on an S3 store, one write buffer on a filesystem store), the read path, the gzip window and the manifest's data-file entries until the manifest is written; with the byte budget PR 8 adds to the PostgreSQL and SQLite streams (`SNAPSHOT_FETCH_BYTES`, a constant beside `DEFAULT_FETCH_BATCH`, not a `[backup]` key in v1) the read path holds at most one page, the budget plus one row of text, plus its decoded form, and neither term grows with `data_file_target_bytes` or with the table, except on `sqlite-memory`, whose open copies the whole table (section 3). A restore holds one line's decode plus one insert batch plus the parsed manifest for the whole job; neither grows with the table. Each running backup also holds one connection from the PostgreSQL data pool for its duration, and on the separate layout one catalog-pool connection while its snapshot opens (section 3); the backend admits at most half the pool as open snapshots and refuses more with `LimitExceeded`; a running restore holds one data connection and has no slot of its own after PR 6; and each PostgreSQL job holds one dedicated claim connection outside the pool. `pool_size` sizes both pools, and the rule is `pool_size >= max(MIN_POOL_SIZE, 2 * max_concurrent_jobs + 10)`: twice the jobs, so the half-pool bound admits every job `max_concurrent_jobs` allows, plus ten for requests; 18 at the defaults, under the default `pool_size` of 20. It applies whether or not `[backup.store]` is configured, because legacy restores are jobs too. Startup raises a smaller `pool_size` to that value with a warning naming both numbers, as it already raises a `pool_size` under `MIN_POOL_SIZE` (10) today. +Memory per running job, which operators size hosts from: a backup holds the upload window (`upload_part_size_bytes * max_concurrent_uploads` on an S3 store, one write buffer on a filesystem store), the read path, the gzip window and the manifest's data-file entries until the manifest is written; with the byte budget PR 8 adds to the PostgreSQL and SQLite streams (`SNAPSHOT_FETCH_BYTES`, a constant beside `DEFAULT_FETCH_BATCH`, not a `[backup]` key in v1) the read path holds at most one page, the budget plus one row of text, plus its decoded form, and neither term grows with `data_file_target_bytes` or with the table, except on `sqlite-memory`, whose open copies the whole table (section 3). A restore holds one line's decode plus one insert batch plus the parsed manifest for the whole job; neither grows with the table. Each running backup also holds one connection from the PostgreSQL data pool for its duration, and on the separate layout one catalog-pool connection while its snapshot opens (section 3); the backend admits at most half the pool (after the raise below) as open snapshots and refuses more with `LimitExceeded`; a running restore holds one data connection and has no slot of its own after PR 6; and each PostgreSQL job holds one dedicated claim connection outside the pool. `pool_size` sizes both pools, and the rule is `pool_size >= max(MIN_POOL_SIZE, 2 * max_concurrent_jobs + 10)`: twice the jobs, so the half-pool bound admits every job `max_concurrent_jobs` allows, plus ten for requests; 18 at the defaults, under the default `pool_size` of 20. It applies whether or not `[backup.store]` is configured, because legacy restores are jobs too. Startup raises a smaller `pool_size` to that value with a warning naming both numbers, as it already raises a `pool_size` under `MIN_POOL_SIZE` (10) today. A configuration error (a missing or denied bucket, a part size, prefix or key-length problem, an insecure endpoint, a root that fails the owner-only rule) is a startup error naming the path or bucket, except for the defaulted SQLite root above, where it is a warning. A transport failure (no region, no credentials, an unreachable endpoint), and a failure of section 4's identity check or manifest probe, do not stop startup, because a store problem must not take the data plane out of rotation (section 9): the failure is logged with its cause, the store component is reported unhealthy, the operations that touch the store refuse with a ValidationException that names the store's kind and not its root (CreateBackup on the off-host path, and RestoreTableFromBackup and DeleteBackup of an off-host backup; Describe and List, legacy backups and Cassandra's legacy create are unaffected), the reconciler acts on no row of the store (its legacy duty, which touches no store, still runs) and resumes no restore, and the check runs again each reconciler cycle until it passes. @@ -247,9 +247,9 @@ Documentation: the differences doc's PITR row, rewritten as section 6 says. ### PR 6: catalog index, `BackupCatalog`, `JobClaim`, the legacy reader Goal: the catalog index of section 4 on every backend and the engine-side Describe, List and Delete over it, with the legacy reader beside it. Its line names the interfaces and carries the acceptance: -- PR 6: catalog index per section 4, the `BackupCatalog` trait and `JobClaim` of section 3 implemented by every backend, the `backup_index` row (a new table or collection on PostgreSQL, SQLite and MongoDB; the reused legacy tables with `store_kind` set, plus `backup_index_by_status`, on Cassandra) and the restore job fields on #384's registration record, with the migration's backfill of `status`, `heartbeat_at` (the epoch, not NULL) and a sentinel `owner_generation` on the registrations that exist (PostgreSQL migration 005 and - catalog 0.0.6 - reserved now; SQLite schema; Cassandra V007 writing that version; MongoDB `backup_index` collection and `tables` fields, the versioned migration step MongoDB lacks today and its bump from `0.0.2` (section 4; the expected version is copied three times, twice in `bootstrapper.rs` and once in `operations.rs`, and this PR collapses it to one constant); `RESTORE_LOCK_SEED` made `pub` with `#[doc(hidden)]` so its copy in `tests/backup_restore.rs` binds to it; the `store_id` column, the `backup_maintenance` record and Cassandra's write order across its three backup tables), the `[backup]` struct in `crates/config` with the job keys of section 7 (everything but `store`, which PR 7 adds) and the `stale_job_after_secs >= 3 * heartbeat_interval_secs` check, since this PR's reconciler and permit read them, `max_concurrent_jobs` (default 4) replacing the per-backend slot semaphores (`restore_server_slot_count` on PostgreSQL and MongoDB, `backup_server_slot_count` on SQLite removed) with the request order of 5.1 step 1 and 5.3 step 1, the permit last, and the reconciler period and maintenance-claim check of section 4, engine-side Describe, List (the handler's existing paging, filtering and cursor exceptions kept, over the catalog index merged with the legacy reader through `list_account_backup_rows`, and a table ARN accepted in `TableName` as main's CreateBackup accepts it), Delete (BackupInUse on a CREATING row and while a restore target registered from the backup is CREATING; DELETING ordering; a ValidationException naming no root for a row of another store) over the new rows, `LegacyBackupReader::create` kept on every backend as the CreateBackup path while no store is configured, with its deprecation warning, and on Cassandra whatever the store configuration, with the `LegacyBackupReader` trait and fallback, the legacy restore registering through the new job fields and, with a dead owner, marked FAILED and its target removed through `remove_restore_target` (section 3); the engine adopts `BackupArn`, addresses every backup row by account and backup id (`BackupKey`; `backup_index` keyed by (`account_id`, `backup_id`), no `store_key_prefix` column) and every restore target by account, name and table id, and derives the API's source-table shapes from `source_table_details_json`; the `catalog_id`, written at init and by the migration for an existing catalog; DeleteBackup's in-use check and DELETING write in one transaction under the row's lock (`start_backup_delete`), and on Cassandra `active_restores` serving new rows; the legacy create taking a `max_concurrent_jobs` permit, and Cassandra's legacy create heartbeated so the reconciler removes a dead one's CREATING row; the job row's `kind` and store columns; #384's MongoDB claim markers, Cassandra `restore_owner`, PostgreSQL `RestoreOwner` and their startup sweeps are replaced by the generation column and the reconciler; `RESTORE_LOCK_SEED` becomes the shared seed. Acceptance: all existing backup tests pass unchanged against backups created by the PR 0 binary (run the binary, not the unit), the existing paging tests and `test_restore_into_a_creating_target_and_a_second_restore_of_its_backup_are_refused` included (the vector tests change in PR 8 and 9a, not here); a catalog at #384's version holding a CREATING restore target left by SIGKILL migrates with that registration's `heartbeat_at` at the epoch, and the first reconciler pass removes the target, run on PostgreSQL and on MongoDB; ListBackups with a table ARN in `TableName` returns what the table name returns; DescribeBackup of an ARN of account B that carries the id of account A's backup returns BackupNotFound; a DeleteBackup and a legacy restore of one backup serialize, main's `delete_backup_winning_before_restore_leaves_no_target` and `restore_registration_winning_before_delete_returns_backup_in_use` pass unchanged for the legacy path, and the same two orderings run against the engine's DeleteBackup and RestoreTableFromBackup over an off-host backup (a `backup_index` row), gated on the `backup_index` row lock, a DeleteBackup that wins leaving the registration BackupNotFound and no target and a registration that wins leaving DeleteBackup BackupInUse; a server built before this PR, started against the migrated catalog, is the mixed-version case section 10 forbids and is not tested. Depends on PR 4 (`BackupArn`) and PR 5 (`SnapshotInfo`). +- PR 6: catalog index per section 4, the `BackupCatalog` trait and `JobClaim` of section 3 implemented by every backend, the `backup_index` row (a new table or collection on PostgreSQL, SQLite and MongoDB; the reused legacy tables with `store_kind` set, plus `backup_index_by_status`, on Cassandra) and the restore job fields on #384's registration record, with the migration's backfill of `status`, `heartbeat_at` (the epoch, not NULL) and a sentinel `owner_generation` on the registrations that exist (migration id 5 on PostgreSQL and SQLite; Cassandra V006 and MongoDB's step `002_backup_catalog_index`, both writing + catalog 0.0.5; + the MongoDB `backup_index` collection and `tables` fields, the versioned migration step MongoDB lacks today and its bump from `0.0.2` to that version (section 4; the expected version is copied three times, twice in `bootstrapper.rs` and once in `operations.rs`, and this PR collapses it to one constant); `RESTORE_LOCK_SEED` made `pub` with `#[doc(hidden)]` so its copy in `tests/backup_restore.rs` binds to it; the `store_id` column, the `backup_maintenance` record and Cassandra's write order across its three backup tables), the `[backup]` struct in `crates/config` with the job keys of section 7 (everything but `store`, which PR 7 adds) and the check that `stale_job_after_secs` is at least three times the effective heartbeat period, `max(heartbeat_interval_secs, JOB_HEARTBEAT_INTERVAL)` (section 7), since this PR's reconciler and permit read them, `max_concurrent_jobs` (default 4, at most 1024) replacing the per-backend restore slot semaphores (`restore_server_slot_count` on PostgreSQL and MongoDB removed; SQLite's `backup_server_slot_count` kept for snapshot opens until PR 8, which folds it into `max_concurrent_jobs`; the legacy create tries it without waiting and answers the one `LimitExceededException` text, `limit_exceeded_message`) with the request order of 5.1 step 1 and 5.3 step 1, the permit last, and the reconciler period and maintenance-claim check of section 4, engine-side Describe, List (the handler's existing paging, filtering and cursor exceptions kept, over the catalog index merged with the legacy reader through `list_account_backup_rows`, and a table ARN accepted in `TableName` as main's CreateBackup accepts it), Delete (BackupInUse on a CREATING row and while a restore target registered from the backup is CREATING; DELETING ordering; a ValidationException naming no root for a row of another store) over the new rows, `LegacyBackupReader::create` kept on every backend as the CreateBackup path while no store is configured, with its deprecation warning, and on Cassandra whatever the store configuration, with the `LegacyBackupReader` trait and fallback, the legacy restore registering through the new job fields and, with a dead owner, marked FAILED and its target removed through `remove_restore_target` (section 3); the engine adopts `BackupArn`, addresses every backup row by account and backup id (`BackupKey`; `backup_index` keyed by (`account_id`, `backup_id`), no `store_key_prefix` column) and every restore target by account, name and table id, and derives the API's source-table shapes from `source_table_details_json`; the `catalog_id`, written at init and by the migration for an existing catalog; DeleteBackup's in-use check and DELETING write in one transaction under the row's lock (`start_backup_delete`), and on Cassandra `active_restores` serving new rows; the legacy create taking a `max_concurrent_jobs` permit, and Cassandra's legacy create heartbeated so the reconciler removes a dead one's CREATING row; the job row's `kind` and store columns; #384's MongoDB claim markers, Cassandra `restore_owner`, PostgreSQL `RestoreOwner` and their startup sweeps are replaced by the generation column and the reconciler; `RESTORE_LOCK_SEED` becomes the shared seed. Acceptance: all existing backup tests pass unchanged against backups created by the PR 0 binary (run the binary, not the unit), the existing paging tests and `test_restore_into_a_creating_target_and_a_second_restore_of_its_backup_are_refused` included (the vector tests change in PR 8 and 9a, not here); a catalog at #384's version holding a CREATING restore target left by SIGKILL migrates with that registration's `heartbeat_at` at the epoch, and the first reconciler pass removes the target, run on PostgreSQL and on MongoDB; ListBackups with a table ARN in `TableName` returns what the table name returns; DescribeBackup of an ARN of account B that carries the id of account A's backup returns BackupNotFound; a DeleteBackup and a legacy restore of one backup serialize, main's `delete_backup_winning_before_restore_leaves_no_target` and `restore_registration_winning_before_delete_returns_backup_in_use` pass unchanged for the legacy path, the `backup_index` row lock observed on PostgreSQL by a two-transaction test of `start_backup_delete` (one holding the row `FOR UPDATE` while it registers a CREATING target, the delete waiting and then answering BackupInUse); the same two orderings run against the engine's DeleteBackup and RestoreTableFromBackup over an off-host backup (a `backup_index` row), gated on that lock, a DeleteBackup that wins leaving the registration BackupNotFound and no target and a registration that wins leaving DeleteBackup BackupInUse, deferred to PR 9a, which adds the first restore of a `backup_index` row (on this PR no restore of one exists); the reconciler's FAILED-target duty of 4(e) on a FAILED record whose target is still CREATING, listed once stale and removed; a server built before this PR, started against the migrated catalog, is the mixed-version case section 10 forbids and is not tested. Depends on PR 4 (`BackupArn`) and PR 5 (`SnapshotInfo`). Invariants honoured: 4(a) and (b) (the two records and their transitions), 4(d) (every row by `BackupKey`, every target by `RestoreTargetKey`), 4(e) (the claim), 4(f) (the maintenance claim its reconciler period checks), 5.2 (DeleteBackup's lock and ordering against a registration), and the Goals' unchanged request and response shapes. @@ -272,15 +272,15 @@ The `restore_jobs` record, #384's registration extended: on PostgreSQL and SQLit status RUNNING | FAILED | DONE kind legacy | manifest: a restore through the `LegacyBackupReader` or from a manifest (set at 5.3 step 4; PR 6's backfill writes legacy) store_kind, store_root, store_id the store the restore reads, recorded at 5.3 step 4 (null for a legacy restore) -owner_generation the same claim as a backup job (5.1 step 3); replaces #384's restore_operation_id (MongoDB) and restore_owner (Cassandra, where a non-key column cannot be renamed: V007 adds owner_generation beside restore_owner, which PR 14 drops) +owner_generation the same claim as a backup job (5.1 step 3); replaces #384's restore_operation_id (MongoDB) and restore_owner (Cassandra, where a non-key column cannot be renamed: V006 adds restore_owner_generation beside restore_owner, which PR 14 drops) heartbeat_at replaces #384's restore_heartbeat / restore_heartbeat_at spool_host the host whose spool holds the downloaded files (5.3 step 3) file_index, items_applied per-batch progress on SQLite and Cassandra (5.3 step 5); unused on PostgreSQL (see `restore_progress`) and MongoDB (per-file record) failure_reason, completed_at ``` -PR 6's migration backfills the new fields on the registrations that exist at migration time, on all four backends: `kind = legacy`, `status = DONE` where the table is not CREATING, and where it is (a restore the upgrade's stop interrupted), `status = RUNNING` with `heartbeat_at` set to the epoch (1970-01-01T00:00:00Z), not NULL, on every backend, so every backend's staleness test selects them (PostgreSQL's test also has an `IS NULL` arm as a second guard, 5.1 step 3), and a sentinel `owner_generation` (0) that no job holds; the first reconciler pass fails those rows as legacy restores (section 3) and removes their targets (`remove_restore_target`), which is what #384's startup sweeps did for such a target. -On Cassandra: On Cassandra the backup row is denormalized into `backups_by_arn`, `backups_by_account` and `backups_by_table`, the three legacy tables reused, with `store_kind IS NOT NULL` marking a new row and `store_kind IS NULL` a legacy one. The two list tables carry no status, so the rule between the three is presence, not status: a `backups_by_account` or `backups_by_table` row exists iff the authoritative `backups_by_arn` row is AVAILABLE. `backups_by_arn` is authoritative and every writer writes it first, so a writer that dies between the writes leaves a state the next sync or reconciler cycle completes; `backups_by_arn` is partitioned by `(account_id, backup_arn)`, so a status-keyed read has no access path in it, and V007 adds `backup_index_by_status`, primary key `((store_root, backup_status), backup_arn)`, one partition per status for a root, which every status write maintains beside the authoritative row. -Versions and the mixed-version rule: MongoDB's expected version is `0.0.2` on main and #384 added its restore fields without a bump; PR 1 leaves MongoDB at `0.0.2`, and PR 6 adds the versioned migration step MongoDB lacks (a `$set` of the version after the collection and field changes, recorded in `schema_history`) and moves it from `0.0.2` to the version the other backends write in PR 6. On PostgreSQL, SQLite and MongoDB it is a new table or collection, `backup_index`, with primary key (`account_id`, `backup_id`) and `backup_id` unique; the legacy `backups` table or collection stays for the `LegacyBackupReader` (section 3), whose queries are scoped to it, so neither reader sees the other's rows. What this leaves is mixed versions on one catalog. A pre-PR-6 server does not read `backup_index`, so it would not list a backup a PR 6 server created, which is the benign direction; the hazard is the reverse: a PR 6 server sees legacy rows only through the frozen reader, so a backup or restore an old server runs during the mixed period is a legacy job whose registration the new server's reconciler does not own (PR 6 replaced #384's sweeps), and a legacy restore the old server abandons is swept by nobody until that server restarts. Section 10 tells operators to stop every pre-PR-6 server before `extenddb migrate`. +PR 6's migration backfills the new fields on the registrations that exist at migration time, on all four backends: `kind = legacy`, `status = DONE` where the table is not CREATING, and where it is (a restore the upgrade's stop interrupted), `status = RUNNING` with `heartbeat_at` set to the epoch (1970-01-01T00:00:00Z), not NULL, on every backend, so every backend's staleness test selects them (the column is NOT NULL after the migration on PostgreSQL, so its test has no `IS NULL` arm), and a sentinel `owner_generation` (0) that no job holds; the first reconciler pass fails those rows as legacy restores (section 3) and removes their targets (`remove_restore_target`), which is what #384's startup sweeps did for such a target. +On Cassandra: On Cassandra the backup row is denormalized into `backups_by_arn`, `backups_by_account` and `backups_by_table`, the three legacy tables reused, with `store_kind IS NOT NULL` marking a new row and `store_kind IS NULL` a legacy one. The two list tables carry no status, so the rule between the three is presence, and it differs by writer: for an index row (`store_kind` set) the list rows exist for as long as the authoritative row does, of whatever status, and every reader re-reads the authoritative row and applies its status; the legacy create keeps its publish-time rule, writing its list rows only when its row becomes AVAILABLE. `backups_by_arn` is authoritative and every writer writes it first, so a writer that dies between the writes leaves a state the next sync or reconciler cycle completes; `backups_by_arn` is partitioned by `(account_id, backup_arn)`, so a status-keyed read has no access path in it, and V006 adds `backup_index_by_status`, primary key `((store_root, backup_status), backup_arn)`, one partition per status for a root, which every status write maintains beside the authoritative row. +Versions and the mixed-version rule: MongoDB's expected version is `0.0.2` on main and #384 added its restore fields without a bump; PR 1 leaves MongoDB at `0.0.2`, and PR 6 adds the versioned migration step MongoDB lacks (a `$set` of the version after the collection and field changes, recorded in `schema_history`) and moves it from `0.0.2` to the version Cassandra's V006 writes, named in 4(i). On PostgreSQL, SQLite and MongoDB it is a new table or collection, `backup_index`, with primary key (`account_id`, `backup_id`) and `backup_id` unique; the legacy `backups` table or collection stays for the `LegacyBackupReader` (section 3), whose queries are scoped to it, so neither reader sees the other's rows. What this leaves is mixed versions on one catalog. A pre-PR-6 server does not read `backup_index`, so it would not list a backup a PR 6 server created, which is the benign direction; the hazard is the reverse: a PR 6 server sees legacy rows only through the frozen reader, so a backup or restore an old server runs during the mixed period is a legacy job whose registration the new server's reconciler does not own (PR 6 replaced #384's sweeps), and a legacy restore the old server abandons is swept by nobody until that server restarts. Section 10 tells operators to stop every pre-PR-6 server before `extenddb migrate`. Interfaces it adds. The trait methods below are PR 6's, with the key each addresses rows by; the methods PR 7, PR 9a and PR 10 add to the same trait are in their sections, and the per-backend mechanics behind each method are this PR's design. Lifetimes elided. ```rust @@ -320,9 +320,9 @@ pub trait JobClaim: Send + Sync { } ``` `BackupEngine` on the storage trait is reduced to what only a backend can do. Removed: `create_backup`, `describe_backup`, `list_backups`, `delete_backup`, `backup_restore_source`, `restore_table_from_backup` (#384 already removed `restore_table_to_point_in_time`). The engine owns those now. Added: `LegacyBackupReader` (section 3 states its contract): the one part that is not frozen is the restore's registration: a legacy restore registers through the `restore_jobs` fields of section 4, with `kind = legacy`, and holds the same claim as a new restore (on Cassandra it also registers in the backup row's `active_restores` map, as every restore there does, section 4, so the frozen DeleteBackup's guard sees it), so the reconciler finds a legacy restore whose owner died as it finds a new one (PR 6 removes #384's per-backend sweeps, and a frozen copy of the registration would be swept by nobody). A legacy RUNNING restore whose owner is dead is marked FAILED and its target removed (`remove_restore_target`, above), as #384's sweep does, and is never resumed: a legacy copy has no data files, spool or progress record to resume from. Cassandra's and SQLite's legacy creates commit a CREATING row before the copy; Cassandra's is the one that needs heartbeating, so it is not frozen either: its insert also sets `owner_generation` and `heartbeat_at`, the engine heartbeats it on the job timer, and the reconciler removes a legacy CREATING row and its `backup_items` through the reader's `delete` once that heartbeat is older than `stale_job_after_secs`; on SQLite, one process, a legacy CREATING row that no running create holds is removed the same way at startup and at runtime; the PostgreSQL and MongoDB legacy creates commit their row AVAILABLE with the copy and leave none. Describe, List, Delete and Restore on the engine consult the new catalog index first and fall back to the reader; List merges the two sources into one (`created_at`, `backup_arn`) order, which works with a bare-ARN cursor because `created_at` is the id's instant on both sides for every id that carries a 14-digit instant (section 4); the two cursor exceptions #384's handler makes are kept (5.2). The frozen reader keeps #384's behaviour for a legacy backup of a table with vector indexes (refused at restore on PostgreSQL, restored without its vector indexes on SQLite, as `docs/differences-from-dynamodb.md` records), and the legacy create keeps main's refusal of such a table, which the engine runs in front of `LegacyBackupReader::create` as main runs it in front of the backend's create (5.1 step 1); only the off-host path backs such tables up and restores them. -What the claim is per backend (4(e) states the rules): PostgreSQL: `pg_try_advisory_lock(hashtextextended('backup:' || backup_id, RESTORE_LOCK_SEED))` on a dedicated, non-pooled connection to the catalog database (outside `pool_size`), taken before the row exists and held for the job's life; the seed is the one constant #384's restore lock already uses, which PR 6 makes `pub` with `#[doc(hidden)]`, so there is one lock key space; the job's heartbeats go through that same connection, so a managed server dropping the idle session surfaces as a failed heartbeat and the job aborts instead of continuing unowned. MongoDB: the row is the claim, born owned by an `update_one` with `upsert: true`, a filter on `owner_generation: {$exists: false}` and an update pipeline that sets the generation and `heartbeat_at: "$$NOW"`, so no row ever exists unowned; a heartbeat that does not return exactly one matched document means the claim may be lost and the job aborts. SQLite: one process, so the claim is an in-process slot; a restart is proof of death, and so is a slot no task holds. Cassandra (a restore target only): the row is the claim, taken with a lightweight transaction `IF owner_generation = ` (or `IF NOT EXISTS` at creation), and each heartbeat is a conditional `UPDATE` on the coordinator's clock; a heartbeat whose `[applied]` is false or that errors aborts the job. The lock is PostgreSQL's `FOR UPDATE` on the `backup_index` row, SQLite's write transaction, a write to the backup's `backup_index` document in a MongoDB session transaction (an `$inc` of a counter, so the write always modifies the document and two such transactions conflict), and on Cassandra the lightweight transactions on the `backups_by_arn` row (section 4). +What the claim is per backend (4(e) states the rules): PostgreSQL: `pg_try_advisory_lock(hashtextextended('backup:' || backup_id, RESTORE_LOCK_SEED))` on a dedicated, non-pooled connection to the catalog database (outside `pool_size`), taken before the row exists and held for the job's life; the seed is the one constant #384's restore lock already uses, which PR 6 makes `pub` with `#[doc(hidden)]`, so there is one lock key space; the job's heartbeats go through that same connection, so a managed server dropping the idle session surfaces as a failed heartbeat and the job aborts instead of continuing unowned. MongoDB: the row is the claim, born owned by an `update_one` with `upsert: true`, a filter on `owner_generation: {$exists: false}` and an update pipeline that sets the generation and `heartbeat_at: "$$NOW"`, so no row ever exists unowned; a heartbeat that does not return exactly one matched document means the claim may be lost and the job aborts. SQLite: one process, so the claim is an in-process slot; a restart is proof of death, and so is a slot no task holds. Cassandra (a restore target only): the row is the claim, taken with a lightweight transaction `IF owner_generation = ` (or `IF NOT EXISTS` at creation), and each heartbeat is a conditional `UPDATE` on the coordinator's clock; a heartbeat whose `[applied]` is false, or that errors three times in a row (one error under a loaded node does not stop a job; a beat that hangs is not counted, and the per-bucket poll of the claim bounds that copy to one more bucket), aborts the job, which then touches neither the row nor its data. The lock is PostgreSQL's `FOR UPDATE` on the `backup_index` row, SQLite's write transaction, a write to the backup's `backup_index` document in a MongoDB session transaction (an `$inc` of a counter, so the write always modifies the document and two such transactions conflict), and on Cassandra the lightweight transactions on the `backups_by_arn` row (section 4). -State transitions it owns: AVAILABLE to DELETING and DELETING to DELETED (5.2); a dead legacy restore to FAILED with its target removed, and a dead legacy CREATING row removed (above); the migration's backfill writes DONE or RUNNING with the epoch heartbeat (above). +State transitions it owns: AVAILABLE to DELETING and DELETING to DELETED (5.2); a dead legacy restore to FAILED with its target removed, the target of a FAILED record removed (the FAILED-target duty, 4(e)), and a dead legacy CREATING row removed (above); the migration's backfill writes DONE or RUNNING with the epoch heartbeat (above). Documentation rows it corrects: the differences doc's `ListBackups` paging row's statement that the ARN form of `TableName` is refused (accepted from PR 6, as the service accepts it; 5.2); no row changes for DeleteBackup racing a restore of the same backup: the two serialize as on main (5.2). Upgrade manual: stop every pre-PR-6 server before `extenddb migrate`, because an old server neither reads `backup_index` nor marks the rows it writes (section 4); a restore in flight at that stop is failed and its target removed on the first reconciler pass (section 4); and the rollback paragraph inverts: today's manual says a rollback to a pre-upgrade catalog removes every backup created after the upgrade because they lived in the catalog, whereas after PR 6 a catalog restored from a pre-upgrade dump loses the `backup_index` rows while the store keeps the bytes, which the old binary cannot read and `backup sync` re-adopts after the re-upgrade. ### PR 7: configuration, the store identity file, `backup purge` @@ -354,14 +354,14 @@ Invariants honoured: 4(a) (every transition of the backup row a job or the recon Interfaces it adds: `extenddb_core::validation::validate_restorable_item` (5.1 step 4), `extenddb_storage::snapshot::paging::SNAPSHOT_FETCH_BYTES`, the worker and the reconciler in `crates/server/src/workers.rs`, the metrics of section 9; the reconciler claims through PR 6's `BackupCatalog::claim_backup`. The snapshot and the claim connection are handed to the backup worker over an in-process channel; the worker is a task in `crates/server/src/workers.rs`, spawned with the server's cancellation token and retained handle like the existing workers. It is needed because four write paths store an `N` value such as `1E126` or `abc` unvalidated on main (#618): BatchWriteItem, TransactWriteItems, UpdateItem on an absent item (which creates the item from its `Key`) and ImportTable, the list #394's `key_order.rs` doc gives. So that one run finds every offending item, the first refusal stops encoding and uploading but not validation: the job reads and checks the rest of the snapshot, logs each refused item (its key, attribute path and reason, with the backup ARN) up to 1000 items and counts the rest, and then fails the backup (step 5) with a FAILED reason that gives the number of refused items and the first one, naming #618 only when that item's refusal is an `N` value; so no backup that no restore can load is reported AVAILABLE. The byte budget (section 7 states the bound): PR 8 replaces #395's cursor on PostgreSQL with pages bounded by bytes in the query, each page a keyset `SELECT` in the same `REPEATABLE READ` transaction, ordered by the data table's primary key and starting after the last key of the page before, with a running byte sum that cuts the page at `SNAPSHOT_FETCH_BYTES` (64 MiB, a constant beside `DEFAULT_FETCH_BATCH`, not a configuration key in v1) and always returns at least one row; the snapshot, the lock and which tables stream `key_ordered` are unchanged. On SQLite a page stops at `DEFAULT_FETCH_BATCH` rows or after the row that reaches `SNAPSHOT_FETCH_BYTES` of item text, whichever comes first, and holds at least one row. The cut rule exists in SQL on PostgreSQL and in Rust on SQLite, so the check that no page holds more than the budget plus one row lives in `extenddb_storage::snapshot_conformance`, which both run; a MongoDB batch is already bounded by the server's 16 MiB reply. Tests that change (5.1 step 1): `create_backup_refuses_a_table_with_vector_indexes` (`crates/engine/src/backup/tests.rs`) stays for that path, PR 8 adds an off-host round trip of a vector table beside it, and `test_create_backup_of_a_vector_table_is_refused` (`tests/test_backup_restore_fidelity.py`) expects the refusal or the round trip according to the server's configured path, which it reads from `EXTENDDB_TEST_BACKUP_STORE=`, a variable the test harness exports and every CI job that runs the suite sets: it derives its branch from the variable, prints which branch ran, and fails if the variable is unset, rather than accepting either outcome; the restore's `ensure_restorable` refusal of every vector backup narrows to "the target has no vector support" only in PR 9a, which rewrites `vector_index_backups_are_refused_in_the_engine` to that case; the frozen legacy reader keeps #384's behaviour for a legacy vector backup (section 3), and section 10 corrects the differences doc's vector row. -State transitions it owns: CREATING to AVAILABLE and to FAILED by the job (5.1 steps 4, 5) and by the reconciler (5.1 step 6); every deleting duty of 4(a). The reconciler, in one place: The reconciler runs at startup and then every `reconcile_interval_secs` (default 60, section 7); it acts on no row of the store while the configured store is unhealthy (the legacy duty touches no store and still runs), and it checks the maintenance claim first, and while one is held it runs only the duties that resolve the states a maintenance command waits on: it finishes DELETING rows of the configured store, resolves CREATING backup rows whose owner is dead and fails RUNNING `restore_jobs` rows whose owner is dead rather than resuming them (a resume is a job, which the claim excludes), removing their targets, and runs the FAILED-target duty (such a target keeps its backup in use, so `--forget-backup` refuses until the reconciler has removed it); it skips every other duty, so it never deletes the prefix of a FAILED or DELETED row a `--move-store` is rewriting. Its duties: resolve a CREATING backup row whose owner is dead to AVAILABLE or FAILED; delete the prefix of a FAILED row once its `heartbeat_at` is older than `stale_job_after_secs`, retrying while the prefix lists non-empty; finish a DELETING row (delete the manifest, then the prefix, then mark DELETED once the prefix lists empty); delete the prefix of a DELETED row that still lists objects; remove a FAILED or DELETED backup row once its prefix is empty and its `completed_at` is older than `failed_row_retention_secs` (default 7 days); claim and resume a RUNNING `restore_jobs` row whose owner is dead (on PostgreSQL only once its `heartbeat_at` is older than `stale_job_after_secs`, 5.3 step 7, and a row with no `restore_progress` row is resumed only if the target's data table is empty, otherwise marked FAILED with reason `target_not_empty`, its target left in place and reported for the operator, since a catalog restored from a dump can name a restore that finished and was written to since; the operator inspects that target at the storage level (a CREATING table accepts no reads) and removes it with `extenddb backup remove-target
` (PR 10), the one command that removes a FAILED restore's CREATING target and its record, through `remove_restore_target`, under the maintenance claim like the other writing commands, since DeleteTable refuses a restore target (5.3 step 6); a legacy restore (`kind = legacy`) is marked FAILED and its target removed the same way instead, section 3; on MongoDB a row with `restore_backfill_pending` is DONE, not RUNNING, and is never resumed while the backfill worker finishes it, 5.3 step 6), skipping it to the next cycle when no job permit is free; finish a FAILED `restore_jobs` row whose target table is still CREATING, the FAILED-target duty (`BackupCatalog::remove_restore_target`, section 3, not DeleteTable, which refuses a restore target; on PostgreSQL after the same delay; never for reason `target_not_empty`, whose target is the operator's, above); remove a CREATING restore target that has no job row (`remove_restore_target`); remove `restore_progress` rows whose table is not a CREATING restore target; remove spool entries under this instance's `spool_path` whose job is not RUNNING with this instance's `spool_host` (5.3 step 3); on Cassandra, remove an `active_restores` entry of a backup row whose target is not a CREATING restore target of that backup; remove a legacy CREATING backup row whose create is dead, with its payload (section 3). The reconciler's writes to a backup row carry what the job's would (`size_bytes`, `item_count`, `source_table_details_json` from the manifest for AVAILABLE; `completed_at` for both outcomes, the retention clock). On PostgreSQL its claim is an advisory lock on a dedicated connection that it closes when the row is resolved. `extenddb backup list --failed` prints FAILED rows with their `failure_reason`. +State transitions it owns: CREATING to AVAILABLE and to FAILED by the job (5.1 steps 4, 5) and by the reconciler (5.1 step 6); every deleting duty of 4(a). The reconciler, in one place: The reconciler runs at startup and then every `reconcile_interval_secs` (default 60, section 7); it acts on no row of the store while the configured store is unhealthy (the legacy duty touches no store and still runs), and it checks the maintenance claim first, and while one is held it runs only the duties that resolve the states a maintenance command waits on: it finishes DELETING rows of the configured store, resolves CREATING backup rows whose owner is dead and fails RUNNING `restore_jobs` rows whose owner is dead rather than resuming them (a resume is a job, which the claim excludes), removing their targets, and runs the FAILED-target duty (such a target keeps its backup in use, so `--forget-backup` refuses until the reconciler has removed it); it skips every other duty, so it never deletes the prefix of a FAILED or DELETED row a `--move-store` is rewriting. Its duties: resolve a CREATING backup row whose owner is dead to AVAILABLE or FAILED; delete the prefix of a FAILED row once its `heartbeat_at` is older than `stale_job_after_secs`, retrying while the prefix lists non-empty; finish a DELETING row (delete the manifest, then the prefix, then mark DELETED once the prefix lists empty); delete the prefix of a DELETED row that still lists objects; remove a FAILED or DELETED backup row once its prefix is empty and its `completed_at` is older than `failed_row_retention_secs` (default 7 days); claim and resume a RUNNING `restore_jobs` row whose owner is dead (once its `heartbeat_at` is older than `stale_job_after_secs` on every backend, 5.3 step 7, and a row with no `restore_progress` row is resumed only if the target's data table is empty, otherwise marked FAILED with reason `target_not_empty`, its target left in place and reported for the operator, since a catalog restored from a dump can name a restore that finished and was written to since; the operator inspects that target at the storage level (a CREATING table accepts no reads) and removes it with `extenddb backup remove-target
` (PR 10), the one command that removes a FAILED restore's CREATING target and its record, through `remove_restore_target`, under the maintenance claim like the other writing commands, since DeleteTable refuses a restore target (5.3 step 6); a legacy restore (`kind = legacy`) is marked FAILED and its target removed the same way instead, section 3; on MongoDB a row with `restore_backfill_pending` is DONE, not RUNNING, and is never resumed while the backfill worker finishes it, 5.3 step 6), skipping it to the next cycle when no job permit is free; finish a FAILED `restore_jobs` row whose target table is still CREATING, the FAILED-target duty (4(e); landed in PR 6, whose `list_dead_restore_jobs` returns such a row once it is stale; `BackupCatalog::remove_restore_target`, section 3, not DeleteTable, which refuses a restore target; never for reason `target_not_empty`, whose target is the operator's, above); remove a CREATING restore target that has no job row (`remove_restore_target`); remove `restore_progress` rows whose table is not a CREATING restore target; remove spool entries under this instance's `spool_path` whose job is not RUNNING with this instance's `spool_host` (5.3 step 3); on Cassandra, remove an `active_restores` entry of a backup row whose target is not a CREATING restore target of that backup; remove a legacy CREATING backup row whose create is dead, with its payload (section 3). The reconciler's writes to a backup row carry what the job's would (`size_bytes`, `item_count`, `source_table_details_json` from the manifest for AVAILABLE; `completed_at` for both outcomes, the retention clock). On PostgreSQL its claim is an advisory lock on a dedicated connection that it closes when the row is resolved. `extenddb backup list --failed` prints FAILED rows with their `failure_reason`. Kill points (testing summary): between a data file upload and the manifest; between the manifest put and the AVAILABLE update; SIGTERM during a backup. Main tests that must pass unchanged: #384's backup tests other than the vector tests named above. Documentation rows it corrects: the differences doc's backup consistency row (its MongoDB half, "not point-in-time" today, which the snapshot read concern of section 3 makes one; its Cassandra half stays true), its vector-backup row ("Backing up and restoring a table that has vector indexes": supported on the off-host path from PR 8 and 9a, refused only into a target without vector support, CreateBackup on the legacy create, without `[backup.store]` or on Cassandra, still refused as on main, the legacy reader's behaviour kept for legacy backups; 5.1 step 1), its restore-interruption row (one reconciler with the duties above in place of the per-backend sweeps), a new row for CreateBackup of a table holding an `N` value #618 let in (the backup fails on the off-host path, where main backs it up and restores it; 5.1 step 4), the limits doc's line cap and file size rows, the usage guide's "backups not supported" line; upgrade manual: on the off-host path a backup of a table holding an `N` value the wire refuses (#618) fails where main backs it up, which meets file-backed SQLite without a `[backup]` section and any deployment once a store is configured (5.1 step 4), the reason reaches only the operator (`backup list --failed`, and the log, which names each offending item's key and attribute path with the backup ARN a tenant's support case can cite), and the way out is to rewrite each named item. ### PR 9a: restore job Goal: the restore job of 5.3 on the same backend over a filesystem store. Its line names the interfaces and carries the acceptance: -- PR 9a: restore job per 5.3, same backend, filesystem store: request creates table and `restore_jobs` row atomically in a detached task and returns CREATING (`TableAlreadyExistsException` or `TableInUseException` on a name race, as #384 returns them; `LimitExceededException` over the job limit); Cassandra claim and heartbeat by lightweight transaction; job verifies every file before loading any, with `.verified` spool markers; `bulk_load(target_table_id, claim, file_index, start_at, items)` per backend with the progress record as 5.3 step 5 specifies for each (PostgreSQL data migration 005 for `restore_progress`, reserved here) and the `BackupCatalog` restore methods of section 3; the MongoDB terminal sequence of 5.3 step 6 (DONE and `restore_backfill_pending` in one transaction, the worker's flip conditioned on the flag); `spool_host` as host and spool path; the item validator PR 8 adds, bound with the target's `max_item_size_bytes` (`N` normalization included); items read through `ExtenddbManifest::reader_prehashed`; the restore's vector refusal narrowed to a target without vector support, with `vector_index_backups_are_refused_in_the_engine` rewritten to that case; duplicate rejection across and within files (#394's per-file key-range check on key-ordered backups, the count after the last file on unordered ones, the MongoDB post-resume count check); the manifest and sidecar spooled for a resume; a PostgreSQL resume or target deletion only after the stale window; conditional ACTIVE flip before the progress rows are deleted, the job's target removal (`remove_restore_target`, not DeleteTable) gated on a matched FAILED write, DeleteTable on a restore target refused as #384 refuses it, the in-use rule of 5.2 checked in step 1 before the permit and again in step 4's registration together with `status = AVAILABLE` under the backup row's lock (MongoDB's `$inc`, Cassandra's `active_restores` lightweight transaction), multi-part-key tables restored; `bulk_load` writing index rows in the batch's transaction, with the base rows, on PostgreSQL and SQLite and the ACTIVE flip setting every index ACTIVE there; download transport errors retried for 15 minutes; a restore stopped by shutdown left RUNNING; the store and `kind` recorded on the job row and a takeover writing its `spool_host`; a PostgreSQL resume with no progress row only into an empty target. Acceptance: round trip preserves items, keys, GSIs, LSIs, throughput, SSE, vector where supported, and restores the table `STANDARD` whatever its class (#384's `table_class_is_not_restored` passes unchanged); tampering file 2's sha256 leaves a table that was never writable and is gone after the job fails, with no item ever written; SIGKILL after file 1's commit then restart writes exactly the backup's item count on every backend (MongoDB via duplicate-key suppression); SIGKILL at each kill point of the testing summary leaves a table that either finishes with the right count or is gone (on MongoDB, killed between the DONE write and the worker's flip, the restarted worker finishes it and the reconciler leaves it alone); an unordered backup carrying one key in two files fails into every target, Cassandra included, and a key-ordered backup whose file leaves its `key_range` fails; a second restore of a backup while a target restored from it is CREATING, and a DeleteBackup of that backup, return `BackupInUseException` on every backend, on MongoDB also between the job's DONE write and the worker's ACTIVE flip, and #384's `test_restore_into_a_creating_target_and_a_second_restore_of_its_backup_are_refused` passes unchanged against the new path; PR 6's two orderings over an off-host backup pass unchanged against this PR's restore; on MongoDB two concurrent registrations of one backup leave one target and return `BackupInUseException` to the other; a restore of an ARN of account B that carries the id of account A's backup returns BackupNotFound; on SQLite and PostgreSQL a Query on each GSI and SearchVectors return the restored items as soon as the target reports ACTIVE; a SIGTERM during a restore leaves its row RUNNING and the restore finishes after the restart; a store outage shorter than the retry bound mid-download leaves the restore RUNNING, and it completes once the store answers. Depends on PR 8. +- PR 9a: restore job per 5.3, same backend, filesystem store: request creates table and `restore_jobs` row atomically in a detached task and returns CREATING (`TableAlreadyExistsException` or `TableInUseException` on a name race, as #384 returns them; `LimitExceededException` over the job limit); Cassandra claim and heartbeat by lightweight transaction; job verifies every file before loading any, with `.verified` spool markers; `bulk_load(target_table_id, claim, file_index, start_at, items)` per backend with the progress record as 5.3 step 5 specifies for each (PostgreSQL data migration 005 for `restore_progress`, reserved here) and the `BackupCatalog` restore methods of section 3; the MongoDB terminal sequence of 5.3 step 6 (DONE and `restore_backfill_pending` in one transaction, the worker's flip conditioned on the flag); `spool_host` as host and spool path; the item validator PR 8 adds, bound with the target's `max_item_size_bytes` (`N` normalization included); items read through `ExtenddbManifest::reader_prehashed`; the restore's vector refusal narrowed to a target without vector support, with `vector_index_backups_are_refused_in_the_engine` rewritten to that case; duplicate rejection across and within files (#394's per-file key-range check on key-ordered backups, the count after the last file on unordered ones, the MongoDB post-resume count check); the manifest and sidecar spooled for a resume; a PostgreSQL resume or target deletion only after the stale window; conditional ACTIVE flip before the progress rows are deleted, the job's target removal (`remove_restore_target`, not DeleteTable) gated on a matched FAILED write, DeleteTable on a restore target refused as #384 refuses it, the in-use rule of 5.2 checked in step 1 before the permit and again in step 4's registration together with `status = AVAILABLE` under the backup row's lock (MongoDB's `$inc`, Cassandra's `active_restores` lightweight transaction), multi-part-key tables restored; `bulk_load` writing index rows in the batch's transaction, with the base rows, on PostgreSQL and SQLite and the ACTIVE flip setting every index ACTIVE there; download transport errors retried for 15 minutes; a restore stopped by shutdown left RUNNING; the store and `kind` recorded on the job row and a takeover writing its `spool_host`; a PostgreSQL resume with no progress row only into an empty target. Acceptance: round trip preserves items, keys, GSIs, LSIs, throughput, SSE, vector where supported, and restores the table `STANDARD` whatever its class (#384's `table_class_is_not_restored` passes unchanged); tampering file 2's sha256 leaves a table that was never writable and is gone after the job fails, with no item ever written; SIGKILL after file 1's commit then restart writes exactly the backup's item count on every backend (MongoDB via duplicate-key suppression); SIGKILL at each kill point of the testing summary leaves a table that either finishes with the right count or is gone (on MongoDB, killed between the DONE write and the worker's flip, the restarted worker finishes it and the reconciler leaves it alone); an unordered backup carrying one key in two files fails into every target, Cassandra included, and a key-ordered backup whose file leaves its `key_range` fails; a second restore of a backup while a target restored from it is CREATING, and a DeleteBackup of that backup, return `BackupInUseException` on every backend, on MongoDB also between the job's DONE write and the worker's ACTIVE flip, and #384's `test_restore_into_a_creating_target_and_a_second_restore_of_its_backup_are_refused` passes unchanged against the new path; the two orderings over an off-host backup (a `backup_index` row), gated on the `backup_index` row lock, run here for the first time, deferred from PR 6 (which has no restore of such a row): a DeleteBackup that wins leaves the registration BackupNotFound and no target, a registration that wins leaves DeleteBackup BackupInUse; on MongoDB two concurrent registrations of one backup leave one target and return `BackupInUseException` to the other; a restore of an ARN of account B that carries the id of account A's backup returns BackupNotFound; on SQLite and PostgreSQL a Query on each GSI and SearchVectors return the restored items as soon as the target reports ACTIVE; a SIGTERM during a restore leaves its row RUNNING and the restore finishes after the restart; a store outage shorter than the retry bound mid-download leaves the restore RUNNING, and it completes once the store answers. Depends on PR 8. Invariants honoured: 4(b) (the restore record and its transitions), 4(d) (every target by `RestoreTargetKey`), 4(e) (the claim, the FAILED-write gate before the target is removed, the PostgreSQL stale window), 4(f) (the job checks the maintenance claim after its row exists), 5.2 (the registration under the backup row's lock), 5.3 as written, section 1's duplicate rule and vector refusal, and the Goals' fidelity list. Interfaces it adds: `bulk_load` (section 3) and the restore methods of `BackupCatalog`: @@ -385,7 +385,7 @@ The spool and the resume (5.3 step 3): A file that verifies gets a `.verif The load (5.3 step 5): PostgreSQL: multi-row INSERT in 500-row batches with the GSI, LSI and vector-index rows in the same transaction, not through the propagation queue (section 3), one data transaction per file. The catalog and the data live in different databases, so the per-file progress record is a row in the data database (`restore_progress(table_id, file_index)`, data migration 005, reserved for PR 9a next to the catalog numbers in PR 1 and PR 6), written in the file's own transaction; the catalog job row holds only the terminal status. A file is therefore applied exactly once. SQLite: 10k-item batches, each its own transaction holding the batch's index rows, with the progress record `(file_index, items_applied)` committed in each batch's transaction; MongoDB: `insert_many(ordered=false)` per file, the progress record written after the insert, a resumed restore re-applying the one file whose record is missing with E11000 duplicate-key errors ignored on that file only (any other write error fails the restore), then `countDocuments` checked against the cumulative `item_count` (section 1); Cassandra: #384's restore write path (`put_restored_item`, eight in flight) in 10k-item batches, the progress record written after each batch with a lightweight transaction conditioned on the job's `owner_generation`, so a crash re-applies at most one batch, which is safe because the target accepts no other writes and a put of an identical item changes nothing. The table's `restore_progress` rows live in the data database on PostgreSQL (step 5), so they cannot be in that transaction; they are deleted after it commits, and a crash between the two leaves orphan rows for an ACTIVE table, which are harmless because the reconciler reads progress only for a CREATING restore target and removes progress rows whose table is not one (section 4). The other order, progress rows first, would make a crash between the two resume the restore from file 0 and fail a finished restore on the first duplicate key. State transitions it owns: RUNNING at registration (5.3 step 4); RUNNING to DONE with the ACTIVE flip, on MongoDB DONE then the worker's flip (5.3 step 6); RUNNING to FAILED with the target removed through `remove_restore_target` only after the FAILED write matched (5.3 step 7); the resume of a dead RUNNING row, on PostgreSQL only after the stale window (5.3 step 7). -Kill points (testing summary): every restore kill point, the fenced-out restore job and the `pg_terminate_backend` case; SIGTERM during a restore. Main tests that must pass unchanged: #384's override tests (`crates/engine/src/backup/tests.rs`), `table_class_is_not_restored`, `test_restore_into_a_creating_target_and_a_second_restore_of_its_backup_are_refused`, and PR 6's two orderings. Documentation rows it corrects: the differences doc's vector-backup row (the restore's refusal narrowed to a target without vector support) and its multi-part-key row (restored, 5.3 step 2); the differences and limits docs' override rows stay (5.3). +Kill points (testing summary): every restore kill point, the fenced-out restore job and the `pg_terminate_backend` case; SIGTERM during a restore. Main tests that must pass unchanged: #384's override tests (`crates/engine/src/backup/tests.rs`), `table_class_is_not_restored` and `test_restore_into_a_creating_target_and_a_second_restore_of_its_backup_are_refused`; the two orderings over a `backup_index` row are PR 9a's own new tests (deferred from PR 6). Documentation rows it corrects: the differences doc's vector-backup row (the restore's refusal narrowed to a target without vector support) and its multi-part-key row (restored, 5.3 step 2); the differences and limits docs' override rows stay (5.3). ### PR 9b: cross-backend matrix @@ -458,7 +458,7 @@ Phase 3, later (v1.x): - Every failure path deletes the prefix or leaves it unreachable from List, and the row never says AVAILABLE for an incomplete prefix. - Restore writes no item before every data file's checksum has verified; a verification failure deletes the CREATING target table, which never accepted a read or a write; the request refuses before creating the table when the target cannot represent the manifest or the key ranges overlap. - A file's data and its progress record commit in one transaction on PostgreSQL and SQLite (per 10k-item batch on SQLite, with the items-applied count), so no item is applied twice; on MongoDB the progress record follows the insert, re-application of the one unrecorded file is idempotent by duplicate-key suppression (code 11000 only; any other write error fails the restore), and the post-resume document count is checked against the manifest; on Cassandra the progress record follows each batch under a lightweight transaction and at most one batch is re-applied. A key-ordered restore runs the per-file key-range check on every file, and an unordered one ends with a count of the target equal to the manifest's item count. On PostgreSQL the ACTIVE flip precedes the deletion of progress rows; on MongoDB the job's DONE write and `restore_backfill_pending` are one transaction, and the worker's ACTIVE flip is conditioned on that flag and CREATING, not on the job's generation. -- Every status write by a job is conditional on its owner generation and on the row still being in the job's state (`CREATING` for a backup, `RUNNING` for a restore); a job whose conditional write matches no row, or whose claim connection errors, performs no further store or table operation, deletes (of a prefix or of a restore target) included; the reconciler takes the same claim as a worker and leaves `heartbeat_at` untouched. +- Every status write by a job is conditional on its owner generation and on the row still being in the job's state (`CREATING` for a backup, `RUNNING` for a restore); a job whose conditional write matches no row, or whose conditional write errors with an unknown outcome, performs no further store or table operation, deletes (of a prefix or of a restore target) included; the reconciler takes the same claim as a worker and leaves `heartbeat_at` untouched. - The ARN is parsed by `BackupArn` once, in the engine, before any catalog or store access; no other code splits ARN strings. - Account scoping is checked on the ARN before any store access, every backup row is read by account and backup id and accepted only when its account, table and ARN equal the request's, and every restore-target method takes the target's account, name and table id. - The backup job validates every item with `validate_restorable_item` before encoding it, keeps validating after the first refusal, and the restore binds the same function. @@ -527,3 +527,5 @@ Changed after a review of the draft against main `9f43d03` and PRs #384, #335, # 23. (2026-10-29, after the eighth review round, a verification of item 22's text against the companions' code and main.) Main's two PostgreSQL race tests exercise the legacy path and stay there; PR 6 adds the same two orderings against the engine's DeleteBackup and RestoreTableFromBackup over an off-host backup, gated on the `backup_index` row lock, which corrects item 22's "run unchanged" (5.2, section 10, PR 6, PR 9a). Cassandra's restore methods take the full `tables` key because a secondary-index read scatters and cannot carry a lightweight transaction, not because of `ALLOW FILTERING` (section 3). #384's Cassandra `remove_restore_target` is conditional on the table id only, and never running it without `abandon_restore_target`'s conditional step is this design's rule for the trait method (section 3). The restore's ACTIVE flip sets the table and every index in one catalog transaction on PostgreSQL and SQLite; on Cassandra the index rows are created ACTIVE and the lightweight transaction on the `tables` row flips the table alone, and Cassandra has no vector indexes (5.3 step 6). The `target_not_empty` exception is stated once, in section 4's duty list, which names the duty; the claim-held list, the maintenance-claim paragraph and 5.3 step 7 refer to it; a FAILED target keeps `--forget-backup` refusing, not waiting; and the operator path for a `target_not_empty` target is `extenddb backup remove-target
`, a PR 10 command under the maintenance claim that is audit-logged (section 4, 5.3, 8, PR 10). `sqlite-memory`'s startup sync takes ownership of a root whose file names a `sqlite-memory` catalog as `--adopt-store` does, so one catalog owns the root at a time there too (section 4). Cassandra's and SQLite's legacy creates both commit a CREATING row before the copy, Cassandra's being the one that needs heartbeating; a DeleteBackup that wins against a registration leaves no target rather than creating none; `backup_index_by_status` is partitioned `((store_root, backup_status), backup_arn)`; PR 14's Cassandra step reads the three tables and deletes legacy rows by key, since CQL has no `IS NULL` predicate; section 3's PR 14 sentence names the `--force-drop-legacy-backups` opt-out; the functions `BackupDocuments::build` calls are private to `backup_format`; `bulk_load` writes index rows in the batch's transaction with the base rows wherever the design restates it; the unhealthy-store rule exempts the reconciler's legacy duty, sync with a resolving option and the health check, and PR 7's acceptance names DeleteBackup of an off-host backup among the refused operations; section 8's virtual-hosting example names the two cases the SDK's rules give (a virtual-hostable name on `localhost`, an access point ARN on any custom endpoint); the sentence join in section 3's PostgreSQL bullet, "sees" in 5.3 step 6 and the trait comment on `remove_restore_target`'s first use are corrected; the status line and item 22 carry the commit's date (section 1, 3, 4, 5.2, 5.3, 7, 8, PR 4, PR 7, PR 9a, PR 14, checklist). The companion facts of this round, as their round-8 fixes state them: `--x-s3` and `--xa-s3` names refused, the credential chain's clients given the proxy-less HTTP client when any credential endpoint setting, the IMDS endpoint setting or the container credentials URI is `http://`, so such a credential request never reaches the proxy (without such a setting the chain uses the SDK's default client and `NO_PROXY` must list `169.254.169.254` and `169.254.170.2`), the settings read from a load that sends nothing, the IMDS endpoint setting checked with `http://` allowed only to loopback, `169.254.169.254` and `[fd00:ec2::254]`, `open` without `region` querying IMDS, `SSL_CERT_FILE` and `SSL_CERT_DIR` replacing the system trust store, and SDK-sourced endpoints checked after the shared config is loaded (#393; section 2, 8, PR 3); the store key prefix trimmed of leading and trailing `/` by `BackupDocuments::build` in agreement with the store, `KeyRangeTracker::next_file` carrying ascent across files so an out-of-order boundary fails at the next file's first item, `build` taking each file paired with its ETag, and the sidecar check taking bytes (#394; section 1, 5.1, PR 4); on the separate layout a data commit that fails after the catalog commit retried on a fresh connection for a bounded time before the request returns, the orphan window being a process death or a data database down past that bound, with the troubleshooting page naming how to find and drop such relations (#395; section 3, PR 5). 24. (2026-10-29.) The series is condensed from twelve remaining PRs to ten: the reconciler (formerly PR 8b) ships with the CreateBackup job as PR 8, since a job without its reconciler leaves rows no one resolves, and the CI changes and the documentation (formerly PR 11 and 12) ship as PR 11, since both finalize after PR 9b. Body references to PR 8a, 8b and 12 are renamed; the revision log keeps the old labels as written (section 3, 4, 7, 10, Phasing). 25. (2026-10-29, the split.) The document is cut from 279,584 bytes to 187,722 bytes: sections 1, 2 and 3 now point at the companions' documentation and keep only the rules later PRs build on (the format: `crates/core/src/backup_format` module docs and `crates/core/testdata/extenddb-manifest/README.md`, #394; the store: `crates/backup-store/README.md` and its `config.rs`, `key.rs` and `lib.rs` docs, #393; the snapshot: `crates/storage/src/snapshot.rs`, the four `snapshot_engine.rs` module docs and `docs/troubleshooting.md`, #395); the companions gain the facts only this document stated (the filesystem `list`'s whole-root `Vec` and the empty-prefix IAM policy in #393's README and `lib.rs`; the pre-#361 sort cost and the reason the fetch size is a constant in #395's `snapshot_engine.rs` and `snapshot.rs`, whose module docs already carried the half-pool measurement; a direction sentence in each companion's module doc saying that this overview states the rules the code's callers honour and the crate docs are the record of what the code does); the dependency-change facts of section 2 (lock entries, licences, pins) and the vacuum and WAL growth figures are dropped, the first being the PR description's record and the second the backup manual's. Sections 4 and 5 keep the invariants, the state machines and the per-operation contracts, by lettered part and numbered step; the schema, the per-backend mechanics, the reconciler's duty list, the sync procedure, the claim per backend, the page query and the memory arithmetic moved into a section per later PR under Phasing (PR 1, 6, 7, 8, 9a, 9b, 10, 11, 13, 14), each with its goal, the invariants it honours by reference, the interfaces it adds with their row keys, the transitions it owns and its acceptance list kept whole; section 10 is a table of which document and row each PR changes, the rows themselves in the owning PR's section; the PR 3, 4 and 5 lines go, their PR descriptions and docs being the record of their deliverables. Heading numbers 1 to 10 and the step numbers of 5.1 and 5.3 are unchanged, so items 1 to 24 still resolve; the three `version-literal-ok` markers stay in section 4(i), PR 1's and PR 6's sections. The rewordings the split needed, each keeping its sentence's meaning, are listed in the split's report (`.review/reviews/r8/DONE-SPLIT.md`). +26. (2026-10-30, PR 6 as implemented on #666's registry.) PR 6's migration is id 5 on PostgreSQL and SQLite (one `MigrationId` list for both, ids 1 to 4 taken), and the Cassandra file is V006 and the MongoDB step `002_backup_catalog_index`, both writing `0.0.5` so that one number names one change on every backend; the literals in 4(i) and PR 6's section are updated, PR 1's line keeps its own numbers and renumbers if it lands after this. +27. (2026-10-30, PR 6.) The FAILED-target duty lands in PR 6: `list_dead_restore_jobs` also returns a FAILED `restore_jobs` row whose target is still CREATING once the row is stale, and the reconciler removes the target without a second FAILED write, which closes the state the job's or the reconciler's failure path leaves when the FAILED write lands and the removal does not (4(e), PR 6, PR 8). PostgreSQL's dead rule is the heartbeat alone, as on MongoDB and Cassandra: the advisory lock is the claim's exclusion, not the liveness test, and the `IS NULL` arm item 21 named does not exist, the column being NOT NULL (4(e), 5.3 step 7, PR 6). The stale floor is computed from the period the jobs run at, `max(heartbeat_interval_secs, JOB_HEARTBEAT_INTERVAL)`, the one constant every backend's heartbeat loop reads until the knob reaches the jobs, and `max_concurrent_jobs` has an upper bound (5.1 step 4, section 7, PR 6). SQLite's `backup_server_slot_count` is kept for snapshot opens until PR 8, the legacy create trying it without waiting and answering the one `LimitExceededException` text (PR 6). The engine's `BackupArnSpellings` carries an optional parsed ARN per spelling, so a legacy backup of a name the store-prefix rules refuse (dot-only, over 255 bytes) still describes, restores and deletes through the legacy reader (4(d), PR 6). DeleteBackup over a row runs the in-use check before the other-store check, as a restore does, builds the description before the DELETING write, and a row whose source table the build cannot read is left out of ListBackups rather than failing the listing (5.2, PR 6). The acceptance cell running the two orderings over a `backup_index` row is deferred to PR 9a, with the row lock observed by a two-transaction test of `start_backup_delete` in PR 6 (PR 6). On Cassandra an index row's list rows exist for as long as the row does, whatever its status, readers re-reading the authoritative row; the legacy create keeps its publish-time rule (PR 6). The reconciler records the worker metrics of section 9 (PR 6). On Cassandra the migration runner writes `catalog_version` after the last file's statements, its backfill step and its `schema_history` row, so a run that fails in the step leaves the previous version and `migrate --yes` runs the file again; a Cassandra job stops after three heartbeat errors in a row, as after a heartbeat that does not apply, and the fence the reconciler puts on a dead create's row takes a fresh `owner_generation` (4(e), PR 6). MongoDB's RestoreTableFromBackup waits a bounded time for the registration and answers `InternalServerError` past it while the registration goes on; what the request leaves is stated in 5.3 and the differences document (5.3, PR 6). CreateBackup checks `server.region` before forming a backup ARN and refuses, naming the setting, under a region outside `[a-z0-9-]`, where main created a backup no operation could then address; the store-prefix rules on the name are applied before the ARN is formed so a name over 255 bytes is refused naming the limit whatever its length (5.1 step 1, PR 6). The `LimitExceededException` text names the bound that was met beside its number (`max_concurrent_jobs`, SQLite's snapshot slots, or PostgreSQL's connection slots for the claim connection), one implementation (`limit_exceeded_message`) every refusal binds (5.1 step 1, section 7, PR 6). Section 7 states, where it bounds snapshots by half the pool, that the pool is the raised one (section 7, PR 6). Item 21's `IS NULL` arm is withdrawn by this item alone; item 21 keeps its wording, as item 22 requires. On Cassandra, where the removal is a claim and then a drop, the FAILED-target duty also lists a FAILED record whose target is DELETING with its registration, and `remove_restore_target` resumes the removal from the drop (4(e), PR 6). On MongoDB the in-use check of a restore's registration and of DeleteBackup reads at the transaction read concern, majority-committed data, while DescribeTable reads at `local`, so a request issued within the majority-commit lag after DescribeTable first shows the target ACTIVE can still answer `BackupInUseException` and succeeds when retried; the differences document's concurrent-restores row states it with the measured lag (5.3, PR 6). The rule that a job performs no further operation after a write it cannot confirm is scoped to the conditional write itself: a legacy restore whose claim connection (the heartbeat's) errors runs 5.3 step 7, the conditional FAILED write on a pooled connection and the removal only when that matched, so on PostgreSQL the target is removed at the next heartbeat rather than by the reconciler once the record is stale (4(e), 5.3 step 7, PR 6). On MongoDB `remove_restore_target` deletes the target's catalog document first and returns the target removed once that is gone: a collection drop that fails after it is logged naming the collections and leaves them for `backup sync` (PR 10), there being no record for a later reconciler cycle to retry from (4(e), PR 6). diff --git a/docs/differences-from-dynamodb.md b/docs/differences-from-dynamodb.md index 80c1f671b..7e9415ce8 100755 --- a/docs/differences-from-dynamodb.md +++ b/docs/differences-from-dynamodb.md @@ -102,30 +102,30 @@ Behaviour stated as measured, and every number below, was observed with release | Area | DynamoDB | ExtendDB | |------|----------|------| | What a restore carries across | Key schema, attribute definitions, items, secondary indexes, billing mode and provisioned throughput, on-demand throughput (table and GSI level), SSE settings. Table class is not carried: a restore is `STANDARD` whatever the source's class (measured). Stream settings, TTL, tags, auto scaling, IAM policies, and CloudWatch settings must be reapplied by hand | The same set on all four backends, with one gap: the table's on-demand throughput is carried, a GSI's is not (ExtendDB records no per-index on-demand maximum, and `OnDemandThroughput` is not a member of its GSI definition). Table class is not restored: a `STANDARD_INFREQUENT_ACCESS` source restores as `STANDARD`, as on the service. Streams, TTL, tags, and deletion protection are not restored. A backup written before catalog 0.0.4 (PostgreSQL, SQLite, Cassandra) has no recorded table definition and restores with its keys, items, and billing mode but no secondary indexes. A provisioned one restores with 5/5 throughput on PostgreSQL and SQLite and with its recorded throughput on Cassandra | -| How `RestoreTableFromBackup` runs | Returns at once with the target `CREATING`; the service copies in the background | The same on all four backends: the response carries the target `CREATING`, and the items are copied by a background task in the server, so a client that times out or disconnects does not stop it. Once the request has been authenticated and authorized, the rest of it, from reading the backup to registering the restore, runs in a task of its own, so a client that disconnects after that does not cancel it either: the restore runs and the client finds the target with `DescribeTable`. A client that disconnects while the request is still being authenticated or authorized cancels it, and nothing is created *(from the code)*. Measured with a client whose 1 ms read timeout closed its connection before any response arrived: on all four backends the restore was registered and finished. Measured with a 60,000-item backup of a table with no secondary indexes: response in 0.032 s (PostgreSQL), 0.012 s (SQLite), 0.053 s (MongoDB), 0.151 s (Cassandra); `ACTIVE` after 1.5 s, 2.1 s, 3.6 s, and 11.2 s. On Cassandra the response waits for the target table's schema to be created: 0.75 to 1.3 s with five restores running at once *(earlier build)*, which is longer than a client read timeout of 1 s. A restore runs inside the server process; if the process stops, see the abandoned-restore row | +| How `RestoreTableFromBackup` runs | Returns at once with the target `CREATING`; the service copies in the background | The same on all four backends: the response carries the target `CREATING`, and the items are copied by a background task in the server, so a client that times out or disconnects does not stop it. Once the request has been authenticated and authorized, the rest of it, from reading the backup to registering the restore, runs in a task of its own, so a client that disconnects after that does not cancel it either: the restore runs and the client finds the target with `DescribeTable`. A client that disconnects while the request is still being authenticated or authorized cancels it, and nothing is created *(from the code)*. Measured with a client whose 1 ms read timeout closed its connection before any response arrived: on all four backends the restore was registered and finished. Measured with a 60,000-item backup of a table with no secondary indexes: response in 0.032 s (PostgreSQL), 0.012 s (SQLite), 0.053 s (MongoDB), 0.151 s (Cassandra); `ACTIVE` after 1.5 s, 2.1 s, 3.6 s, and 11.2 s. On Cassandra the response waits for the target table's schema to be created: 0.75 to 1.3 s with five restores running at once *(earlier build)*, which is longer than a client read timeout of 1 s. On MongoDB the request waits at most 60 s for its background task to register the `CREATING` target (a bound against a connection pool that has stopped handing out connections); past that it answers `InternalServerError` while the task goes on, so the target can appear after the error, holding the backup in use (`BackupInUseException` to `DeleteBackup`) and the name until the copy ends; `DescribeTable` is where a client finds it *(from the code)*. A restore runs inside the server process; if the process stops, see the abandoned-restore row | | `RestoreSummary` | The restore response and `DescribeTable` while `CREATING` report `SourceBackupArn`, `SourceTableArn`, `RestoreDateTime` (the backup's `BackupCreationDateTime`, not the time the restore ran), and `RestoreInProgress=true`. Once the table is `ACTIVE`, `DescribeTable` omits `RestoreSummary` | The same on all four backends. No response that describes a table (`CreateTable`, `UpdateTable`, `DeleteTable`, `DescribeTable`, `RestoreTableFromBackup`) carries `RestoreSummary` unless the table is `CREATING`; what the service's `UpdateTable` returns for a restored table was not measured | | Reads on a restore target | A `GetItem` or `Scan` on the target while `CREATING` is `ResourceNotFoundException`; the first read after `DescribeTable` reports `ACTIVE` succeeds (measured) | The same on all four backends. The server's key-info cache stores a negative entry (5 s TTL by default) for a name the catalog does not have, and does not store one for a name the catalog has in a state other than `ACTIVE`: on a not-found answer it asks the catalog once more with `DescribeTable`, and authorization records the name as missing in the key-info snapshot it hands the engine, so the engine does not look it up again. A `GetItem`, `PutItem`, `UpdateItem`, `DeleteItem`, `Query`, or `Scan` on the target while it is `CREATING` therefore costs exactly one `table_key_info` and one `describe_table` catalog load, uncached, per request (on SQLite five SQL statements: one for the key info, four for `DescribeTable`; on PostgreSQL `describe_table` is two queries), plus one `describe_table` on the first such request for the throttle registry; the first read after `ACTIVE` loads once more and is then cached. Measured on SQLite with a `GetItem` loop on a `CREATING` restore target of 100,000 items: 40 requests refused, 40 key-info and 41 `DescribeTable` loads in the server's statement log; with the hand-off to the engine removed, 80 and 81. In an earlier run of the fidelity test the first `GetItem` after `ACTIVE` answered in 3 ms after 72 refused reads. | | `RestoreTableFromBackup` overrides | Current request members are `BillingModeOverride`, `ProvisionedThroughputOverride`, `GlobalSecondaryIndexOverride`, `LocalSecondaryIndexOverride`, `SSESpecificationOverride`, `OnDemandThroughputOverride`, and `VectorIndexOverride`; DynamoDB applies them to the restored table | See the overrides table below. A refused override is refused before the target table is created | | Members of the `RestoreTableFromBackup` response | The `CREATING` description omits `DeletionProtectionEnabled`, the table's `ProvisionedThroughput.NumberOfDecreasesToday`, and each GSI's `IndexStatus` | The same on all four backends; `DescribeTable` reports the first two. Each GSI's `ProvisionedThroughput` keeps `NumberOfDecreasesToday`; what the service sends there was not measured | -| Restoring into a name that exists | `TableAlreadyExistsException`; into a name that is itself a `CREATING` restore target, `TableInUseException`: "Table: is already being restored from backup: " for the same backup, and "Attempt to change a resource which is still in use: Table: is being used." for a different one (both measured) | `TableAlreadyExistsException` on all four backends for an existing table; `TableInUseException` for a name that is a `CREATING` restore target, whichever backup is restored, always with the same-backup text naming the backup the existing target is being restored from (the service's text for a different backup is not reproduced). Each backend checks the name before the backup and before a restore slot is taken, so the same answer comes back whatever state the backup is in (a missing backup reaches the backend only when it is deleted after the engine has read it) and a client retrying its own restore is answered at once; the target insert is the fence that tells an existing table from a `CREATING` target, so two simultaneous requests cannot both pass. The existing table or target is unchanged | +| Restoring into a name that exists | `TableAlreadyExistsException`; into a name that is itself a `CREATING` restore target, `TableInUseException`: "Table: is already being restored from backup: " for the same backup, and "Attempt to change a resource which is still in use: Table: is being used." for a different one (both measured) | `TableAlreadyExistsException` on all four backends for an existing table; `TableInUseException` for a name that is a `CREATING` restore target, whichever backup is restored, always with the same-backup text naming the backup the existing target is being restored from (the service's text for a different backup is not reproduced). The engine checks the name before the backup and before the job permit is taken, so the same answer comes back whatever state the backup is in and whether or not a permit is free, and a client retrying its own restore is answered at once; each backend checks the name again under its registration (a missing backup reaches the backend only when it is deleted after the engine has read it), whose target insert is the fence that tells an existing table from a `CREATING` target, so two simultaneous requests cannot both pass. The existing table or target is unchanged | | `UpdateTable` on a `CREATING` restore target | `ResourceNotFoundException` (measured on DynamoDB) | `ResourceInUseException` on all four backends, the engine's mapping of a table that is not `ACTIVE`. A plain `CreateTable` into the target's name is `ResourceInUseException` "Table already exists: ", the service's text for a taken name (measured on DynamoDB), without the backup ARN. The target is unchanged | -| `BackupArn` that is malformed or belongs to another account | Not an ARN of the right shape, too long (over 1024), a control character, a slash or `..` in the region: `ValidationException` "Invalid Request: BackupArn is not valid". The right shape with an id that is not a backup id (not hex, a wildcard): `ValidationException` "Invalid Backup ARN" (both measured; which input gets which text was captured for one input only: a bare 13-digit id is "Invalid Request: BackupArn is not valid"). Surrounding whitespace, a trailing newline for one, is trimmed, and an ARN naming another region of the caller's account resolves (measured). One in another partition is `ValidationException` (measured; text not captured) | The same two texts on all four backends, split by ExtendDB's parser: a string that is not `arn:aws:dynamodb:::table//backup/` with a region in `[a-z0-9-]`, a digit-only account, a name in `[a-zA-Z0-9_.-]`, only printable ASCII, and at most 1024 bytes after trimming is "Invalid Request: BackupArn is not valid" (so is one in a partition other than `aws`, the only one ExtendDB issues); one of that shape whose id is neither `[0-9]{14}-[0-9a-f]{8}` nor a shape an earlier build issued (13 digits, `-`, 8 hex digits, which every backend issued before the shared id; 13 bare digits, which PostgreSQL v0.1.0 to v0.1.2 issued) is "Invalid Backup ARN"; no build has put a UUID in a backup ARN, so none resolves. Surrounding ASCII whitespace is trimmed (only a trailing newline was measured on the service; trimming all surrounding whitespace is ExtendDB's generalisation), and the region is checked for shape only: the backup is looked up under the ARN rebuilt in this deployment's `server.region` first and, when that is not found and the caller wrote a different ARN, under the ARN as written, so another region's resolves and a backup created under an earlier `server.region` stays reachable by the ARN `ListBackups` reports for it, though not by that ARN rewritten with the new region (see the upgrade manual on `server.region`). A bare 13-digit id (the shape PostgreSQL v0.1.0 to v0.1.2 issued) passes the id check, so such an ARN that names no backup is `BackupNotFoundException`, where the service answers `ValidationException` "Invalid Request: BackupArn is not valid" (measured on DynamoDB). A `server.region` outside `[a-z0-9-]` makes the server's own ARNs "not valid"; the server starts and logs a warning saying so. A well-formed ARN naming another account, of any number of digits, is `AccessDeniedException` on `DescribeBackup`, `DeleteBackup`, and `RestoreTableFromBackup` (not measured on the service). The same checks apply to `ListBackups`'s `ExclusiveStartBackupArn`, where another account's or region's cursor is instead a `ValidationException` (see `ListBackups` paging). With authorization on (not dev mode), `DescribeBackup` and `DeleteBackup` first refuse a `BackupArn` that does not parse as a backup ARN at all (a table ARN, an `xrn:` prefix, an empty field) with "Invalid Backup ARN" before any policy is evaluated, as the service does (measured on DynamoDB), so those shapes get that text instead of the engine's. Every other `BackupArn` is authorized as the backup it names in this deployment (the table and id, in the server's region and the caller's account, surrounding whitespace trimmed as the engine trims it) and then checked by the engine as above. Same on all four backends | +| `BackupArn` that is malformed or belongs to another account | Not an ARN of the right shape, too long (over 1024), a control character, a slash or `..` in the region: `ValidationException` "Invalid Request: BackupArn is not valid". The right shape with an id that is not a backup id (not hex, a wildcard): `ValidationException` "Invalid Backup ARN" (both measured; which input gets which text was captured for one input only: a bare 13-digit id is "Invalid Request: BackupArn is not valid"). Surrounding whitespace, a trailing newline for one, is trimmed, and an ARN naming another region of the caller's account resolves (measured). One in another partition is `ValidationException` (measured; text not captured) | The same two texts on all four backends, split by ExtendDB's parser: a string that is not `arn:aws:dynamodb:::table//backup/` with a region in `[a-z0-9-]`, a digit-only account, a name in `[a-zA-Z0-9_.-]`, only printable ASCII, and at most 1024 bytes after trimming is "Invalid Request: BackupArn is not valid" (so is one in a partition other than `aws`, the only one ExtendDB issues); one of that shape whose id is neither `[0-9]{14}-[0-9a-f]{8}` nor a shape an earlier build issued (13 digits, `-`, 8 hex digits, which every backend issued before the shared id; 13 bare digits, which PostgreSQL v0.1.0 to v0.1.2 issued) is "Invalid Backup ARN"; no build has put a UUID in a backup ARN, so none resolves. Surrounding ASCII whitespace is trimmed (only a trailing newline was measured on the service; trimming all surrounding whitespace is ExtendDB's generalisation), and the region is checked for shape only: the backup is looked up under the ARN rebuilt in this deployment's `server.region` first and, when that is not found and the caller wrote a different ARN, under the ARN as written, so another region's resolves and a backup created under an earlier `server.region` stays reachable by the ARN `ListBackups` reports for it, though not by that ARN rewritten with the new region (see the upgrade manual on `server.region`). A bare 13-digit id (the shape PostgreSQL v0.1.0 to v0.1.2 issued) passes the id check, so such an ARN that names no backup is `BackupNotFoundException`, where the service answers `ValidationException` "Invalid Request: BackupArn is not valid" (measured on DynamoDB). Under a `server.region` outside `[a-z0-9-]` no backup ARN can be formed, so `CreateBackup` is refused with a `ValidationException` naming `server.region`, and the backups taken before the upgrade under such a region keep an ARN the three operations refuse as "not valid"; the server starts and logs a warning saying so. A well-formed ARN naming another account, of any number of digits, is `AccessDeniedException` on `DescribeBackup`, `DeleteBackup`, and `RestoreTableFromBackup` (not measured on the service). The same checks apply to `ListBackups`'s `ExclusiveStartBackupArn`, where another account's or region's cursor is instead a `ValidationException` (see `ListBackups` paging). With authorization on (not dev mode), `DescribeBackup` and `DeleteBackup` first refuse a `BackupArn` that does not parse as a backup ARN at all (a table ARN, an `xrn:` prefix, an empty field) with "Invalid Backup ARN" before any policy is evaluated, as the service does (measured on DynamoDB), so those shapes get that text instead of the engine's. Every other `BackupArn` is authorized as the backup it names in this deployment (the table and id, in the server's region and the caller's account, surrounding whitespace trimmed as the engine trims it) and then checked by the engine as above. Same on all four backends | | Tag conditions on backups | The backup resource has no `aws:ResourceTag` condition key: a backup has no tags to match | `DescribeBackup`, `DeleteBackup`, and `RestoreTableFromBackup` are authorized on the backup ARN as a resource with no tags. An `aws:ResourceTag/` condition finds the key absent for a backup (and `dynamodb:ResourceTag/` is absent for every resource), so, for example, an Allow conditioned on `StringEquals` never grants on a backup and a Deny conditioned on `StringEquals` never denies one. This holds whatever an earlier build stored under the backup ARN (it accepted `TagResource` on one; see Tagging), and a backup does not take its table's tags. For `RestoreTableFromBackup`, the seven item actions evaluated on `table/` (see the authorization rows below) read that ARN's resource tags. A target that does not exist has no tags, so an `aws:ResourceTag/` condition on those actions finds none and does not grant the restore; a target that exists is evaluated with its tags, and a request that is otherwise valid then passes authorization and fails `TableAlreadyExistsException` (or `TableInUseException` for a `CREATING` restore target) when the condition is met, and fails `AccessDeniedException` naming the first item action when it is not (measured on SQLite). Whether the service reads a target's tags in these evaluations was not measured. Scope backup permissions by ARN (`table//backup/*`). Measured on PostgreSQL and SQLite for the Allow case, including tags written under a backup ARN by the earlier build and then served by this one; the Deny case and role sessions rest on unit tests of the authorization path, which is the same for every backend | | Authorization of `ListBackups` | `dynamodb:ListBackups` is evaluated on `arn:aws:dynamodb:::table/*/backup/*`, or on `table//backup/*` when `TableName` filters the list (measured on DynamoDB) | The same on all four backends (the check runs in the server before the backend is reached). A statement on `*` or `table/*` covers every list; one on `table//backup/*` (or `table//*`) covers only the list filtered to that table; one on the table's own ARN, `table/`, covers none. The unfiltered list is one resource, so a Deny on `table//backup/*` denies the list filtered to that table and not the unfiltered list that would include its backups, which follows from the resources the service evaluates. A `TableName` that is not a string is evaluated as no filter; the engine then refuses the request. These resources have been evaluated since v0.1.14; the admin guide's compatibility notes describe what earlier builds evaluated | | Authorization of `RestoreTableFromBackup` | `dynamodb:RestoreTableFromBackup` on the backup ARN, and `dynamodb:Scan`, `dynamodb:Query`, `dynamodb:GetItem`, `dynamodb:PutItem`, `dynamodb:UpdateItem`, `dynamodb:DeleteItem`, and `dynamodb:BatchWriteItem` on `table/`, each of which must be allowed; `dynamodb:RestoreTableFromBackup` is not evaluated on the target (measured on DynamoDB) | The same eight evaluations on all four backends, in that order: the backup first, then the target under each item action in the order listed, stopping at the first denial, which the `AccessDeniedException` names (`User: ... is not authorized to perform: dynamodb:Scan on resource: arn:aws:dynamodb:::table/`); the service's order was not measured. A missing `TargetTableName` is evaluated as `table/` (an empty name, which no statement but `*`-style wildcards covers); the engine then refuses the request. The seven item-action evaluations read the target ARN's resource tags: a target that does not exist has none, one that exists is evaluated with its tags and the restore then fails `TableAlreadyExistsException` (see Tag conditions on backups). Every DynamoDB condition key (`dynamodb:LeadingKeys`, `dynamodb:Attributes`, `dynamodb:Select`, `dynamodb:ReturnValues`, `dynamodb:ReturnConsumedCapacity`, `dynamodb:FullTableScan`, `dynamodb:EnclosingOperation`) is absent in these seven evaluations, so `ForAllValues:*` and `Null: true` conditions pass and bare `StringEquals` and `Bool` conditions fail: an Allow of the item actions conditioned on `ForAllValues:StringEquals dynamodb:LeadingKeys` grants the restore of every item in the backup, an Allow of `dynamodb:Scan` conditioned on `Bool dynamodb:FullTableScan: false` blocks every restore, and a Deny of `dynamodb:Scan` conditioned on `Bool dynamodb:FullTableScan: true` blocks none; the keys the service sets in these evaluations were not measured. The backup ARN is evaluated as the backup it names in this deployment: the table and backup id from the request, in the server's region and the caller's account, whatever partition, region or account the caller wrote, with surrounding whitespace trimmed as the engine trims it. The engine resolves the same table and id (under the server's region first, then as written for a backup created under an earlier `server.region`), so it never acts on a backup other than the one evaluated. `DescribeBackup` and `DeleteBackup` are evaluated the same way. These resources have been evaluated since v0.1.14 (see the admin guide's compatibility notes). A Deny of `dynamodb:RestoreTableFromBackup` on a table does not block restores into it; deny one of the seven item actions on it instead | | Multi-part base table keys (preview) | Not supported | A backup of a table created with `enable_multipart_keys` is refused at restore with `ValidationException`; the item paths address such tables by their first HASH and RANGE attribute only *(from the code and storage-level tests; not checked on a running server)* | -| `DeleteBackup` and `DeleteTable` during a restore | `DeleteTable` on the `CREATING` target is `ResourceInUseException`. `DeleteBackup` during a restore was not measured | While the target is `CREATING`, `DeleteTable` on it is `ResourceInUseException` on all four backends, as on the service. `DeleteBackup` on its source backup is `BackupInUseException` on all four: that refusal is ExtendDB's contract, not measured against the service. Both succeed once the target is `ACTIVE`. How the restore and `DeleteBackup` are ordered differs *(from the code)*. PostgreSQL and SQLite register the restore in the transaction that creates the target, holding the backup row, so a concurrent `DeleteBackup` is ordered strictly before (the restore then finds no backup) or after (it finds the restore). MongoDB has no cross-collection transaction: a restore claims the backup document, inserts the target's catalog document, and releases the claim; `DeleteBackup` claims the backup the same way. Cassandra uses two lightweight transactions on different partitions: the target's row is inserted first (`IF NOT EXISTS`, which reserves the name), then the restore is registered on the backup's row (`IF backup_status = 'AVAILABLE'`), which `DeleteBackup` claims on the same partition; a concurrent `DescribeTable` can therefore see a `CREATING` target for a restore that then fails with `BackupNotFoundException`, after which the target is removed. On MongoDB, if `DeleteBackup` returns an internal error or its request is cancelled after the backup's data drop started, the backup stays `AVAILABLE` and in use (`DeleteBackup` and `RestoreTableFromBackup` return `BackupInUseException`) until the heartbeat sweep finishes the delete, about 45 to 80 s later; it is not left restorable without its data. A `DeleteBackup` that stopped while a restore from the same backup is `CREATING` is finished by the sweep after that restore becomes `ACTIVE`. So a `DeleteBackup` that returned an error can still take effect later: besides the case above, one that failed after claiming the backup (for example because it found a restore from it `CREATING`) and whose claim could not then be released, neither in the request nor by the claim guard's release on drop, is finished by the sweep (once no restore from the backup is `CREATING`), although the client was told it failed (not reproduced) *(all from the code and storage-level tests)*. MongoDB reads the catalog from the primary with read concern `local` regardless of the connection string, which the claims rely on; the connection string must use `readPreference=primary` (any other read preference is refused at startup). The write concern follows the connection string and must be `majority` (the server default): a connection string with `w=1` is refused at startup, because a claim acknowledged under `w=1` just before a primary failover can be rolled back | -| Concurrent restores of one backup | A second `RestoreTableFromBackup` of a backup while a restore from it is still `CREATING` is `BackupInUseException` (measured) | The same on all four backends: the second restore is refused with `BackupInUseException` and creates nothing; the check runs inside the transaction that registers the restore (PostgreSQL, with the backup row locked `FOR UPDATE`; SQLite), inside the claim on the backup document followed by a check for a `CREATING` target of the backup (MongoDB), or inside the lightweight transaction on the backup row (Cassandra), so two simultaneous requests cannot both pass. Once the first target is `ACTIVE` the backup restores again. A restore of a backup whose `DeleteBackup` is in progress answers differently by backend (the service's answer was not measured): `BackupInUseException` "Backup is being deleted" on MongoDB, where the backup stays `AVAILABLE` under a delete claim until its data is dropped; `BackupNotFoundException` on Cassandra, where the delete marks it `DELETING` first; and on PostgreSQL and SQLite the restore waits for the delete's transaction and then finds no backup (`BackupNotFoundException`). Earlier builds accepted concurrent restores of one backup on PostgreSQL, SQLite, and Cassandra, and on MongoDB after a 2 s retry | -| Restore interrupted by the server stopping | Managed by the service | The copy runs in the server process, so a stopped process leaves its target `CREATING`. Each backend removes such a target without operator action, logs the removal with the table name (at warn; at info on Cassandra), and then lets `DeleteBackup` and a new restore into the name succeed. Measured after `SIGKILL` mid-copy and an immediate restart *(earlier build)*: **PostgreSQL** removes it on a control-plane pass once the target is older than 60 s and its restore lock is free (removed 60 s after the kill). **SQLite** removes it when the next server starts (0.1 s after the restart). **MongoDB** and **Cassandra** remove it once its owner's heartbeat is more than 60 s old, by a sweep that runs at startup and every 15 s, so about 45 to 80 s after the owner stopped (the heartbeat's last refresh can be up to one 15 s interval before the stop; measured: 60 s on MongoDB, 62 s on Cassandra). Until then `DeleteTable` and `DeleteBackup` are refused as during a running restore. Not covered *(from the code)*: on MongoDB, a target whose base copy had finished is not swept but is finished by the index backfill worker after a restart; if that backfill fails on every attempt, the worker retries it every 5 s with a warning and never gives up, so the target stays `CREATING` (and `DeleteTable` and `DeleteBackup` stay refused) until the cause is fixed or the target's catalog document is removed by hand (no failing case was found to reproduce this). On Cassandra, a target left `CREATING` by a build before catalog 0.0.4 carries no heartbeat and is not swept. The MongoDB and Cassandra heartbeats compare wall clocks across servers (MongoDB uses the database server's clock for refreshes), so a server whose clock is far behind can have a live restore swept | +| `DeleteBackup` and `DeleteTable` during a restore | `DeleteTable` on the `CREATING` target is `ResourceInUseException`. `DeleteBackup` during a restore was not measured | While the target is `CREATING`, `DeleteTable` on it is `ResourceInUseException` on all four backends, as on the service. `DeleteBackup` on its source backup is `BackupInUseException` on all four: that refusal is ExtendDB's contract, not measured against the service. Both succeed once the target is `ACTIVE`. How the restore and `DeleteBackup` are ordered differs *(from the code)*. PostgreSQL and SQLite register the restore in the transaction that creates the target, holding the backup row, so a concurrent `DeleteBackup` is ordered strictly before (the restore then finds no backup) or after (it finds the restore). MongoDB runs the restore's registration and `DeleteBackup`'s claim each as one session transaction that first increments a lock field on the backup's catalog document while it is `AVAILABLE`, so the two conflict and one orders wholly before the other: a `DeleteBackup` that wins leaves the registration no `AVAILABLE` backup (`BackupNotFoundException`), a registration that wins makes the `DeleteBackup` `BackupInUseException`. Cassandra uses two lightweight transactions on different partitions: the target's row is inserted first (`IF NOT EXISTS`, which reserves the name), then the restore is registered on the backup's row (`IF backup_status = 'AVAILABLE'`), which `DeleteBackup` claims on the same partition; a concurrent `DescribeTable` can therefore see a `CREATING` target for a restore that then fails with `BackupNotFoundException`, after which the target is removed. On MongoDB `DeleteBackup` writes the backup `DELETING` before it drops the data and `DELETED` after; a `DeleteBackup` whose request fails or is cancelled after the drop started leaves the backup `DELETING`, which `DescribeBackup`, `ListBackups` and a restore treat as gone (`BackupNotFoundException`), and a retry of `DeleteBackup` finishes the drop and answers `DELETED` *(from the code and storage-level tests)*. MongoDB reads the catalog from the primary with read concern `local` regardless of the connection string, which the lock and the job records rely on; the connection string must use `readPreference=primary` (any other read preference is refused at startup). The write concern follows the connection string and must be `majority` (the server default): a connection string with `w=1` is refused at startup, because a `DELETING` write or a job record acknowledged under `w=1` just before a primary failover can be rolled back | +| Concurrent restores of one backup | A second `RestoreTableFromBackup` of a backup while a restore from it is still `CREATING` is `BackupInUseException` (measured) | The same on all four backends: the second restore is refused with `BackupInUseException` and creates nothing; the check runs inside the transaction that registers the restore (PostgreSQL, with the backup row locked `FOR UPDATE`; SQLite), inside the claim on the backup document followed by a check for a `CREATING` target of the backup (MongoDB), or inside the lightweight transaction on the backup row (Cassandra), so two simultaneous requests cannot both pass. Once the first target is `ACTIVE` the backup restores again; on MongoDB the check reads majority-committed data (at the default transaction read concern) while `DescribeTable` reads at `local`, so a `RestoreTableFromBackup` or `DeleteBackup` issued within the majority-commit lag after `DescribeTable` first shows the target `ACTIVE` can still be `BackupInUseException` and succeeds when retried (the lag is the journal flush plus any concurrent batch write still holding an earlier oplog slot; measured at 7 to 12 ms, once in about 2,000 observations, with a 3,000-item insert running beside the write). A restore of a backup whose `DeleteBackup` is in progress answers `BackupNotFoundException` on every backend (the service's answer was not measured), by two mechanisms: on MongoDB and Cassandra the delete marks the backup `DELETING` before it drops the data, and the restore finds no `AVAILABLE` backup; on PostgreSQL and SQLite the restore waits for the delete's transaction and then finds no backup. Earlier builds accepted concurrent restores of one backup on PostgreSQL, SQLite, and Cassandra, and on MongoDB after a 2 s retry | +| Restore interrupted by the server stopping | Managed by the service | The copy runs in the server process, so a stopped process leaves its target `CREATING` with a RUNNING restore job record whose heartbeat stops. The engine's reconciler, once at startup and then every `backup.reconcile_interval_secs` (default 60 s), marks the record FAILED and removes the target once the heartbeat is older than `backup.stale_job_after_secs` (default 300 s) on the database's clock, on every backend alike, logs the removal with the table name, and then lets `DeleteBackup` and a new restore into the name succeed *(from the code)*. On SQLite a restart ages every record of the stopped process at once (one process serves the file), so the first cycle removes it without the stale window. Until then `DeleteTable` and `DeleteBackup` are refused as during a running restore. Not covered *(from the code)*: on MongoDB, a target whose base copy had finished has its record DONE and is left alone by the reconciler; the index backfill worker finishes it after a restart, and if that backfill fails on every attempt the worker retries it every 5 s with a warning and never gives up, so the target stays `CREATING` (and `DeleteTable` and `DeleteBackup` stay refused) until the cause is fixed or the target's catalog document is removed by hand (no failing case was found to reproduce this). A target left `CREATING` by a build before this release is backfilled by the catalog migration as a RUNNING record with a heartbeat at the epoch, so the first cycle removes it. Heartbeats and the stale window are measured on the database server's clock, so the application hosts' clocks play no part | | Point-in-time recovery | 35-day window | Not supported. `RestoreTableToPointInTime` is refused with `ValidationException` ("Point-in-time recovery restore is not yet supported"). `UpdateContinuousBackups` is not refused: enabling point-in-time recovery succeeds, and `DescribeContinuousBackups` then reports it `ENABLED` with an `EarliestRestorableDateTime` 35 days back, although nothing is recorded and no restore to any of those times is possible. Same on all four backends | | Where backups live | Managed storage, independent of the table | Inside the database the server already uses, so a lost database loses its backups too: on PostgreSQL in the catalog database (`backups`, `backup_items`, `backup_definitions`, `table_restores`), on SQLite in the one database file, on MongoDB as a `backups` metadata collection in the catalog database and one data collection per backup in the data database, on Cassandra as backup metadata in the catalog keyspace and item payloads in the account's keyspace. Take database-level backups for off-host copies until the backup store program ships | -| Backup creation | `CreateBackup` returns while the backup is `CREATING`; `ListBackups` and `DescribeBackup` expose that state until it becomes `AVAILABLE`. `BackupName` and `TableName` are checked as the model says: length 3 to 255 and `[a-zA-Z0-9_.-]+`, with "1 validation error detected: Value '' at 'backupName' failed to satisfy constraint: " texts, joined under "2 validation errors detected:" when both fail (measured). `TableName` may be the table's ARN (measured). A table that does not exist or is still `CREATING` is `TableNotFoundException` "Table not found: " (measured) | `CreateBackup` copies the items before it returns and returns the backup `AVAILABLE` on all four backends. On PostgreSQL, MongoDB, and Cassandra a concurrent `ListBackups` never saw it `CREATING` *(earlier build)*; on SQLite the backup row is visible as `CREATING` to other requests while the copy runs. Measured *(earlier build)*: 1.3 s for 60,000 items on PostgreSQL, 1.9 s for 60,000 on SQLite, 0.2 s for 20,000 on MongoDB, 5.1 s for 20,000 on Cassandra. MongoDB reports `BackupSizeBytes` 0. The name checks are the service's, with its texts (the length clause before the pattern clause when both fail; the service's order was not captured), counting characters, with `TableName` taking the deployment's table-name limits; before them a 200 KB name was stored and returned in every `ListBackups` page (a 500 on Cassandra) and a NUL byte was a 500 on PostgreSQL. `TableName` as an ARN (`arn:aws:dynamodb:::table/`) is accepted on all four backends: another region's resolves, another account's is `AccessDeniedException` (neither measured on the service); an SDK built from the model refuses the ARN form client-side, so only a hand-built request sends it. `TableNotFoundException` "Table not found: " for a missing or `CREATING` table on all four backends, built from the request's name (the backends' own payloads differ) | +| Backup creation | `CreateBackup` returns while the backup is `CREATING`; `ListBackups` and `DescribeBackup` expose that state until it becomes `AVAILABLE`. `BackupName` and `TableName` are checked as the model says: length 3 to 255 and `[a-zA-Z0-9_.-]+`, with "1 validation error detected: Value '' at 'backupName' failed to satisfy constraint: " texts, joined under "2 validation errors detected:" when both fail (measured). `TableName` may be the table's ARN (measured). A table that does not exist or is still `CREATING` is `TableNotFoundException` "Table not found: " (measured) | `CreateBackup` copies the items before it returns and returns the backup `AVAILABLE` on all four backends. On PostgreSQL, MongoDB, and Cassandra a concurrent `ListBackups` never saw it `CREATING` *(earlier build)*; on SQLite the backup row is visible as `CREATING` to other requests while the copy runs. Measured *(earlier build)*: 1.3 s for 60,000 items on PostgreSQL, 1.9 s for 60,000 on SQLite, 0.2 s for 20,000 on MongoDB, 5.1 s for 20,000 on Cassandra. MongoDB reports `BackupSizeBytes` 0. The name checks are the service's, with its texts (the length clause before the pattern clause when both fail; the service's order was not captured), counting characters, with `TableName` taking the deployment's table-name limits; before them a 200 KB name was stored and returned in every `ListBackups` page (a 500 on Cassandra) and a NUL byte was a 500 on PostgreSQL. `TableName` as an ARN (`arn:aws:dynamodb:::table/`) is accepted on all four backends: another region's resolves, another account's is `AccessDeniedException` (neither measured on the service); an SDK built from the model refuses the ARN form client-side, so only a hand-built request sends it. `TableNotFoundException` "Table not found: " for a missing or `CREATING` table on all four backends, built from the request's name (the backends' own payloads differ). A table the deployment could create but whose name no backup ARN can carry is refused with `ValidationException` where the service backs up any table it can create: a name over 255 bytes (reachable only with `[limits] max_table_name_length` raised) or made only of dots ("TableName cannot be backed up: a backup ARN cannot be formed for it: invalid backup ARN: table name of N bytes is longer than 255" / "... invalid backup ARN: table name \"...\" is only dots"), and every table under a `server.region` outside `[a-z0-9-]` (the `BackupArn` row); the checks run after the table's ACTIVE probe and before the job permit, so the refusal takes no slot | | `DeleteBackup` response | The full `DescribeBackup` description with `BackupStatus` `DELETED`, including `SourceTableDetails.ProvisionedThroughput` (a required member) and `SourceTableFeatureDetails` (measured) | The same on all four backends: the engine reads the backup's recorded definition before the delete and fills the description as `DescribeBackup` does, with `BackupStatus` `DELETED`. A backup whose definition this build cannot read or that does not parse is still deleted and described without feature details (logged at warn), which means a damaged `PROVISIONED` backup is described with on-demand `ProvisionedThroughput` `{0, 0}`: ExtendDB's choice, deletability over the throughput member of a row that is being removed, where `DescribeBackup` on the same row is an `InternalServerError`; a failure to reach the database fails the request and deletes nothing. | | `DescribeBackup` when the definition cannot be read | Not applicable | A definition row that does not parse, a renamed column, or a revoked privilege (`Internal` from the backend) is an `InternalServerError`, as it is for `RestoreTableFromBackup` on the same read; only a definition version this build does not know is described without `SourceTableFeatureDetails`. | | Consistency of a backup | Point-in-time snapshot of the table | **PostgreSQL** and **SQLite** take the items from one snapshot. How *(from the code)*: PostgreSQL holds the table's catalog row shared from the first definition read until the data snapshot (`REPEATABLE READ`, pinned by a read of the table before the row is released) is taken, so the definition recorded is the one in force for the items read; the catalog and the data are separate databases, so this is a barrier, not one cross-database snapshot. SQLite reads all items in one deferred read transaction opened while the engine's write lock was held for the definition read, so writers continue during the copy; that long reader pins WAL checkpointing, so the WAL grows for the duration of the backup by the size of the backup rows written plus any concurrent writes. **MongoDB** (`$out` from the live collection) and **Cassandra** (paging the live table) are not point-in-time: items copied early reflect the table as it was when the copy started, items copied late reflect writes made during the copy. A backup can therefore hold a write without an earlier write it followed, a state the table was never in. Measured *(earlier build)* with a writer that updated an early item and then a late item, pair after pair, during `CreateBackup`: 16 of 66 pairs on MongoDB and 123 of 800 on Cassandra had the late write without the early one; 0 on PostgreSQL (of 228) and SQLite (of 230). Back up a MongoDB or Cassandra table while it is not being written if the backup must be consistent | -| `ListBackups` paging | At most 100 summaries per page, with `LastEvaluatedBackupArn`; `Limit` 1 to 100. `TimeRangeLowerBound`, `TimeRangeUpperBound`, and `BackupType` filter the list. `TableName` is checked as the model says (3 to 255, `[a-zA-Z0-9_.-]+`, the "N validation error(s) detected" texts). An `ExclusiveStartBackupArn` of another account is `ValidationException` "Invalid Request: The supplied ExclusiveStartBackupArn does not match the account"; of another region, or of a table other than `TableName`'s, `ValidationException` ("does not match the region", "does not match the table name predicate"); a time bound before the epoch is `ValidationException`; a lower bound above the upper bound is `ValidationException` "Time range lower bound must be less than or equal to upper bound" (all measured) | The same paging on all four backends: `Limit` outside 1 to 100 is `ValidationException`, `ExclusiveStartBackupArn` resumes after the given backup, and summaries are ordered by creation time then ARN. Summaries carry `TableId`. `TimeRangeLowerBound` (inclusive) and `TimeRangeUpperBound` (exclusive) keep backups created in that range, and `BackupType` keeps backups of that type (`USER` when absent, every type for `ALL`); every ExtendDB backup is `USER`, so `SYSTEM` and `AWS_BACKUP` list nothing. The filters apply before paging, so `Limit` and the cursor count only backups that pass. Measured on SQLite and PostgreSQL with whole-second bounds between two backups taken 2 s apart; the millisecond edges are from the engine's unit tests. The request checks above are the service's, with its texts where captured: the account text in full; the region and table-name texts complete the captured fragments on the account text's pattern ("Invalid Request: The supplied ExclusiveStartBackupArn does not match the region" / "... the table name predicate"), checked in that order after the ARN's shape; a bound before the epoch is refused with ExtendDB's own text naming the member; equal bounds list nothing. An `ExclusiveStartBackupArn` that is not in the listing (excluded by the filters, or deleted since the previous page) is placed by the creation time in its id; one whose id carries no time (the 13-digit ids of earlier builds, with or without the hex suffix: only the 14-digit form carries one) is then a `ValidationException`. A cursor in another region is accepted only when it is the stored ARN of a backup of the account (or of `TableName`), which is what `LastEvaluatedBackupArn` hands out for a backup created under an earlier `server.region`, so paging passes such a backup; any other cursor in another region is refused ("does not match the region"). If that backup is deleted between two pages, its ARN is no longer a stored one and the next page is refused the same way (a same-region cursor would be placed by its time instead); start the listing again. A `server.region` outside `[a-z0-9-]` gives every cursor the server hands out a region the ARN parser refuses, so on such a deployment the second page of any listing is "Invalid Request: BackupArn is not valid". `TableName` here is a table name only: the ARN form `CreateBackup` accepts is refused with the pattern clause (the service's behaviour for `ListBackups` with an ARN was not measured). A numeric `TableName` is refused with the text the service uses for a map ("Start of structure or map found where not expected"), not measured for a number. Every `ListBackups` request reads all of the account's backups (or all of the table's, with `TableName`) from storage and pages them in memory, so walking N backups 100 at a time reads about N²/100 rows in total *(from the code)* | +| `ListBackups` paging | At most 100 summaries per page, with `LastEvaluatedBackupArn`; `Limit` 1 to 100. `TimeRangeLowerBound`, `TimeRangeUpperBound`, and `BackupType` filter the list. `TableName` is checked as the model says (3 to 255, `[a-zA-Z0-9_.-]+`, the "N validation error(s) detected" texts). An `ExclusiveStartBackupArn` of another account is `ValidationException` "Invalid Request: The supplied ExclusiveStartBackupArn does not match the account"; of another region, or of a table other than `TableName`'s, `ValidationException` ("does not match the region", "does not match the table name predicate"); a time bound before the epoch is `ValidationException`; a lower bound above the upper bound is `ValidationException` "Time range lower bound must be less than or equal to upper bound" (all measured) | The same paging on all four backends: `Limit` outside 1 to 100 is `ValidationException`, `ExclusiveStartBackupArn` resumes after the given backup, and summaries are ordered by creation time then ARN. Summaries carry `TableId`. `TimeRangeLowerBound` (inclusive) and `TimeRangeUpperBound` (exclusive) keep backups created in that range, and `BackupType` keeps backups of that type (`USER` when absent, every type for `ALL`); every ExtendDB backup is `USER`, so `SYSTEM` and `AWS_BACKUP` list nothing. The filters apply before paging, so `Limit` and the cursor count only backups that pass. Measured on SQLite and PostgreSQL with whole-second bounds between two backups taken 2 s apart; the millisecond edges are from the engine's unit tests. The request checks above are the service's, with its texts where captured: the account text in full; the region and table-name texts complete the captured fragments on the account text's pattern ("Invalid Request: The supplied ExclusiveStartBackupArn does not match the region" / "... the table name predicate"), checked in that order after the ARN's shape; a bound before the epoch is refused with ExtendDB's own text naming the member; equal bounds list nothing. An `ExclusiveStartBackupArn` that is not in the listing (excluded by the filters, or deleted since the previous page) is placed by the creation time in its id; one whose id carries no time (the 13-digit ids of earlier builds, with or without the hex suffix: only the 14-digit form carries one) is then a `ValidationException`. A cursor in another region is accepted only when it is the stored ARN of a backup of the account (or of `TableName`), which is what `LastEvaluatedBackupArn` hands out for a backup created under an earlier `server.region`, so paging passes such a backup; any other cursor in another region is refused ("does not match the region"). If that backup is deleted between two pages, its ARN is no longer a stored one and the next page is refused the same way (a same-region cursor would be placed by its time instead); start the listing again. Under a `server.region` outside `[a-z0-9-]` the cursor is still the stored ARN: a listing of backups created under a valid region pages, and a cursor that is the ARN of a backup an earlier release created under the invalid region is refused with "Invalid Request: BackupArn is not valid". `TableName` is a table name or a table ARN, as `CreateBackup` accepts it (the service accepts both, measured on DynamoDB): the ARN form lists what the name lists, another account's ARN is `AccessDeniedException`, and the cursor's table check runs against the resolved name. A numeric `TableName` is refused with the text the service uses for a map ("Start of structure or map found where not expected"), not measured for a number. Every `ListBackups` request reads all of the account's backups (or all of the table's, with `TableName`) from storage and pages them in memory, so walking N backups 100 at a time reads about N²/100 rows in total *(from the code)* | | Backup ARN and creation time | Backup id is 14-digit zero-padded epoch milliseconds, `-`, 8 hex digits; `BackupCreationDateTime` has sub-second precision | The same shape on all four backends; `BackupCreationDateTime` is the millisecond in the ARN | -| Running two servers against one MongoDB catalog | Not applicable | MongoDB uses the fixed database names `extenddb_catalog` and `extenddb_data` whatever `--catalog-db` and `--data-db` say, and takes no serve lock: two servers on one replica set share one catalog and both start. The backup and restore sweeps act only on work whose heartbeat is more than 60 s old, so one server does not remove another live server's restore *(from the code and storage-level tests; not checked with two servers)*. Running more than one server per catalog is otherwise unsupported (see the deployment guide) | +| Running two servers against one MongoDB catalog | Not applicable | MongoDB uses the fixed database names `extenddb_catalog` and `extenddb_data` whatever `--catalog-db` and `--data-db` say, and takes no serve lock: two servers on one replica set share one catalog and both start. Each server's reconciler acts only on a restore whose job record's heartbeat is older than `backup.stale_job_after_secs` (default 300 s) on the database's clock, so one server does not remove another live server's restore *(from the code and storage-level tests)*. Running more than one server per catalog is otherwise unsupported (see the deployment guide) | ### `RestoreTableFromBackup` overrides diff --git a/docs/dynamodb-limits.md b/docs/dynamodb-limits.md index 2b37b9023..0044bd5de 100755 --- a/docs/dynamodb-limits.md +++ b/docs/dynamodb-limits.md @@ -128,8 +128,8 @@ Source: [AWS DynamoDB Service Quotas](https://docs.aws.amazon.com/amazondynamodb | Limit | DynamoDB Value | Status | Notes | |-------|---------------|--------|-------| -| Concurrent restores | 50 | Partial | PostgreSQL: one server process runs at most 4 restores per account and at most half its `pool_size` (after the minimum of 10 is applied; 10 with the default `pool_size` of 20), capped at 16, in total. A restore that cannot get both an account slot and a server slot within 30 seconds is refused with `LimitExceededException`. Measured on an earlier build of this branch, with four restores running in one account: a fifth in the same account waited for a slot and answered after 3.1 s, when one of the four finished, while another account's restore answered in 0.05 s. The 30-second refusal is from the code and storage-level tests, not from a running server. MongoDB: the same bound, 4 per account and `min(16, pool/2)` per server where the pool is the engine client's `maxPoolSize` (`max_connections`, or the driver default of 10, so 5 restores), refused with `LimitExceededException` after 30 seconds; a restore request that is not registered within 60 seconds answers an internal error while the restore continues (storage-level tests; measured with 20 concurrent restores on a served default build: all accepted, server responsive). SQLite and Cassandra have no restore limit (five started at once were all answered without waiting for each other, on an earlier build); SQLite bounds `CreateBackup` instead: at most `pool_size / 2 - 1` (4 by default) at once, `LimitExceededException` after 30 seconds | -| `BackupName` and `TableName` on `CreateBackup`, `TableName` on `ListBackups` | Length 3 to 255, `[a-zA-Z0-9_.-]+` | Enforced | The service model's constraints, with its texts ("1 validation error detected: Value '' at 'backupName' failed to satisfy constraint: ...", "2 validation errors detected: ...; ..." when length and pattern both fail). Length is counted in characters. `BackupName` takes the model's 3 to 255; a `TableName` member takes the deployment's `[limits]` `min_table_name_length` and `max_table_name_length` (defaults 3 and 255), as `CreateTable` does, so a table the deployment can create can be backed up and listed. `CreateBackup`'s `TableName` may also be the table's ARN; `ListBackups`'s may not. The value is echoed in the message as sent | +| Concurrent restores | 50 | Partial | Every backend: one server process runs at most `[backup] max_concurrent_jobs` (default 4, at most 1024) backup and restore jobs at once, `CreateBackup` and `RestoreTableFromBackup` together and not per account; a request over the bound is refused at once with `LimitExceededException` ("Too many backup and restore jobs are running on this server (at most N at once, the [backup] max_concurrent_jobs bound); retry later") and creates nothing, with no wait (storage-level and engine tests; measured on PostgreSQL with `max_concurrent_jobs = 1`: a running restore made a CreateBackup and a second restore refuse in under 60 ms). On PostgreSQL `pool_size` and `catalog_pool_size` are raised to at least `2 * max_concurrent_jobs + 10` so every admitted job has a connection. SQLite also bounds the snapshot slots `CreateBackup` and table snapshots (#395) share, a count derived from `[storage.sqlite] pool_size` (the formula is stated once, on that setting's documentation in `SqliteConfig`; 4 for the default pool) so half the pool stays free for ordinary requests; a `CreateBackup` that finds no slot free is refused at once with the same `LimitExceededException`, its parenthesis naming that bound and its count in place of `max_concurrent_jobs` ("at most N at once, the [storage.sqlite] pool_size snapshot-slot bound"); a PostgreSQL job whose dedicated claim connection the database server refuses is refused with the same exception naming the server's connection slots ("the database server has no connection slot left for the job"), with no count. A MongoDB restore request that is not registered within 60 seconds answers an internal error while the restore continues | +| `BackupName` and `TableName` on `CreateBackup`, `TableName` on `ListBackups` | Length 3 to 255, `[a-zA-Z0-9_.-]+` | Enforced | The service model's constraints, with its texts ("1 validation error detected: Value '' at 'backupName' failed to satisfy constraint: ...", "2 validation errors detected: ...; ..." when length and pattern both fail). Length is counted in characters. `BackupName` takes the model's 3 to 255; a `TableName` member takes the deployment's `[limits]` `min_table_name_length` and `max_table_name_length` (defaults 3 and 255), as `CreateTable` does, so a table the deployment can create can be listed, and can be backed up unless no backup ARN can carry its name: a name over 255 bytes (with `max_table_name_length` raised) or made only of dots can be created but `CreateBackup` refuses it with `ValidationException` "TableName cannot be backed up: a backup ARN cannot be formed for it: invalid backup ARN: table name of N bytes is longer than 255" or "... invalid backup ARN: table name \"...\" is only dots" (the service backs up any table it can create; see the differences document's backup creation row). Both members may also be the table's ARN. The value is echoed in the message as sent | | `BackupArn` | 37 to 1024 characters | Maximum enforced; minimum implied | Surrounding ASCII whitespace is trimmed first, then over 1024 bytes, not of the backup ARN shape, a control or non-ASCII character, or a region outside `[a-z0-9-]`, an account that is not digits, or a table name outside `[a-zA-Z0-9_.-]` is `ValidationException` "Invalid Request: BackupArn is not valid"; the right shape with an id that is not a backup id is "Invalid Backup ARN"; a digit-only account that is not the caller's is `AccessDeniedException`. No minimum is checked: the shortest string of the right shape is longer than 37 characters. Another region of the caller's account resolves (the server's region is tried first, then the ARN as written). `ListBackups`'s `ExclusiveStartBackupArn` must also match the account, the region, and `TableName` when given | | `ListBackups` page size | 100 | Enforced | `Limit` 1 to 100, default 100; more rows come back through `LastEvaluatedBackupArn` and `ExclusiveStartBackupArn`. Each request reads every backup of the account (or of the table, with `TableName`) and pages in memory, so walking N backups reads about N²/100 rows. `TimeRangeLowerBound`, `TimeRangeUpperBound`, and `BackupType` filter the list before paging; a bound before the epoch, or a lower bound above the upper bound ("Time range lower bound must be less than or equal to upper bound"), is `ValidationException` | | Backup contents | Key schema, attribute definitions, items, indexes, billing mode, provisioned throughput, on-demand throughput (table and GSI level), SSE. Table class is not carried (a restore is `STANDARD`) | Partial | ExtendDB restores the same set, except a GSI's on-demand throughput (it records none per index); table class is not restored, as on the service. Streams, TTL, tags, and deletion protection are not restored, as on the service. Backups are stored inside the database the server uses; take database-level backups for off-host copies | diff --git a/docs/getting-started.md b/docs/getting-started.md index 91ec7a27d..813191e39 100755 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -1397,9 +1397,9 @@ The `storage.postgres.catalog_pool_size` setting controls the maximum number of **When to increase:** If you see elevated latency under concurrent load, the pool may be saturated. Requests queue at the pool level when all connections are in use. Increase `pool_size` (and `catalog_pool_size` if auth is enabled) to allow more concurrent transactions. -**Relationship to PostgreSQL `max_connections`:** The total connection footprint is `2 * pool_size + catalog_pool_size`, plus one connection per vector index build running on the server and one while a schema migration runs. The doubled term is the catalog pool and the data pool, both sized by `pool_size`; the third pool is the management pool sized by `catalog_pool_size`. A deployment whose catalog and data share one database is the only shape where the two `pool_size` pools collapse into one, and `extenddb init` does not produce it. Those extra sessions are opened outside both pools on purpose, for two different reasons. A build's ownership lock is session-scoped, so it needs a session that ends when the build does: a pooled connection would return to the pool still holding the lock. The migration lock is transaction-scoped, and needs a pinned connection because its explicit transaction has to retain one PostgreSQL backend until COMMIT, which is what stops a transaction-pooling proxy from letting the lock move between backends. PostgreSQL's default `max_connections` is 100, and the default pool sizes already need 60 of it. Ensure `2 * pool_size + catalog_pool_size`, plus the number of vector index builds you expect to overlap, does not exceed your PostgreSQL `max_connections` setting. +**Relationship to PostgreSQL `max_connections`:** The total connection footprint is `2 * pool_size + catalog_pool_size + [backup] max_concurrent_jobs`, plus one connection per vector index build running on the server and one while a schema migration runs. The `max_concurrent_jobs` term (default 4) is one dedicated connection per running backup or restore job, outside every pool, which holds the job's session-level advisory lock (its claim). The doubled term is the catalog pool and the data pool, both sized by `pool_size`; the third pool is the management pool sized by `catalog_pool_size`. A deployment whose catalog and data share one database is the only shape where the two `pool_size` pools collapse into one, and `extenddb init` does not produce it. Those extra sessions are opened outside both pools on purpose, for two different reasons. A build's ownership lock is session-scoped, so it needs a session that ends when the build does: a pooled connection would return to the pool still holding the lock. The migration lock is transaction-scoped, and needs a pinned connection because its explicit transaction has to retain one PostgreSQL backend until COMMIT, which is what stops a transaction-pooling proxy from letting the lock move between backends. PostgreSQL's default `max_connections` is 100, and the defaults already need 64 of it. Ensure `2 * pool_size + catalog_pool_size + max_concurrent_jobs`, plus the number of vector index builds you expect to overlap, does not exceed your PostgreSQL `max_connections` setting. -**Example:** To support 50 concurrent data operations with auth enabled, set both pools to 50 in `extenddb.toml`. That configuration can open 150 connections, so PostgreSQL needs `max_connections` raised to at least 150 plus headroom for overlapping vector index builds. The default of 100 is not enough for it: an operator who sizes for 100 here meets `FATAL: sorry, too many clients already` at exactly the concurrency the setting was chosen to support. +**Example:** To support 50 concurrent data operations with auth enabled, set both pools to 50 in `extenddb.toml`. That configuration can open 154 connections (the default `max_concurrent_jobs` of 4 included), so PostgreSQL needs `max_connections` raised to at least 154 plus headroom for overlapping vector index builds. The default of 100 is not enough for it: an operator who sizes for 100 here meets `FATAL: sorry, too many clients already` at exactly the concurrency the setting was chosen to support. ```toml [storage.postgres] diff --git a/docs/manuals/01-architecture-guide.md b/docs/manuals/01-architecture-guide.md index 4978c3668..ab4f2fd3e 100755 --- a/docs/manuals/01-architecture-guide.md +++ b/docs/manuals/01-architecture-guide.md @@ -205,9 +205,9 @@ extenddb uses a dual-database architecture: - **Catalog database** (e.g., `extenddb_catalog`): Stores table metadata, account/user/group/role/policy definitions, access keys, settings, stream metadata, and metrics. Shared across all accounts. - **Data database** (e.g., `extenddb_catalog_data`): Stores user items, GSI/LSI data, and stream records. Each table gets its own PostgreSQL table. -On PostgreSQL and SQLite the catalog is versioned by its migration ledger, the `extenddb_migrations` table: the catalog version is the highest migration recorded in every store, migration 4 for this release, and `serve` reads the catalog's status at startup and refuses a catalog that is behind it or carries a breaking migration it does not know. The Cassandra catalog version is 0.0.4, stored in the `settings` table under the key `catalog_version`; Cassandra has no startup check (see the Upgrade Manual). -The MongoDB backend tracks its own catalog version, 0.0.2. -A mismatch between the compiled-in version and the stored one prevents a PostgreSQL or SQLite server from starting; run `extenddb migrate` to upgrade. +On PostgreSQL and SQLite the catalog is versioned by its migration ledger, the `extenddb_migrations` table: the catalog version is the highest migration recorded in every store, migration 5 for this release, and `serve` reads the catalog's status at startup and refuses a catalog that is behind it or carries a breaking migration it does not know. +The MongoDB and Cassandra backends track their own catalog version string, 0.0.5 for this release, and `serve` refuses a catalog at another version. +On every backend a catalog at another version than the binary's prevents the server from starting; run `extenddb migrate --yes` to upgrade (see the Upgrade Manual). ## Pluggable Architecture diff --git a/docs/manuals/05-admin-guide.md b/docs/manuals/05-admin-guide.md index 1c2589181..134d05218 100755 --- a/docs/manuals/05-admin-guide.md +++ b/docs/manuals/05-admin-guide.md @@ -90,7 +90,7 @@ These settings require a server restart to take effect. |-----|---------|-------------| | `bind_addr` | `127.0.0.1` | Network interface to bind | | `port` | `18443` | HTTP port | -| `region` | `us-east-1` | AWS region for ARN generation. Baked into every backup ARN the server issues, so it must stay within `[a-z0-9-]` (a value outside that class makes the server's own backup ARNs "not valid"; the server logs a warning at startup). A backup ARN of the caller's account in another region is resolved in the server's region first, then as written; see the upgrade manual before changing this on a deployment with backups | +| `region` | `us-east-1` | AWS region for ARN generation. Baked into every backup ARN the server issues, so it must stay within `[a-z0-9-]` (under a value outside that class `CreateBackup` is refused with a `ValidationException` naming `server.region`, since no backup ARN can be formed; backups taken before the upgrade under such a region keep an ARN `DescribeBackup`, `DeleteBackup`, and `RestoreTableFromBackup` refuse as "not valid"; the server logs a warning at startup). A backup ARN of the caller's account in another region is resolved in the server's region first, then as written; see the upgrade manual before changing this on a deployment with backups | #### [storage] @@ -106,9 +106,9 @@ These settings require a server restart to take effect. | `pool_size` | `20` | Maximum concurrent database connections (minimum: 10) | | `catalog_pool_size` | (= `pool_size`) | Maximum connections for management/authz pool (minimum: 10) | -Restores on PostgreSQL are bounded per server process: at most 4 per account and at most half of `pool_size` in total (after its minimum of 10 is applied, capped at 16; 10 with the default `pool_size` of 20). A restore that cannot get both within 30 seconds is refused with `LimitExceededException`. Each running restore holds one connection from this pool for its whole copy, plus one dedicated connection outside the pool for its lock. +Backup and restore jobs are bounded per server process by `[backup] max_concurrent_jobs` (default 4, below), not per account, and a request over the bound is refused at once with `LimitExceededException`. The server raises `pool_size` and `catalog_pool_size` to at least `2 * max_concurrent_jobs + 10` and logs the raise with both numbers, so every job the bound admits has a connection. Each running restore holds one connection from this pool for its whole copy, plus one dedicated connection outside the pool for its claim. -`RestoreTableFromBackup` on PostgreSQL holds a session-level advisory lock for the whole copy, so ExtendDB must reach the catalog database directly or through a session-mode pooler. Behind a transaction-mode pooler (for example PgBouncer with `pool_mode = transaction`) restore locks are not released and the abandoned-restore sweep can remove a restore that is still running; do not run restores through one. +`RestoreTableFromBackup` on PostgreSQL holds a session-level advisory lock (the job's claim) for the whole copy, so ExtendDB must reach the catalog database directly or through a session-mode pooler. Behind a transaction-mode pooler (for example PgBouncer with `pool_mode = transaction`) the lock lands on a pooled server connection and is not released with the job, so later jobs can be refused as already claimed; do not run backups or restores through one. #### [storage.sqlite] @@ -117,6 +117,20 @@ Restores on PostgreSQL are bounded per server process: at most 4 per account and | `path` | `extenddb.sqlite` | Database file (`init --sqlite-path` writes it); `:memory:` for an ephemeral database | | `pool_size` | `10` | Maximum connections. Writes are serialized by the engine whatever the size. A file database raises `pool_size = 1` to 2 and logs it at info (`SQLite pool_size 1 raised to 2 for a file-backed database: ...`), because a backup holds one reader connection for its whole duration while its batches write through another. Use 3 or more so a running backup does not contend with ordinary requests for connections | +#### [backup] + +The knobs backup and restore jobs and the reconciler read; every key has a default, and the section may be left out. + +| Key | Default | Description | +|-----|---------|-------------| +| `data_file_target_bytes` | `67108864` | Uncompressed bytes per backup data file (minimum 32 MiB); read by the store-backed backup writer | +| `heartbeat_interval_secs` | `15` | The job heartbeat timer. The in-catalog backup and restore paths of this release heartbeat every 15 s whatever this says | +| `stale_job_after_secs` | `300` | Heartbeat age after which a job's owner is dead and the reconciler fails the job and removes its restore target. Must be at least three heartbeat periods, where the period is the larger of `heartbeat_interval_secs` and the 15 s the jobs of this release heartbeat at (so 45 at the least, whatever `heartbeat_interval_secs` says); on SQLite a long write transaction delays a heartbeat, so keep a margin above that floor | +| `failed_row_retention_secs` | `604800` | How long FAILED backup rows are kept for the operator (7 days) | +| `reconcile_interval_secs` | `60` | The reconciler's cycle; it also runs once at startup | +| `max_concurrent_jobs` | `4` | Backup and restore jobs this instance runs at once (at most 1024); a request over it is `LimitExceededException`. On PostgreSQL the pool sizes are raised to `2 * max_concurrent_jobs + 10` when smaller | +| `spool_path` | (unset) | Restore download area for an S3 store, used by a later release | + #### [auth] | Key | Default | Description | diff --git a/docs/manuals/07-upgrade-manual.md b/docs/manuals/07-upgrade-manual.md index f238d5d08..142302fbd 100755 --- a/docs/manuals/07-upgrade-manual.md +++ b/docs/manuals/07-upgrade-manual.md @@ -4,7 +4,7 @@ ## Current Status -This release versions the PostgreSQL and SQLite catalogs by a migration ledger and carries catalog migrations 1 to 4, which `extenddb migrate --list` prints. **Every PostgreSQL and SQLite deployment created by a 0.1.x release runs `extenddb migrate --yes` once** before a server of this release starts on it: the server refuses a catalog with pending migrations. Migration 1 brings the catalog of every 0.1.x release to the v1.0 schema. Cassandra stays at catalog 0.0.4 and MongoDB at 0.0.2, and their sections below apply unchanged. +This release versions the PostgreSQL and SQLite catalogs by a migration ledger and carries catalog migrations 1 to 5, which `extenddb migrate --list` prints. **Every PostgreSQL and SQLite deployment created by a 0.1.x release runs `extenddb migrate --yes` once** before a server of this release starts on it: the server refuses a catalog with pending migrations. Migration 1 brings the catalog of every 0.1.x release to the v1.0 schema. Cassandra and MongoDB keep their version strings and both move to catalog 0.0.5 (Cassandra `0.0.4 -> 0.0.5` through `V006__backup_catalog_index.cql`, MongoDB `0.0.2 -> 0.0.5` through its step `002_backup_catalog_index`), the same change as migration 5: run `extenddb migrate --yes` once on each with every pre-upgrade server stopped, as [the section below](#cassandra-and-mongodb-catalog-005) says. A backup that a 0.1.x release took can restore without its global and local secondary indexes: a SQLite backup taken by 0.1.14 records only the table's key schema and attribute definitions, and its restore creates the table with its items and no index. Take a new backup of each table after upgrading to v1.0, and keep the 0.1.x backups for their items only. @@ -37,7 +37,24 @@ Running `extenddb migrate --yes` on an up-to-date catalog changes nothing. Exit status 6 means the migration lock, or a running server, was still held when `--lock-timeout` ran out; run the command again once it is free. Every refusal prints the command that resolves it. A dev build's `serve` applies a Behind or Uninitialized catalog at startup instead of refusing it. The `serve` codes are those of `serve --foreground`; in daemon mode `serve` exits 1 and writes the refusal to syslog. -A migration is additive when the previous release serves the catalog correctly with it applied, and breaking otherwise. Migrations 1 to 4 are breaking: do not run a 0.1.x server against a migrated catalog. A 0.1.x binary that writes to a migrated catalog (its `migrate --yes`, or a SQLite dev server) leaves it WrittenByOlderRelease. +A migration is additive when the previous release serves the catalog correctly with it applied, and breaking otherwise. Migrations 1 to 5 are breaking: do not run a 0.1.x server against a migrated catalog. A 0.1.x binary that writes to a migrated catalog (its `migrate --yes`, or a SQLite dev server) leaves it WrittenByOlderRelease. + +Migration 5 adds the catalog index of backups kept in a backup store (`backup_index`), turns each restore registration into a restore job record (`table_restores` gains its status, kind, claim and progress columns), adds the maintenance claim record and writes the catalog's random id. Stop every server built before this migration before running `extenddb migrate --yes`: such a server neither reads `backup_index` nor marks the restore registrations it writes, so a restore it ran during the upgrade would be owned by nobody. A restore in flight at that stop is left as a RUNNING job record whose owner is dead; the first reconciler pass of the new server (within `backup.reconcile_interval_secs`, default 60 s) marks it FAILED and removes its CREATING target, so the name is free again and the table has to be restored anew. + +A rollback to the previous binary needs two things: the catalog restored from the pre-upgrade dump (the previous binary refuses a catalog carrying migration 5, exit status 4), and the `[backup]` section removed from the configuration file, which the previous binary refuses as an unknown field before it reaches the catalog (`unknown field 'backup'`). No backup store exists yet in this release, so the dump loses nothing a server wrote outside the catalog. + +### Cassandra and MongoDB: catalog 0.0.5 + +Cassandra and MongoDB migrate through their bootstrappers, not the ledger, and keep a version string; this release moves both to `0.0.5` with the change migration 5 makes on the SQL backends: Cassandra's `V006__backup_catalog_index.cql` (the index columns and `backup_index_by_status`, the restore job columns on `tables`, `backup_maintenance`, the catalog id) and MongoDB's step `002_backup_catalog_index` (the `backup_index` and `backup_maintenance` collections and their indexes, the `restore_job` record on the restore targets, the catalog id), each recorded in `schema_history`. + +```bash +extenddb stop --config extenddb.toml # every server of the previous release +extenddb migrate --config extenddb.toml # exit 3: catalog 0.0.4 -> 0.0.5 (Cassandra) or 0.0.2 -> 0.0.5 (MongoDB) +extenddb migrate --config extenddb.toml --yes +extenddb serve --config extenddb.toml +``` + +Stop every pre-upgrade server first, for the reason given for migration 5: a server of the previous release neither reads the index nor marks the restore registrations it writes. A restore in flight at that stop is backfilled as a RUNNING job record whose owner is dead, and the first reconciler pass of the new server fails it and removes its CREATING target. On both backends `serve` now refuses to start on a catalog whose version is not the one it serves (exit status 1, the message naming `extenddb migrate --yes`); the previous release's Cassandra server started on an unmigrated catalog and refused only the backup operations, logging an abandoned-restore sweep error every 15 seconds, and this release has no such sweep: the reconciler runs every `backup.reconcile_interval_secs`. The Cassandra `migrate` takes the admin credentials as the [0.0.4 section](#cassandra) shows. On a 0.0.4 Cassandra catalog the step takes seconds (a filtered scan of `tables` for restore registrations); on a 0.0.2 MongoDB catalog likewise. On Cassandra the version is written last, after the step and the `schema_history` row: a `migrate --yes` that fails in the step leaves the catalog at 0.0.4 with V006 unrecorded, `serve` keeps refusing it, and the next `migrate --yes` runs V006 again (every statement of it is idempotent) and finishes. Neither upgrade is reversible in place; take a snapshot of the catalog with your usual tooling before migrating. ## Writing a New Migration @@ -182,7 +199,7 @@ To roll back, stop the server, put the copied files back in place of the current On SQLite, `extenddb serve` reads the catalog's migration ledger before it opens the database and refuses a catalog with pending migrations, exiting 3 under `--foreground` with each pending migration and the `extenddb migrate --config --yes` command to run (in daemon mode it exits 1 and writes the refusal to syslog); a dev build applies them at startup instead, taking the same copy. -On startup the SQLite server removes any restore target left in `CREATING` by a server that stopped mid-copy, which is only safe because one server at a time can hold the file. A backup keeps a WAL read snapshot open while it copies; checkpoints cannot reclaim pages needed by that snapshot, so allow disk headroom for WAL growth proportional to the backup duration and concurrent write volume. Set `pool_size` to 3 or more (see the Admin Guide); the server runs at most `pool_size / 2 - 1` backups at once (4 with the default of 10) and answers `LimitExceededException` to a `CreateBackup` that waited 30 seconds for a slot. Running backups hold one connection each for their read snapshot plus one shared writer (`n` backups hold `n + 1` connections), so from `pool_size` 4 up a flood of backups leaves at least half the pool to other requests; with a `pool_size` of 2 or 3 the one admitted backup holds two of them. A GSI added by `UpdateTable` with `ProvisionedThroughput` on a `PROVISIONED` table made every `DescribeTable` of that table fail on earlier builds (`Internal("index pt: missing field NumberOfDecreasesToday")`); this build writes the index's throughput in the shape `DescribeTable` reads and reads the shape earlier builds wrote (as the same units with `NumberOfDecreasesToday` 0), so such a table is describable again after the upgrade without any change to the database file. +On startup the SQLite server removes any restore target left in `CREATING` by a server that stopped mid-copy, which is only safe because one server at a time can hold the file. A backup keeps a WAL read snapshot open while it copies; checkpoints cannot reclaim pages needed by that snapshot, so allow disk headroom for WAL growth proportional to the backup duration and concurrent write volume. Set `pool_size` to 3 or more (see the Admin Guide); the server runs at most `pool_size / 2 - 1` backups at once (4 with the default of 10) and answers `LimitExceededException` at once to a `CreateBackup` that finds no slot free. Running backups hold one connection each for their read snapshot plus one shared writer (`n` backups hold `n + 1` connections), so from `pool_size` 4 up a flood of backups leaves at least half the pool to other requests; with a `pool_size` of 2 or 3 the one admitted backup holds two of them. A GSI added by `UpdateTable` with `ProvisionedThroughput` on a `PROVISIONED` table made every `DescribeTable` of that table fail on earlier builds (`Internal("index pt: missing field NumberOfDecreasesToday")`); this build writes the index's throughput in the shape `DescribeTable` reads and reads the shape earlier builds wrote (as the same units with `NumberOfDecreasesToday` 0), so such a table is describable again after the upgrade without any change to the database file. #### Cassandra @@ -196,7 +213,7 @@ extenddb serve --config extenddb.toml `` is the role `init` used (`--cassandra-user`). `EXTENDDB_CASSANDRA_PASSWORD` keeps the password out of process arguments; `--cassandra-pass` still works and wins if both are given. Without a user and password `migrate` connects as the operating-system user with no password, which a cluster using `PasswordAuthenticator` refuses; the error reports a connection timeout. -There is no startup version gate on Cassandra. A 0.0.4 server started before `extenddb migrate --yes` serves every table and item operation normally; CreateBackup, RestoreTableFromBackup, and DeleteBackup are refused with a `ValidationException` ("Backups are unavailable until the ExtendDB catalog is migrated: ... Run `extenddb migrate --yes` with the admin credentials.") until the catalog is migrated; a running server notices the migration without a restart, and the server logs an abandoned-restore sweep error (`Undefined column name restore_source_backup_arn`) at startup and every 15 seconds. ListBackups and DescribeBackup work and show backups taken before the upgrade. V005 only adds columns. This manual has no tested rollback for Cassandra; take a snapshot of the catalog and account keyspaces with your usual Cassandra tooling before migrating. +The 0.0.4 server had no startup version gate on Cassandra. A 0.0.4 server started before `extenddb migrate --yes` serves every table and item operation normally; CreateBackup, RestoreTableFromBackup, and DeleteBackup are refused with a `ValidationException` ("Backups are unavailable until the ExtendDB catalog is migrated: ... Run `extenddb migrate --yes` with the admin credentials.") until the catalog is migrated; a running server notices the migration without a restart, and the 0.0.4 server logs an abandoned-restore sweep error (`Undefined column name restore_source_backup_arn`) at startup and every 15 seconds (this release's server has no such sweep, and does not start on a catalog behind it: see [catalog 0.0.5](#cassandra-and-mongodb-catalog-005)). ListBackups and DescribeBackup work and show backups taken before the upgrade. V005 only adds columns. This manual has no tested rollback for Cassandra; take a snapshot of the catalog and account keyspaces with your usual Cassandra tooling before migrating. Upgrade every server to 0.0.4 before restoring a backup. A 0.0.3 server still running against the migrated catalog does not know about restores in progress: its DeleteTable deletes a table that is being restored and its DeleteBackup deletes a backup a restore is copying from, and that restore then fails and its target table is removed. @@ -395,12 +412,14 @@ region the caller wrote, so after a change a statement must name the new region admin guide covers this). Keep `server.region` within `[a-z0-9-]`. The `BackupArn` parser accepts only -that class, as the service does, so a region such as `LOCAL`, `us_east_1`, or -`dev.local` makes every backup ARN the server issues `ValidationException` -"Invalid Request: BackupArn is not valid" on `DescribeBackup`, `DeleteBackup`, -and `RestoreTableFromBackup`, and `ListBackups` refuses the cursor it hands out, -so every listing stops at its first page. The server starts with such a region -and logs a warning naming the consequence at startup. +that class, as the service does, so under a region such as `LOCAL`, `us_east_1`, +or `dev.local` no backup ARN can be formed: `CreateBackup` is refused with +`ValidationException` naming `server.region`, every backup an earlier release +issued under such a region keeps an ARN that is `ValidationException` "Invalid +Request: BackupArn is not valid" on `DescribeBackup`, `DeleteBackup`, and +`RestoreTableFromBackup`, and `ListBackups` refuses a cursor that is such an ARN +(a listing of backups created under a valid region pages). The server starts +with such a region and logs a warning naming the consequence at startup. ### v0.1.6 — IAM: bare operators on multivalued condition keys are no-ops (BR-7085) diff --git a/docs/manuals/11-deployment-guide.md b/docs/manuals/11-deployment-guide.md index 77624ce38..b27176654 100755 --- a/docs/manuals/11-deployment-guide.md +++ b/docs/manuals/11-deployment-guide.md @@ -211,7 +211,7 @@ extenddb init \ ### Database (PostgreSQL) - [ ] Use a dedicated PostgreSQL user for extenddb with minimal privileges -- [ ] Configure PostgreSQL `max_connections` ≥ extenddb `pool_size` + 3 +- [ ] Configure PostgreSQL `max_connections` ≥ `2 * pool_size + catalog_pool_size + max_concurrent_jobs`, plus one per overlapping vector index build and one during a schema migration - [ ] Enable PostgreSQL TLS - [ ] Set up automated backups (pg_dump, WAL archiving, or managed service snapshots) - [ ] Monitor PostgreSQL disk usage, connection count, and query performance @@ -286,7 +286,7 @@ pool_size = 50 # Increase for higher concurrency Ensure PostgreSQL `max_connections` accommodates the total pool size plus overhead. -Backup and restore work is admitted against the pool so it cannot take every connection from ordinary requests. On PostgreSQL and MongoDB one server runs at most 4 `RestoreTableFromBackup` copies per account and `min(16, pool/2)` in total (PostgreSQL: `pool_size`; MongoDB: the engine's driver pool, `maxPoolSize` in the connection string or `max_connections`, default 10, so 5 restores); a restore that waits more than 30 seconds for a slot is refused with `LimitExceededException` and creates nothing. On SQLite one server runs at most `pool_size / 2 - 1` `CreateBackup` copies at once (between 1 and 16; 4 for the default `pool_size` of 10), each holding its snapshot connection for its duration, and refuses a backup that waited 30 seconds with `LimitExceededException`. A MongoDB restore request that is not registered within 60 seconds answers an internal error while the restore goes on in the background; the MongoDB driver does not enforce `waitQueueTimeoutMS`, so the admission bound is what keeps a flood of restores from holding every pooled connection. +Backup and restore work is bounded per server by `[backup] max_concurrent_jobs` (default 4): `CreateBackup` and `RestoreTableFromBackup` jobs together, on every backend, and a request over the bound is refused at once with `LimitExceededException` and creates nothing (there is no wait). On PostgreSQL each running job holds one pooled connection for its copy plus one dedicated connection outside the pool for its claim, and startup raises `pool_size` and `catalog_pool_size` to at least `2 * max_concurrent_jobs + 10` with a warning naming both numbers (the admin guide's `[backup]` table). On SQLite a `CreateBackup` also holds its snapshot connection for its duration, and a server admits at most `pool_size / 2 - 1` of them at once (between 1 and 16; 4 for the default `pool_size` of 10), so that half the pool stays free for ordinary requests; a `CreateBackup` over that bound is refused at once with the same `LimitExceededException`. A MongoDB restore request that is not registered within 60 seconds answers an internal error while the restore goes on in the background; the MongoDB driver does not enforce `waitQueueTimeoutMS`, so the admission bound is what keeps a flood of restores from holding every pooled connection. ### PostgreSQL Tuning @@ -295,7 +295,7 @@ Key PostgreSQL settings for extenddb workloads: - `shared_buffers`: 25% of available RAM - `effective_cache_size`: 75% of available RAM - `work_mem`: 64MB (for sort operations in Query/Scan) -- `max_connections`: ≥ `2 * pool_size + catalog_pool_size`, plus one per vector index build you expect to overlap and one during a schema migration. `pool_size` sizes both the catalog and the data pool, and `max_connections` is per cluster, so the default pool sizes need 60 of PostgreSQL's default 100 +- `max_connections`: ≥ `2 * pool_size + catalog_pool_size + max_concurrent_jobs`, plus one per vector index build you expect to overlap and one during a schema migration. `pool_size` sizes both the catalog and the data pool, each backup or restore job holds one claim connection outside every pool, and `max_connections` is per cluster, so the default pool sizes and the default `max_concurrent_jobs` of 4 need 64 of PostgreSQL's default 100 ### Monitoring Queries diff --git a/docs/schema/postgres.txt b/docs/schema/postgres.txt index f3164d0a5..24671a937 100644 --- a/docs/schema/postgres.txt +++ b/docs/schema/postgres.txt @@ -3,7 +3,9 @@ table access_keys table accounts table admin_users table backup_definitions +table backup_index table backup_items +table backup_maintenance table backups table continuous_backups table extenddb_migrations @@ -42,10 +44,32 @@ column admin_users.password_hash: text NOT NULL column admin_users.created_at: timestamp with time zone NOT NULL DEFAULT now() column backup_definitions.backup_arn: text NOT NULL column backup_definitions.definition: jsonb NOT NULL +column backup_index.account_id: text NOT NULL +column backup_index.backup_id: text NOT NULL +column backup_index.table_name: text NOT NULL +column backup_index.backup_arn: text NOT NULL COLLATE "C" +column backup_index.backup_name: text NOT NULL +column backup_index.status: text NOT NULL +column backup_index.store_kind: text NOT NULL +column backup_index.store_root: text NOT NULL +column backup_index.store_id: text NOT NULL +column backup_index.size_bytes: bigint NOT NULL DEFAULT 0 +column backup_index.item_count: bigint NOT NULL DEFAULT 0 +column backup_index.created_at: timestamp with time zone NOT NULL +column backup_index.completed_at: timestamp with time zone +column backup_index.failure_reason: text +column backup_index.source_table_details_json: jsonb NOT NULL +column backup_index.owner_generation: bigint NOT NULL +column backup_index.heartbeat_at: timestamp with time zone NOT NULL column backup_items.backup_arn: text NOT NULL column backup_items.pk: text NOT NULL column backup_items.sk: text column backup_items.item_data: jsonb NOT NULL +column backup_maintenance.singleton: boolean NOT NULL DEFAULT true +column backup_maintenance.holder: text NOT NULL +column backup_maintenance.owner_generation: bigint NOT NULL +column backup_maintenance.claimed_at: timestamp with time zone NOT NULL DEFAULT now() +column backup_maintenance.heartbeat_at: timestamp with time zone NOT NULL DEFAULT now() column backups.backup_arn: text NOT NULL column backups.backup_name: text NOT NULL column backups.table_id: text NOT NULL @@ -166,6 +190,18 @@ column table_restores.table_id: text NOT NULL column table_restores.source_backup_arn: text NOT NULL column table_restores.source_table_arn: text NOT NULL column table_restores.restore_date_time: timestamp with time zone NOT NULL +column table_restores.status: text NOT NULL +column table_restores.kind: text NOT NULL +column table_restores.store_kind: text +column table_restores.store_root: text +column table_restores.store_id: text +column table_restores.owner_generation: bigint NOT NULL +column table_restores.heartbeat_at: timestamp with time zone NOT NULL +column table_restores.spool_host: text +column table_restores.file_index: bigint +column table_restores.items_applied: bigint +column table_restores.failure_reason: text +column table_restores.completed_at: timestamp with time zone column tables.account_id: text NOT NULL column tables.table_name: text NOT NULL column tables.key_schema: jsonb NOT NULL @@ -208,6 +244,8 @@ index accounts_account_name_key: CREATE UNIQUE INDEX accounts_account_name_key O index accounts_pkey: CREATE UNIQUE INDEX accounts_pkey ON public.accounts USING btree (account_id) index admin_users_pkey: CREATE UNIQUE INDEX admin_users_pkey ON public.admin_users USING btree (admin_name) index backup_definitions_pkey: CREATE UNIQUE INDEX backup_definitions_pkey ON public.backup_definitions USING btree (backup_arn) +index backup_index_pkey: CREATE UNIQUE INDEX backup_index_pkey ON public.backup_index USING btree (account_id, backup_id) +index backup_maintenance_pkey: CREATE UNIQUE INDEX backup_maintenance_pkey ON public.backup_maintenance USING btree (singleton) index backups_pkey: CREATE UNIQUE INDEX backups_pkey ON public.backups USING btree (backup_arn) index continuous_backups_pkey: CREATE UNIQUE INDEX continuous_backups_pkey ON public.continuous_backups USING btree (account_id, table_name) index extenddb_migrations_pkey: CREATE UNIQUE INDEX extenddb_migrations_pkey ON public.extenddb_migrations USING btree (id) @@ -225,6 +263,9 @@ index iam_user_tags_pkey: CREATE UNIQUE INDEX iam_user_tags_pkey ON public.iam_u index iam_users_pkey: CREATE UNIQUE INDEX iam_users_pkey ON public.iam_users USING btree (account_id, user_name) index iam_users_user_arn_key: CREATE UNIQUE INDEX iam_users_user_arn_key ON public.iam_users USING btree (user_arn) index idempotency_tokens_pkey: CREATE UNIQUE INDEX idempotency_tokens_pkey ON public.idempotency_tokens USING btree (token) +index idx_backup_index_account_table: CREATE INDEX idx_backup_index_account_table ON public.backup_index USING btree (account_id, table_name) +index idx_backup_index_backup_id: CREATE UNIQUE INDEX idx_backup_index_backup_id ON public.backup_index USING btree (backup_id) +index idx_backup_index_store_status: CREATE INDEX idx_backup_index_store_status ON public.backup_index USING btree (store_kind, store_root, store_id, status) index idx_backup_items_arn: CREATE INDEX idx_backup_items_arn ON public.backup_items USING btree (backup_arn) index idx_backups_table: CREATE INDEX idx_backups_table ON public.backups USING btree (account_id, table_name) index idx_idempotency_tokens_created: CREATE INDEX idx_idempotency_tokens_created ON public.idempotency_tokens USING btree (created_at) @@ -234,6 +275,7 @@ index idx_metrics_bucket: CREATE INDEX idx_metrics_bucket ON public.metrics USIN index idx_stream_records_created: CREATE INDEX idx_stream_records_created ON public.stream_records USING btree (created_at) index idx_stream_shards_table: CREATE INDEX idx_stream_shards_table ON public.stream_shards USING btree (table_id) index idx_table_restores_backup: CREATE INDEX idx_table_restores_backup ON public.table_restores USING btree (source_backup_arn) +index idx_table_restores_status: CREATE INDEX idx_table_restores_status ON public.table_restores USING btree (status, heartbeat_at) index idx_tables_pending_transition: CREATE INDEX idx_tables_pending_transition ON public.tables USING btree (status_transition_at) WHERE (status_transition_at IS NOT NULL) index idx_vector_indexes_index_id: CREATE UNIQUE INDEX idx_vector_indexes_index_id ON public.vector_indexes USING btree (index_id) index indexes_pkey: CREATE UNIQUE INDEX indexes_pkey ON public.indexes USING btree (table_id, index_name) @@ -254,7 +296,10 @@ constraint accounts.accounts_pkey: PRIMARY KEY (account_id) constraint admin_users.admin_users_pkey: PRIMARY KEY (admin_name) constraint backup_definitions.backup_definitions_backup_arn_fkey: FOREIGN KEY (backup_arn) REFERENCES backups(backup_arn) ON DELETE CASCADE constraint backup_definitions.backup_definitions_pkey: PRIMARY KEY (backup_arn) +constraint backup_index.backup_index_pkey: PRIMARY KEY (account_id, backup_id) constraint backup_items.backup_items_backup_arn_fkey: FOREIGN KEY (backup_arn) REFERENCES backups(backup_arn) ON DELETE CASCADE +constraint backup_maintenance.backup_maintenance_pkey: PRIMARY KEY (singleton) +constraint backup_maintenance.backup_maintenance_singleton_check: CHECK (singleton) constraint backups.backups_pkey: PRIMARY KEY (backup_arn) constraint continuous_backups.continuous_backups_pkey: PRIMARY KEY (account_id, table_name) constraint extenddb_migrations.extenddb_migrations_pkey: PRIMARY KEY (id) diff --git a/docs/schema/sqlite.txt b/docs/schema/sqlite.txt index ab0b7ce86..6b993969b 100644 --- a/docs/schema/sqlite.txt +++ b/docs/schema/sqlite.txt @@ -1,3 +1,6 @@ +index idx_backup_index_account_table on backup_index: CREATE INDEX idx_backup_index_account_table ON backup_index (account_id, table_name) +index idx_backup_index_backup_id on backup_index: CREATE UNIQUE INDEX idx_backup_index_backup_id ON backup_index (backup_id) +index idx_backup_index_store_status on backup_index: CREATE INDEX idx_backup_index_store_status ON backup_index (store_kind, store_root, store_id, status) index idx_backup_items_arn on backup_items: CREATE INDEX idx_backup_items_arn ON backup_items (backup_arn) index idx_backups_table on backups: CREATE INDEX idx_backups_table ON backups (account_id, table_name) index idx_gsi_pending_claim on gsi_pending: CREATE INDEX idx_gsi_pending_claim ON gsi_pending (worker_partition, ready_at, id) @@ -8,13 +11,16 @@ index idx_metrics_bucket on metrics: CREATE INDEX idx_metrics_bucket ON metrics index idx_stream_records_created on stream_records: CREATE INDEX idx_stream_records_created ON stream_records (created_at) index idx_stream_shards_table on stream_shards: CREATE INDEX idx_stream_shards_table ON stream_shards (table_id) index idx_table_restores_backup on table_restores: CREATE INDEX idx_table_restores_backup ON table_restores (source_backup_arn) +index idx_table_restores_status on table_restores: CREATE INDEX idx_table_restores_status ON table_restores (status, heartbeat_at) index idx_tables_pending_transition on tables: CREATE INDEX idx_tables_pending_transition ON tables (status_transition_at) WHERE status_transition_at IS NOT NULL index idx_vector_indexes_index_id on vector_indexes: CREATE UNIQUE INDEX idx_vector_indexes_index_id ON vector_indexes (index_id) table access_keys on access_keys: CREATE TABLE access_keys ( access_key_id TEXT PRIMARY KEY, secret_key_encrypted BLOB NOT NULL, account_id TEXT NOT NULL, user_name TEXT NOT NULL, is_active INTEGER NOT NULL DEFAULT 1, created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')), FOREIGN KEY (account_id, user_name) REFERENCES iam_users(account_id, user_name) ON DELETE CASCADE ) table accounts on accounts: CREATE TABLE accounts ( account_id TEXT PRIMARY KEY, account_name TEXT NOT NULL UNIQUE, created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')) ) table admin_users on admin_users: CREATE TABLE admin_users ( admin_name TEXT PRIMARY KEY, password_hash TEXT NOT NULL, created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')) ) table backup_definitions on backup_definitions: CREATE TABLE backup_definitions ( backup_arn TEXT PRIMARY KEY REFERENCES backups(backup_arn) ON DELETE CASCADE, definition TEXT NOT NULL ) +table backup_index on backup_index: CREATE TABLE backup_index ( account_id TEXT NOT NULL, backup_id TEXT NOT NULL, table_name TEXT NOT NULL, backup_arn TEXT NOT NULL, backup_name TEXT NOT NULL, status TEXT NOT NULL, store_kind TEXT NOT NULL, store_root TEXT NOT NULL, store_id TEXT NOT NULL, size_bytes INTEGER NOT NULL DEFAULT 0, item_count INTEGER NOT NULL DEFAULT 0, created_at TEXT NOT NULL, completed_at TEXT, failure_reason TEXT, source_table_details_json TEXT NOT NULL, owner_generation INTEGER NOT NULL, heartbeat_at TEXT NOT NULL, PRIMARY KEY (account_id, backup_id) ) table backup_items on backup_items: CREATE TABLE backup_items ( id INTEGER PRIMARY KEY AUTOINCREMENT, backup_arn TEXT NOT NULL REFERENCES backups(backup_arn) ON DELETE CASCADE, pk TEXT NOT NULL, sk TEXT, item_data TEXT NOT NULL ) +table backup_maintenance on backup_maintenance: CREATE TABLE backup_maintenance ( singleton INTEGER PRIMARY KEY CHECK (singleton = 1), holder TEXT NOT NULL, owner_generation INTEGER NOT NULL, claimed_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')), heartbeat_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')) ) table backups on backups: CREATE TABLE backups ( backup_arn TEXT PRIMARY KEY, backup_name TEXT NOT NULL, table_id TEXT NOT NULL, table_name TEXT NOT NULL, account_id TEXT NOT NULL, backup_status TEXT NOT NULL DEFAULT 'AVAILABLE', backup_type TEXT NOT NULL DEFAULT 'USER', backup_size_bytes INTEGER NOT NULL DEFAULT 0, item_count INTEGER NOT NULL DEFAULT 0, key_schema TEXT NOT NULL, attribute_definitions TEXT NOT NULL, billing_mode TEXT NOT NULL DEFAULT 'PAY_PER_REQUEST', provisioned_throughput TEXT, stream_specification TEXT, created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')) ) table continuous_backups on continuous_backups: CREATE TABLE continuous_backups ( account_id TEXT NOT NULL, table_name TEXT NOT NULL, pitr_enabled INTEGER NOT NULL DEFAULT 0, earliest_restorable TEXT, latest_restorable TEXT, PRIMARY KEY (account_id, table_name) ) table extenddb_migrations on extenddb_migrations: CREATE TABLE extenddb_migrations (id INTEGER PRIMARY KEY, name TEXT NOT NULL, kind TEXT NOT NULL, checksum TEXT NOT NULL, outcome TEXT NOT NULL, notes TEXT NOT NULL DEFAULT '', applied_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')), applied_by TEXT NOT NULL) @@ -37,7 +43,7 @@ table seq_counters on seq_counters: CREATE TABLE seq_counters ( name TEXT PRIMAR table settings on settings: CREATE TABLE settings ( key TEXT PRIMARY KEY, value TEXT NOT NULL ) table stream_records on stream_records: CREATE TABLE stream_records ( shard_id TEXT NOT NULL REFERENCES stream_shards(shard_id) ON DELETE CASCADE, sequence_number TEXT NOT NULL, table_id TEXT NOT NULL, event_name TEXT NOT NULL, record_data TEXT NOT NULL, created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')), PRIMARY KEY (shard_id, sequence_number) ) table stream_shards on stream_shards: CREATE TABLE stream_shards ( shard_id TEXT PRIMARY KEY, table_id TEXT NOT NULL REFERENCES tables(table_id) ON DELETE CASCADE, parent_shard_id TEXT, starting_sequence_number TEXT NOT NULL, ending_sequence_number TEXT, created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')) ) -table table_restores on table_restores: CREATE TABLE table_restores ( table_id TEXT PRIMARY KEY REFERENCES tables(table_id) ON DELETE CASCADE, source_backup_arn TEXT NOT NULL, source_table_arn TEXT, restore_date_time TEXT NOT NULL ) +table table_restores on table_restores: CREATE TABLE table_restores ( table_id TEXT PRIMARY KEY REFERENCES tables(table_id) ON DELETE CASCADE, source_backup_arn TEXT NOT NULL, source_table_arn TEXT, restore_date_time TEXT NOT NULL , status TEXT, kind TEXT, store_kind TEXT, store_root TEXT, store_id TEXT, owner_generation INTEGER, heartbeat_at TEXT, spool_host TEXT, file_index INTEGER, items_applied INTEGER, failure_reason TEXT, completed_at TEXT) table tables on tables: CREATE TABLE tables ( account_id TEXT NOT NULL REFERENCES accounts(account_id) ON DELETE CASCADE, table_name TEXT NOT NULL, key_schema TEXT NOT NULL, attribute_definitions TEXT NOT NULL, billing_mode TEXT NOT NULL DEFAULT 'PAY_PER_REQUEST', provisioned_throughput TEXT, stream_specification TEXT, table_status TEXT NOT NULL DEFAULT 'CREATING', creation_date_time TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')), table_size_bytes INTEGER NOT NULL DEFAULT 0, item_count INTEGER NOT NULL DEFAULT 0, table_arn TEXT NOT NULL, table_id TEXT NOT NULL, ttl_attribute TEXT, deletion_protection_enabled INTEGER NOT NULL DEFAULT 0, status_transition_at TEXT, stream_label TEXT, ttl_index_ready INTEGER NOT NULL DEFAULT 0, table_class TEXT, sse_specification TEXT, on_demand_throughput TEXT, PRIMARY KEY (account_id, table_name), CONSTRAINT tables_table_id_unique UNIQUE (table_id) ) table tags on tags: CREATE TABLE tags ( resource_arn TEXT NOT NULL, tag_key TEXT NOT NULL, tag_value TEXT NOT NULL, PRIMARY KEY (resource_arn, tag_key) ) table vector_indexes on vector_indexes: CREATE TABLE vector_indexes ( table_id TEXT NOT NULL, index_id TEXT NOT NULL, index_name TEXT NOT NULL, dimensions INTEGER NOT NULL, distance_function TEXT NOT NULL, vector_attribute TEXT NOT NULL, search_schema TEXT, projection TEXT NOT NULL, index_status TEXT NOT NULL DEFAULT 'CREATING', backfilling INTEGER, -- Items the backfill skipped because their stored bytes cannot enter the -- index (unparseable row, malformed or wrong-dimension vector). NULL until -- a backfill has completed; 0 afterwards when nothing was skipped. Kept so -- an operator can see that an ACTIVE index deliberately omits rows, rather -- than the build looping forever on them or dying part-way. skipped_item_count INTEGER, PRIMARY KEY (table_id, index_name), CONSTRAINT vector_indexes_table_id_fkey FOREIGN KEY (table_id) REFERENCES tables(table_id) ON DELETE CASCADE, CONSTRAINT chk_vector_dimensions_positive CHECK (dimensions > 0), CONSTRAINT chk_vector_backfilling_bool CHECK (backfilling IS NULL OR backfilling IN (0, 1)), -- An ACTIVE index must not carry the member at all, which is what the -- service does. Enforced here as well as in core, so a bug in the backend -- cannot persist a state the wire contract forbids. CONSTRAINT chk_vector_active_has_no_backfilling CHECK (index_status <> 'ACTIVE' OR backfilling IS NULL) ) diff --git a/extenddb.sample.toml b/extenddb.sample.toml index 91869028a..2443ccd3d 100755 --- a/extenddb.sample.toml +++ b/extenddb.sample.toml @@ -53,13 +53,15 @@ # one against the data database, which `init` # always creates separately. Total PostgreSQL # connections used: 2 * pool_size + - # catalog_pool_size, plus one per vector index + # catalog_pool_size + [backup] max_concurrent_jobs + # (one claim connection per running backup or + # restore job), plus one per vector index # build running on this server and one while a # schema migration runs. max_connections is # per cluster, not per database, so both pools - # draw on the same budget. The build session and - # the migration connection sit OUTSIDE every - # pool, for two different reasons. + # draw on the same budget. The claim connections, + # the build session and the migration connection + # sit OUTSIDE every pool, for different reasons. # A build's ownership lock is SESSION-scoped, so # it needs a session that ends when the build # ends: a pooled connection returns to the pool @@ -71,10 +73,10 @@ # otherwise let the lock move between backends. # Increase for higher concurrency; ensure # PostgreSQL max_connections >= 2 * pool_size - # + catalog_pool_size + the number of vector - # index builds you expect to overlap. The - # defaults already need 60 of PostgreSQL's - # default 100. + # + catalog_pool_size + max_concurrent_jobs + + # the number of vector index builds you expect + # to overlap. The defaults already need 64 of + # PostgreSQL's default 100. # catalog_pool_size = 20 # Maximum concurrent connections for the # management/catalog pool (authz, IAM, console). # Defaults to pool_size if not set. @@ -173,6 +175,21 @@ # Read logs with: journalctl -t extenddb # No configuration needed — output always goes to syslog. +# [backup] +# Backup and restore jobs and the reconciler. Every key has a default; the +# section may be left out. stale_job_after_secs must be at least three +# heartbeat periods, the larger of heartbeat_interval_secs and the 15 s the +# jobs of this release heartbeat at (45 at the least); max_concurrent_jobs is +# at most 1024; on PostgreSQL pool_size and catalog_pool_size are raised to +# at least 2 * max_concurrent_jobs + 10. +# data_file_target_bytes = 67108864 # Uncompressed bytes per data file (minimum 32 MiB) +# heartbeat_interval_secs = 15 # Job heartbeat timer +# stale_job_after_secs = 300 # Heartbeat age after which a job's owner is dead +# failed_row_retention_secs = 604800 # FAILED tombstones kept for the operator (7 days) +# reconcile_interval_secs = 60 # Reconciler cycle +# max_concurrent_jobs = 4 # Backup and restore jobs running at once on this instance +# spool_path = "/var/lib/extenddb/spool" # Restore download area for an S3 store + # Import/Export file operations are disabled by default (secure default). # Configure allowed directories separately for imports and exports. # All file paths are canonicalized and must resolve under one of the diff --git a/tests/test_resource_tag_migration.py b/tests/test_resource_tag_migration.py index a5ffcf43c..f37a54ec7 100644 --- a/tests/test_resource_tag_migration.py +++ b/tests/test_resource_tag_migration.py @@ -290,14 +290,31 @@ def deployment(tmp_path): server.stop() +BACKUP_CATALOG_INDEX_COLUMNS = ( + "status", "kind", "store_kind", "store_root", "store_id", "owner_generation", + "heartbeat_at", "spool_host", "file_index", "items_applied", "failure_reason", + "completed_at", +) + + def _as_released_catalog(sqlite_path: str) -> None: """Make a catalog this binary created read as one a release before 1.0 left: no ledger rows (the runner reads an empty ledger as it reads a - missing one), a dotted catalog_version, and the marker row every released - baseline seeded. The runner then applies every migration, the - condition-key rewrite included, as it does on a released catalog.""" + missing one), a dotted catalog_version, the marker row every released + baseline seeded, and the baseline's shape for every table migration 1 + checks: the objects migration 5 (backup_catalog_index) adds are removed, + since no release before 1.0 had them and the check after migration 1 + refuses a `table_restores` carrying its twelve job columns. The runner + then applies every migration, the condition-key rewrite included, as it + does on a released catalog.""" conn = sqlite3.connect(sqlite_path) try: + conn.execute("DROP INDEX IF EXISTS idx_table_restores_status") + for column in BACKUP_CATALOG_INDEX_COLUMNS: + conn.execute(f"ALTER TABLE table_restores DROP COLUMN {column}") + conn.execute("DROP TABLE IF EXISTS backup_index") + conn.execute("DROP TABLE IF EXISTS backup_maintenance") + conn.execute("DELETE FROM settings WHERE key = 'catalog_id'") conn.execute("DELETE FROM extenddb_migrations") conn.execute( "UPDATE settings SET value = '0.0.3' WHERE key = 'catalog_version'"