feat(backup): catalog index, BackupCatalog, JobClaim, legacy reader [backup/restore 6]

.github/workflows/integration-cassandra.yml

@@ -92,8 +92,10 @@ jobs:
       - name: Direct storage-trait integration tests
         # Ported from the plug-in repo's tests/rust suite. Parallel-safe:
         # each test provisions its own account and keyspace; the shared
-        # control-plane setting test restores what it mutates.
-        run: cargo test -p extenddb-storage-cassandra --test direct_integration
+        # control-plane setting test restores what it mutates. The
+        # test-hooks feature compiles the tests that pause an operation or
+        # shorten a heartbeat (crates/storage-cassandra/src/test_hooks.rs).
+        run: cargo test -p extenddb-storage-cassandra --features test-hooks --test direct_integration
         env:
           EXTENDDB_TEST_CASSANDRA: required
 

CONTRIBUTING.md

@@ -291,26 +291,27 @@ change is code in `crates/storage-mongodb/src/bootstrapper.rs`.
 
 ### Example: a nullable column on `tables`
 
-Migration 5, `table_note`, adds a nullable `note TEXT` column to the catalog's
-`tables` table on SQLite and PostgreSQL.
+Migration 6, `table_note` (6 being the next free id at the time of writing;
+migration 5 is `backup_catalog_index`), adds a nullable `note TEXT` column to
+the catalog's `tables` table on SQLite and PostgreSQL.
 
 In `crates/storage/src/migrations/registry.rs`, at the end of the
 `migration_ids!` list:
 
 ```rust
     /// Adds a nullable `note` column to the `tables` catalog table.
-    TableNote = 5, "table_note",
+    TableNote = 6, "table_note",
         "add a nullable note column to the tables catalog";
 ```
 
 PostgreSQL has `ADD COLUMN IF NOT EXISTS`, so its body is a file,
-`crates/storage-postgres/migrations/0005_table_note.sql`:
+`crates/storage-postgres/migrations/0006_table_note.sql`:
 
 ```sql
 -- Copyright 2026 ExtendDB contributors
 -- SPDX-License-Identifier: Apache-2.0
 --
--- Migration 5, table_note, PostgreSQL catalog database: a nullable note
+-- Migration 6, table_note, PostgreSQL catalog database: a nullable note
 -- column on tables. IF NOT EXISTS makes a second run change nothing.
 
 ALTER TABLE tables ADD COLUMN IF NOT EXISTS note TEXT;
@@ -325,7 +326,7 @@ In `crates/storage-postgres/src/migrations/mod.rs`, the arm in `migration()`:
             bodies: vec![file_body!(
                 StoreKind::Catalog,
                 "migrations",
-                "0005_table_note.sql"
+                "0006_table_note.sql"
             )],
         },
 ```
@@ -336,19 +337,20 @@ and the id in the transactional arm of `runs_in()`:
         MigrationId::Baseline
         | MigrationId::ResourceTagConditionKeys
         | MigrationId::SortKeyDefinitions
+        | MigrationId::BackupCatalogIndex
         | MigrationId::TableNote => RunsIn::Transaction,
 ```
 
 SQLite has no `ADD COLUMN IF NOT EXISTS`, and a file with a plain
 `ADD COLUMN` fails when it runs a second time, so the SQLite body is code
 that checks for the column first.
-`crates/storage-sqlite/src/migrations/m0005_table_note.rs`:
+`crates/storage-sqlite/src/migrations/m0006_table_note.rs`:
 
 ```rust
 // Copyright 2026 ExtendDB contributors
 // SPDX-License-Identifier: Apache-2.0
 
-//! Migration 5, `table_note`, SQLite: a nullable `note` column on `tables`.
+//! Migration 6, `table_note`, SQLite: a nullable `note` column on `tables`.
 //!
 //! SQLite has no `ADD COLUMN IF NOT EXISTS`, so the body reads the table's
 //! columns first and adds the column only when it is missing. A second run
@@ -360,10 +362,10 @@ use extenddb_storage::migrations::{CodeBody, Done, MigrationError};
 use super::SqliteStores;
 
 /// This file, whose SHA-256 the checksum test pins.
-pub(crate) const SOURCE: &str = include_str!("m0005_table_note.rs");
+pub(crate) const SOURCE: &str = include_str!("m0006_table_note.rs");
 
 /// Recorded as this migration's checksum. Changes only with a new migration.
-const FINGERPRINT: &str = "0005_table_note/sqlite/1";
+const FINGERPRINT: &str = "0006_table_note/sqlite/1";
 
 pub(crate) struct TableNote;
 
@@ -402,7 +404,7 @@ beside the others:
 ```rust
 #[rustfmt::skip]
 #[allow(clippy::all, deprecated)]
-mod m0005_table_note;
+mod m0006_table_note;
 ```
 
 and the arm in `migration()`:
@@ -413,8 +415,8 @@ and the arm in `migration()`:
             convergent: true,
             bodies: vec![StoreBody::code(
                 StoreKind::Catalog,
-                &m0005_table_note::TableNote,
-                m0005_table_note::SOURCE,
+                &m0006_table_note::TableNote,
+                m0006_table_note::SOURCE,
             )],
         },
 ```
@@ -426,7 +428,7 @@ rows of `tables`:
 
 ```rust
 #[tokio::test]
-async fn migration_5_adds_the_note_column_to_tables() {
+async fn migration_6_adds_the_note_column_to_tables() {
     let db = temp_db("m5");
     load_dump(&db, DEV_0_1_14).await;
     run(&db).await.expect("migrate");
@@ -437,12 +439,12 @@ async fn migration_5_adds_the_note_column_to_tables() {
     )
     .await;
     text.push_str(&dump(&db, "SELECT table_name, note FROM tables ORDER BY table_name").await);
-    golden("0005_table_note", &text);
+    golden("0006_table_note", &text);
 }
 ```
 
-`EXTENDDB_BLESS_EXPECTED=1 cargo test -p extenddb-storage-sqlite --lib migration_5`
-writes `crates/storage-sqlite/tests/fixtures/golden/0005_table_note.txt`, one
+`EXTENDDB_BLESS_EXPECTED=1 cargo test -p extenddb-storage-sqlite --lib migration_6`
+writes `crates/storage-sqlite/tests/fixtures/golden/0006_table_note.txt`, one
 tab-separated line per row under each query:
 
 ```
@@ -457,15 +459,15 @@ The PostgreSQL body is a file, so it has no golden test.
 The example declares `Kind::Breaking`. A nullable column can be additive by
 the rule in step 4, but declaring it additive needs the old-binary job, which
 this example does not add. As a breaking migration it makes a 1.x binary
-without migration 5 refuse a catalog that has it, and a rollback restores the
+without migration 6 refuse a catalog that has it, and a rollback restores the
 backup.
 
 The same PR changes the code that reads and writes `tables.note` in each
 backend's catalog store, and commits the `docs/schema/sqlite.txt` and
 `docs/schema/postgres.txt` that the integration jobs regenerate. It changes
 no per-table DDL, because `tables` is a catalog table, and adds no `RELEASED`
-entry. `extenddb migrate --list` then shows migration 5, and on a catalog at
-migration 4 `extenddb migrate` lists migration 5 as pending and exits 3.
+entry. `extenddb migrate --list` then shows migration 6, and on a catalog at
+migration 5 `extenddb migrate` lists migration 6 as pending and exits 3.
 
 ### Running the migration tests
 

crates/app/src/init_helpers.rs

@@ -300,6 +300,20 @@ key_path = "{tls_key}"
 # max_import_item_count = 10000000
 # max_export_item_count = 10000000
 
+# [backup]
+# Backup and restore jobs. stale_job_after_secs must be at least three
+# heartbeat periods, the larger of heartbeat_interval_secs and the 15 s the
+# jobs of this release heartbeat at (45 at the least); pool_size is raised to
+# at least 2 * max_concurrent_jobs + 10 on PostgreSQL; max_concurrent_jobs
+# is at most 1024.
+# data_file_target_bytes = 67108864    # Uncompressed bytes per data file (minimum 32 MiB)
+# heartbeat_interval_secs = 15         # Job heartbeat timer
+# stale_job_after_secs = 300           # Heartbeat age after which a job's owner is dead
+# failed_row_retention_secs = 604800   # FAILED tombstones kept for the operator (7 days)
+# reconcile_interval_secs = 60         # Reconciler cycle
+# max_concurrent_jobs = 4              # Backup and restore jobs running at once on this instance
+# spool_path = "/var/lib/extenddb/spool" # Restore download area for an S3 store
+
 # [import]
 # paths = []                     # Allowed directories for import operations