feat(backup): CreateBackup job, worker and reconciler duties [backup/restore 8]

.github/workflows/integration-cassandra.yml

@@ -148,6 +148,10 @@ jobs:
       - name: Build release (cassandra backend)
         run: cargo build --release --no-default-features --features cassandra
       - name: Run Cassandra pytest suite
+        env:
+          # run-cassandra-tests initializes the server without [backup.store],
+          # and Cassandra takes the in-catalog create with one.
+          EXTENDDB_TEST_BACKUP_STORE: none
         run: devtools/run-cassandra-tests -- --pytest --comprehensive --parallel
 
   cassandra-rust:
@@ -199,6 +203,10 @@ jobs:
       - name: Build release (cassandra backend)
         run: cargo build --release --no-default-features --features cassandra
       - name: Run Cassandra rust integration suite
+        env:
+          # run-cassandra-tests initializes the server without [backup.store]:
+          # the Rust backup tests restore the in-catalog create's backups.
+          EXTENDDB_TEST_BACKUP_STORE: none
         run: devtools/run-cassandra-tests -- --rust-integration
 
   cassandra-production-build:

.github/workflows/integration-mongodb.yml

@@ -49,6 +49,10 @@ jobs:
       - name: Build release (mongodb backend)
         run: cargo build --release --no-default-features --features mongodb-test-hooks
       - name: Run MongoDB pytest suite
+        env:
+          # run-mongodb-tests initializes the server without [backup.store]:
+          # CreateBackup is the in-catalog create.
+          EXTENDDB_TEST_BACKUP_STORE: none
         run: devtools/run-mongodb-tests -- --pytest --comprehensive --parallel
 
   # SnapshotEngine consistency check against a disposable replica set. The
@@ -117,6 +121,10 @@ jobs:
       - name: Build release (mongodb backend)
         run: cargo build --release --no-default-features --features mongodb-test-hooks
       - name: Run MongoDB rust integration suite
+        env:
+          # run-mongodb-tests initializes the server without [backup.store]:
+          # the Rust backup tests restore the in-catalog create's backups.
+          EXTENDDB_TEST_BACKUP_STORE: none
         run: devtools/run-mongodb-tests -- --rust --rust-integration
 
   mongodb-production-build:

.github/workflows/integration.yml

@@ -67,19 +67,34 @@ jobs:
           password=$(echo "$output" | grep -oP 'Password: \K\S+')
           echo "admin_password=$password" >> "$GITHUB_OUTPUT"
 
+      # A filesystem backup store, so this job's CreateBackup is the
+      # store-backed job (PostgreSQL has no default root). The suite reads
+      # EXTENDDB_TEST_BACKUP_STORE=filesystem below to expect that path.
+      - name: Configure a filesystem backup store
+        run: |
+          mkdir -m 0700 "$RUNNER_TEMP/backups"
+          printf '\n[backup.store]\nkind = "filesystem"\n[backup.store.filesystem]\npath = "%s/backups"\n' \
+            "$RUNNER_TEMP" >> extenddb.toml
+
       - name: Start ExtendDB
         run: |
           # --write-pid-file so devtools/run-tests can restart the server with
           # 'extenddb stop' when it needs to apply a config change.
+          # The wait is for the backup store component to be healthy, not for
+          # the word "healthy" anywhere in the reply: a server whose store
+          # failed its check reports "unhealthy" (which that word matches)
+          # and the suite would then fail on a backup cell with no pointer to
+          # the cause.
           ./target/release/extenddb serve --config extenddb.toml --foreground --write-pid-file &
           for i in $(seq 1 30); do
-            if curl -sk https://127.0.0.1:18443/health | grep -q healthy; then
-              echo "Server ready"
+            if curl -sk https://127.0.0.1:18443/health | python3 -c 'import json, sys; sys.exit(0 if json.load(sys.stdin).get("components", {}).get("backup_store", {}).get("status") == "healthy" else 1)' 2>/dev/null; then
+              echo "Server ready with a healthy backup store"
               exit 0
             fi
             sleep 1
           done
-          echo "Server failed to start"
+          echo "Server failed to start, or its backup store did not become healthy"
+          curl -sk https://127.0.0.1:18443/health || true
           exit 1
 
       - uses: actions/setup-python@v5
@@ -150,8 +165,17 @@ jobs:
           EXTENDDB_ADMIN_USER: admin
           EXTENDDB_ADMIN_PASSWORD: ${{ steps.init.outputs.admin_password }}
           EXTENDDB_TEST_PG_ADMIN_CONNECTION_STRING: postgresql://postgres:devpass@127.0.0.1:5432
+          EXTENDDB_TEST_BACKUP_STORE: filesystem
         run: devtools/run-tests --extenddb --pytest --comprehensive --parallel --filter "not import_export"
 
+      # The backup store and job suites start servers of their own, so they
+      # are excluded from run-tests' line and run by name.
+      - name: Run the backup store and job suites
+        env:
+          EXTENDDB_TEST_PG_CONNECTION_STRING: postgresql://postgres:devpass@127.0.0.1:5432
+          EXTENDDB_TEST_PG_ADMIN_CONNECTION_STRING: postgresql://postgres:devpass@127.0.0.1:5432
+        run: python3 -m pytest tests/test_backup_store_config.py tests/test_backup_job.py -v -rP -p no:cacheprovider
+
       # run-tests restarts the server daemonized to apply the import/export
       # config, and a daemonized server logs to syslog, not the job log. Without
       # this dump, the storage error behind a client-visible 500 leaves no
@@ -220,15 +244,21 @@ jobs:
         run: |
           # --write-pid-file so devtools/run-tests can restart the server with
           # 'extenddb stop' when it needs to apply a config change.
+          # The wait is for the backup store component to be healthy, not for
+          # the word "healthy" anywhere in the reply: a server whose store
+          # failed its check reports "unhealthy" (which that word matches)
+          # and the suite would then fail on a backup cell with no pointer to
+          # the cause.
           ./target/release/extenddb serve --config extenddb.toml --foreground --write-pid-file &
           for i in $(seq 1 30); do
-            if curl -sk https://127.0.0.1:18443/health | grep -q healthy; then
-              echo "Server ready"
+            if curl -sk https://127.0.0.1:18443/health | python3 -c 'import json, sys; sys.exit(0 if json.load(sys.stdin).get("components", {}).get("backup_store", {}).get("status") == "healthy" else 1)' 2>/dev/null; then
+              echo "Server ready with a healthy backup store"
               exit 0
             fi
             sleep 1
           done
-          echo "Server failed to start"
+          echo "Server failed to start, or its backup store did not become healthy"
+          curl -sk https://127.0.0.1:18443/health || true
           exit 1
 
       - uses: actions/setup-python@v5
@@ -243,14 +273,132 @@ jobs:
           EXTENDDB_TEST_ENDPOINT: https://127.0.0.1:18443
           EXTENDDB_ADMIN_USER: admin
           EXTENDDB_ADMIN_PASSWORD: ${{ steps.init.outputs.admin_password }}
+          # A file-backed SQLite server without [backup.store] writes to the
+          # defaulted root beside the database file.
+          EXTENDDB_TEST_BACKUP_STORE: filesystem
         run: devtools/run-tests --extenddb --pytest --comprehensive --parallel --filter "not import_export"
 
+      - name: Run the backup store and job suites
+        run: python3 -m pytest tests/test_backup_store_config.py tests/test_backup_job.py -v -rP -p no:cacheprovider
+
       # Same rationale as the postgres job: the daemonized server logs to
       # syslog, so this dump is the only server-side evidence on failure.
       - name: Dump server syslog on failure
         if: failure()
         run: sudo journalctl -t extenddb --no-pager -n 500 || true
 
+  # The S3 leg: the SQLite build against an S3 backup store on moto. The
+  # backup tests of the main suite (the tests named "backup") run against it
+  # with EXTENDDB_TEST_BACKUP_STORE=s3, and the fidelity suite's health check
+  # confirms the server's store is the S3 one; the store configuration
+  # suite's S3 cells need the same endpoint. The job suite
+  # (tests/test_backup_job.py) starts servers of its own with the SQLite
+  # build's defaulted filesystem root, as on the SQLite job: its deployment
+  # fixture has no S3 branch, so it covers the job and not this store.
+  run-integration-sqlite-s3:
+    runs-on: extenddb_ubuntu-2404_4-core
+    timeout-minutes: 40
+    steps:
+      - uses: actions/checkout@v6
+
+      - uses: dtolnay/rust-toolchain@stable
+
+      - uses: Swatinem/rust-cache@v2
+        with:
+          cache-on-failure: true
+
+      - name: Build release (SQLite backend)
+        run: cargo build --release -p extenddb --no-default-features --features sqlite
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - name: Install Python dependencies
+        run: pip install -r requirements.txt
+
+      # moto is a pip package pinned in requirements-moto.txt (test.yml runs
+      # the same server for the store crate's conformance suite); its boto3
+      # floor differs from requirements.txt's pin, so it gets an environment
+      # of its own.
+      - name: Start S3 mock
+        run: |
+          python3 -m venv "$RUNNER_TEMP/moto"
+          "$RUNNER_TEMP/moto/bin/pip" install -r requirements-moto.txt
+          nohup "$RUNNER_TEMP/moto/bin/moto_server" -p 9000 > moto.log 2>&1 &
+          for _ in $(seq 1 30); do
+            if curl -sf http://127.0.0.1:9000/ > /dev/null; then exit 0; fi
+            sleep 1
+          done
+          echo "moto_server did not start"; cat moto.log; exit 1
+
+      - name: Initialize ExtendDB
+        id: init
+        env:
+          AWS_ACCESS_KEY_ID: testing
+          AWS_SECRET_ACCESS_KEY: testing
+          AWS_REGION: us-east-1
+        run: |
+          output=$(./target/release/extenddb init --backend sqlite --config extenddb.toml 2>&1)
+          echo "$output"
+          password=$(echo "$output" | grep -oP 'Password: \K\S+')
+          echo "admin_password=$password" >> "$GITHUB_OUTPUT"
+          python3 -c 'import boto3; boto3.client("s3", endpoint_url="http://127.0.0.1:9000", region_name="us-east-1").create_bucket(Bucket="extenddb-ci-backups")'
+          printf '\n[backup.store]\nkind = "s3"\n[backup.store.s3]\nbucket = "extenddb-ci-backups"\nendpoint = "http://127.0.0.1:9000"\nforce_path_style = true\nregion = "us-east-1"\n' >> extenddb.toml
+
+      - name: Start ExtendDB
+        env:
+          AWS_ACCESS_KEY_ID: testing
+          AWS_SECRET_ACCESS_KEY: testing
+          AWS_REGION: us-east-1
+          AWS_EC2_METADATA_DISABLED: "true"
+        run: |
+          # An S3 store is checked after the listener is up and reports
+          # "checking" until its first check returns; a CreateBackup inside
+          # that window is refused, so the wait is for the component to be
+          # healthy, which also names a store that failed its check here
+          # rather than in a backup cell.
+          ./target/release/extenddb serve --config extenddb.toml --foreground --write-pid-file &
+          for i in $(seq 1 30); do
+            if curl -sk https://127.0.0.1:18443/health | python3 -c 'import json, sys; sys.exit(0 if json.load(sys.stdin).get("components", {}).get("backup_store", {}).get("status") == "healthy" else 1)' 2>/dev/null; then
+              echo "Server ready with a healthy backup store"
+              exit 0
+            fi
+            sleep 1
+          done
+          echo "Server failed to start, or its backup store did not become healthy"
+          curl -sk https://127.0.0.1:18443/health || true
+          exit 1
+
+      - name: Run the backup tests against the S3 store
+        env:
+          EXTENDDB_TEST_ENDPOINT: https://127.0.0.1:18443
+          EXTENDDB_ADMIN_USER: admin
+          EXTENDDB_ADMIN_PASSWORD: ${{ steps.init.outputs.admin_password }}
+          EXTENDDB_TEST_BACKUP_STORE: s3
+          # run-tests restarts the server daemonized, which needs the store's
+          # credentials in its environment too.
+          AWS_ACCESS_KEY_ID: testing
+          AWS_SECRET_ACCESS_KEY: testing
+          AWS_REGION: us-east-1
+          AWS_EC2_METADATA_DISABLED: "true"
+        run: devtools/run-tests --extenddb --pytest --parallel --filter "backup"
+
+      # The store configuration suite's S3 cells against moto; the job suite
+      # against the build's defaulted filesystem root (the job comment above).
+      - name: Run the store configuration suite's S3 cells and the job suite
+        env:
+          EXTENDDB_TEST_S3_ENDPOINT: http://127.0.0.1:9000
+        run: python3 -m pytest tests/test_backup_store_config.py tests/test_backup_job.py -v -rP -p no:cacheprovider
+
+      - name: Dump server syslog on failure
+        if: failure()
+        run: sudo journalctl -t extenddb --no-pager -n 500 || true
+
+      - name: Print the S3 mock's log
+        if: failure()
+        run: cat moto.log || true
+
   run-integration-dev-mode:
     # dev-mode was shipped with no CI coverage at all, which is how the batch and
     # transaction authorization regression reached main: the build compiled, so
@@ -384,6 +532,9 @@ jobs:
           # tests mandatory. The refusal suite moves to the novector job, which is
           # the only place those assertions still mean anything.
           EXTENDDB_EXPECT_VECTORS: "1"
+          # This server has no [backup.store]: CreateBackup is the in-catalog
+          # create and the backup tests restore its backups.
+          EXTENDDB_TEST_BACKUP_STORE: none
         run: devtools/run-tests --extenddb --rust-integration --release
 
       # The control plane for vector indexes is not reachable over the wire while
@@ -479,6 +630,9 @@ jobs:
           # the backend reports, so without this it could skip every assertion
           # and still report green.
           EXTENDDB_EXPECT_VECTORS: "0"
+          # No [backup.store] on this server either; the filter runs no backup
+          # test, and the variable says so for whoever widens it.
+          EXTENDDB_TEST_BACKUP_STORE: none
         run: >-
           devtools/run-tests --extenddb --rust-integration --release
           --filter vector_index_unsupported
@@ -560,6 +714,11 @@ jobs:
           # which is what would happen if the backend lost the capability. Pinning the
           # expectation turns that skip into a failure.
           EXTENDDB_EXPECT_VECTORS: "1"
+          # A file-backed SQLite server without [backup.store] has the defaulted
+          # filesystem root beside its database file, so CreateBackup is the
+          # asynchronous, store-backed one: the backup tests wait for AVAILABLE,
+          # delete what they create, and expect the restore's refusal.
+          EXTENDDB_TEST_BACKUP_STORE: filesystem
         run: devtools/run-tests --extenddb --rust-integration --release
 
       # The daemonized server logs to syslog; dump it so server-side failures
@@ -575,6 +734,7 @@ jobs:
       [
         run-integration,
         run-integration-sqlite,
+        run-integration-sqlite-s3,
         run-integration-dev-mode,
         run-rust-integration,
         run-rust-integration-postgres-novector,
@@ -585,6 +745,7 @@ jobs:
       - run: |
           if [ "${{ needs.run-integration.result }}" != "success" ] || \
              [ "${{ needs.run-integration-sqlite.result }}" != "success" ] || \
+             [ "${{ needs.run-integration-sqlite-s3.result }}" != "success" ] || \
              [ "${{ needs.run-integration-dev-mode.result }}" != "success" ] || \
              [ "${{ needs.run-rust-integration.result }}" != "success" ] || \
              [ "${{ needs.run-rust-integration-postgres-novector.result }}" != "success" ] || \

CONTRIBUTING.md

@@ -47,8 +47,9 @@ cargo build --workspace
 # Rust unit tests
 cargo test --workspace
 
-# Integration tests (requires a running extenddb server)
-devtools/run-tests --extenddb --pytest
+# Integration tests (requires a running extenddb server; the variable says
+# where that server writes its backups, see the developer test guide)
+EXTENDDB_TEST_BACKUP_STORE=filesystem devtools/run-tests --extenddb --pytest
 ```
 
 ### Code Style

crates/app/Cargo.toml

@@ -13,6 +13,7 @@ license.workspace = true
 # the thin bin enforces that it is paired with a dev/CI backend.
 dev-mode = ["extenddb-config/dev-mode"]
 mongodb-test-hooks = ["extenddb-server/mongodb-test-hooks"]
+backup-test-hooks = ["extenddb-server/backup-test-hooks"]
 
 [dependencies]
 extenddb-auth = { workspace = true }

crates/app/src/init_helpers.rs

@@ -303,7 +303,7 @@ key_path = "{tls_key}"
 # [backup]
 # Backup and restore jobs. stale_job_after_secs must be at least three
 # heartbeat periods, the larger of heartbeat_interval_secs and the 15 s the
-# jobs of this release heartbeat at (45 at the least); pool_size is raised to
+# in-catalog jobs heartbeat at (45 at the least); pool_size is raised to
 # at least 2 * max_concurrent_jobs + 10 on PostgreSQL; max_concurrent_jobs
 # is at most 1024.
 # data_file_target_bytes = 67108864    # Uncompressed bytes per data file (minimum 32 MiB)
@@ -317,9 +317,10 @@ key_path = "{tls_key}"
 #                                        # it defaults to backup-spool/ beside the database file
 
 # [backup.store]
-# Where CreateBackup's backups will live, from a later release (this release's
-# CreateBackup writes in-catalog backups with any store, and the store serves
-# restores and deletes of the backups it holds). Absent on file-backed SQLite: a filesystem
+# Where CreateBackup writes its backups on PostgreSQL, SQLite and MongoDB
+# (CreateBackup answers CREATING and a job of the server uploads; Cassandra
+# writes in-catalog backups with or without a store), and where restores and
+# deletes of store-backed backups read. Absent on file-backed SQLite: a filesystem
 # store at backups/ beside the database file. Absent on PostgreSQL, MongoDB,
 # Cassandra and an in-memory SQLite database: no store, and CreateBackup
 # writes in-catalog backups with a deprecation warning. kind = "disabled"