robinnsc · Nov 2, 2026
@@ -50,9 +50,11 @@ jobs:
         run: cargo build --release --no-default-features --features mongodb-test-hooks
       - name: Run MongoDB pytest suite
         env:
-          # run-mongodb-tests initializes the server without [backup.store]:
-          # CreateBackup is the in-catalog create.
-          EXTENDDB_TEST_BACKUP_STORE: none
+          # run-mongodb-tests gives the server a filesystem [backup.store]
+          # under its output directory: CreateBackup is the store-backed job
+          # and the restore cells (tests/test_mongodb_backup_restore.py)
+          # rewrite prefixes under that root.
+          EXTENDDB_TEST_BACKUP_STORE: filesystem
         run: devtools/run-mongodb-tests -- --pytest --comprehensive --parallel
 
   # SnapshotEngine consistency check against a disposable replica set. The
@@ -122,9 +124,9 @@ jobs:
         run: cargo build --release --no-default-features --features mongodb-test-hooks
       - name: Run MongoDB rust integration suite
         env:
-          # run-mongodb-tests initializes the server without [backup.store]:
-          # the Rust backup tests restore the in-catalog create's backups.
-          EXTENDDB_TEST_BACKUP_STORE: none
+          # run-mongodb-tests gives the server a filesystem [backup.store]:
+          # the Rust backup tests restore the store-backed job's backups.
+          EXTENDDB_TEST_BACKUP_STORE: filesystem
         run: devtools/run-mongodb-tests -- --rust --rust-integration
 
   mongodb-production-build:

@@ -168,14 +168,34 @@ jobs:
           EXTENDDB_TEST_BACKUP_STORE: filesystem
         run: devtools/run-tests --extenddb --pytest --comprehensive --parallel --filter "not import_export"
 
+      # moto for the job suite's store-outage cell, the one cell that
+      # exercises `spool_path` on PostgreSQL (an S3 download into the spool,
+      # the retry under a cut, the spool entry removed); without the endpoint
+      # the cell skips. Same environment as the s3 job's.
+      - name: Start S3 mock
+        run: |
+          python3 -m venv "$RUNNER_TEMP/moto"
+          "$RUNNER_TEMP/moto/bin/pip" install -r requirements-moto.txt
+          nohup "$RUNNER_TEMP/moto/bin/moto_server" -p 9000 > moto.log 2>&1 &
+          for _ in $(seq 1 30); do
+            if curl -sf http://127.0.0.1:9000/ > /dev/null; then exit 0; fi
+            sleep 1
+          done
+          echo "moto_server did not start"; cat moto.log; exit 1
+
       # The backup store and job suites start servers of their own, so they
       # are excluded from run-tests' line and run by name.
       - name: Run the backup store and job suites
         env:
           EXTENDDB_TEST_PG_CONNECTION_STRING: postgresql://postgres:devpass@127.0.0.1:5432
           EXTENDDB_TEST_PG_ADMIN_CONNECTION_STRING: postgresql://postgres:devpass@127.0.0.1:5432
+          EXTENDDB_TEST_S3_ENDPOINT: http://127.0.0.1:9000
         run: python3 -m pytest tests/test_backup_store_config.py tests/test_backup_job.py -v -rP -p no:cacheprovider
 
+      - name: Print the S3 mock's log
+        if: failure()
+        run: cat moto.log || true
+
       # run-tests restarts the server daemonized to apply the import/export
       # config, and a daemonized server logs to syslog, not the job log. Without
       # this dump, the storage error behind a client-visible 500 leaves no
@@ -292,9 +312,11 @@ jobs:
   # with EXTENDDB_TEST_BACKUP_STORE=s3, and the fidelity suite's health check
   # confirms the server's store is the S3 one; the store configuration
   # suite's S3 cells need the same endpoint. The job suite
-  # (tests/test_backup_job.py) starts servers of its own with the SQLite
-  # build's defaulted filesystem root, as on the SQLite job: its deployment
-  # fixture has no S3 branch, so it covers the job and not this store.
+  # (tests/test_backup_job.py) starts servers of its own: most of its cells
+  # on the SQLite build's defaulted filesystem root, as on the SQLite job,
+  # and the store-outage cell on an S3 store at the moto endpoint, which it
+  # cuts mid-download and restores, so this job runs one restore from an S3
+  # store.
   run-integration-sqlite-s3:
     runs-on: extenddb_ubuntu-2404_4-core
     timeout-minutes: 40
@@ -385,7 +407,9 @@ jobs:
         run: devtools/run-tests --extenddb --pytest --parallel --filter "backup"
 
       # The store configuration suite's S3 cells against moto; the job suite
-      # against the build's defaulted filesystem root (the job comment above).
+      # against the build's defaulted filesystem root, except the store-outage
+      # cell, which runs against moto through EXTENDDB_TEST_S3_ENDPOINT (the
+      # job comment above).
       - name: Run the store configuration suite's S3 cells and the job suite
         env:
           EXTENDDB_TEST_S3_ENDPOINT: http://127.0.0.1:9000
@@ -717,7 +741,8 @@ jobs:
           # A file-backed SQLite server without [backup.store] has the defaulted
           # filesystem root beside its database file, so CreateBackup is the
           # asynchronous, store-backed one: the backup tests wait for AVAILABLE,
-          # delete what they create, and expect the restore's refusal.
+          # delete what they create, and restore the store-backed backups
+          # (the round trip).
           EXTENDDB_TEST_BACKUP_STORE: filesystem
         run: devtools/run-tests --extenddb --rust-integration --release
 

@@ -0,0 +1,181 @@
+# Copyright 2026 ExtendDB contributors
+# SPDX-License-Identifier: Apache-2.0
+name: Cross-backend restore matrix (nightly)
+
+# The cross-backend restore matrix (design PR 9b): a backup taken on one
+# backend restores, through the store, into another. Each cell starts a source
+# and a target server from the per-backend binaries, copies the source's backup
+# prefix under a root the target owns, seeds the target's `backup_index` row as
+# `backup sync --adopt-store` will (PR 10), restores and compares. The cells are
+# heavy (two servers each, four backends, PostgreSQL + a MongoDB replica set +
+# Cassandra) so they run nightly rather than on every pull request.
+
+on:
+  schedule:
+    - cron: "0 7 * * *"
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+# One run at a time: the cells drop and re-create fixed MongoDB database and
+# Cassandra keyspace names, so two runs on one runner would destroy each
+# other's catalogs. A second run queues behind the first.
+concurrency:
+  group: ${{ github.workflow }}
+  cancel-in-progress: false
+
+jobs:
+  matrix:
+    runs-on: extenddb_ubuntu-2404_4-core
+    timeout-minutes: 120
+    services:
+      postgres:
+        image: pgvector/pgvector:pg16
+        env:
+          POSTGRES_PASSWORD: devpass
+        ports:
+          - 127.0.0.1:55432:5432
+        options: >-
+          --health-cmd "pg_isready -U postgres"
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 10
+      cassandra:
+        image: cassandra:4.1
+        ports:
+          - 127.0.0.1:9042:9042
+        env:
+          CASSANDRA_CLUSTER_NAME: extenddb-ci
+          HEAP_NEWSIZE: 128M
+          MAX_HEAP_SIZE: 1024M
+        options: >-
+          --health-cmd "cqlsh -e 'SELECT release_version FROM system.local'"
+          --health-interval 15s
+          --health-timeout 10s
+          --health-retries 20
+          --health-start-period 60s
+    steps:
+      - uses: actions/checkout@v6
+      - uses: dtolnay/rust-toolchain@stable
+      - uses: Swatinem/rust-cache@v2
+        with:
+          cache-on-failure: true
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+      - name: Install Python dependencies
+        # requirements-matrix.txt adds the MongoDB and Cassandra drivers the
+        # script's seed and teardown use on those backends.
+        run: pip install -r requirements.txt -r requirements-matrix.txt
+
+      - name: Install the PostgreSQL client
+        # The matrix script seeds and tears down every PostgreSQL source and
+        # target through the host's psql; the runner image is not assumed to
+        # carry one.
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y --no-install-recommends postgresql-client
+
+      - name: Check the host tools the matrix script calls
+        run: |
+          psql --version
+          python3 -c "import pymongo, cassandra; print(pymongo.version, cassandra.__version__)"
+
+      - name: Configure Cassandra authentication
+        run: |
+          docker exec ${{ job.services.cassandra.id }} sed -i \
+            's/^authenticator: .*/authenticator: PasswordAuthenticator/' \
+            /etc/cassandra/cassandra.yaml
+          docker restart ${{ job.services.cassandra.id }}
+          for i in $(seq 1 90); do
+            if docker exec ${{ job.services.cassandra.id }} \
+                cqlsh -u cassandra -p cassandra \
+                -e 'SELECT release_version FROM system.local' >/dev/null 2>&1; then
+              echo "Cassandra ready with PasswordAuthenticator after ${i}s"
+              exit 0
+            fi
+            sleep 2
+          done
+          echo "Cassandra did not become ready" >&2
+          exit 1
+
+      - name: Start a MongoDB replica set
+        # GitHub `services:` cannot express a replica set (--replSet + initiate
+        # + wait-for-PRIMARY), so start it directly, as devtools/run-mongodb-tests
+        # does. Published on loopback only; no authentication.
+        run: |
+          docker run -d --name matrix-mongo -p 127.0.0.1:27117:27017 mongo:7 \
+            mongod --replSet rs0 --bind_ip_all
+          for i in $(seq 1 30); do
+            docker exec matrix-mongo mongosh --quiet --eval 'db.runCommand({ping:1}).ok' 2>/dev/null \
+              | grep -qx 1 && break
+            sleep 1
+          done
+          docker exec matrix-mongo mongosh --quiet --eval \
+            'rs.initiate({_id:"rs0",members:[{_id:0,host:"localhost:27017"}]})' >/dev/null 2>&1 || true
+          for i in $(seq 1 30); do
+            state=$(docker exec matrix-mongo mongosh --quiet --eval 'rs.status().members[0].stateStr' 2>/dev/null | tail -1)
+            [ "$state" = PRIMARY ] && { echo "replica set PRIMARY after ${i}s"; exit 0; }
+            sleep 1
+          done
+          echo "MongoDB replica set did not elect a PRIMARY" >&2
+          exit 1
+
+      - name: Build the four backend binaries
+        # One target directory, four sequential builds (cargo rebuilds only
+        # the feature-dependent crates between them), each binary copied to
+        # the path the script's --bin defaults use
+        # (target/rel-<backend>/release/extenddb). Every `init` the script
+        # runs writes the TLS material under ~/.extenddb/tls when absent, so
+        # nothing has to generate it here.
+        run: |
+          for b in postgres sqlite mongodb cassandra; do
+            cargo build --release -p extenddb --no-default-features --features "$b"
+            mkdir -p "target/rel-$b/release"
+            cp target/release/extenddb "target/rel-$b/release/extenddb"
+          done
+
+      - name: Run the matrix
+        run: |
+          set -o pipefail
+          mkdir -p matrix-logs
+          rc=0
+          plain="postgres sqlite mongodb"
+          targets="postgres sqlite mongodb cassandra"
+          for s in $plain; do
+            for t in $targets; do
+              echo "=== $s -> $t ==="
+              python3 devtools/backup-restore-matrix --source "$s" --target "$t" \
+                --work "matrix-logs/$s-$t" || rc=1
+            done
+          done
+          # The duplicate-key cell into every target, from a PostgreSQL source.
+          for t in $targets; do
+            echo "=== postgres -> $t --duplicate-key ==="
+            python3 devtools/backup-restore-matrix --source postgres --target "$t" --duplicate-key \
+              --work "matrix-logs/postgres-$t-dup" || rc=1
+          done
+          # The vector cells: into PostgreSQL and SQLite the vector index is
+          # restored and answers; into MongoDB and Cassandra the manifest is
+          # refused before any table.
+          for s in postgres sqlite; do
+            for t in postgres sqlite mongodb cassandra; do
+              echo "=== $s -> $t --vector ==="
+              python3 devtools/backup-restore-matrix --source "$s" --target "$t" --vector \
+                --work "matrix-logs/$s-$t-vector" || rc=1
+            done
+          done
+          exit $rc
+
+      - name: Upload the matrix logs on failure
+        if: failure()
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02  # v4.6.2
+        with:
+          name: matrix-logs
+          path: matrix-logs
+          if-no-files-found: ignore
+
+      - name: Stop the MongoDB replica set
+        if: always()
+        run: docker rm -f matrix-mongo >/dev/null 2>&1 || true
@@ -50,8 +50,25 @@ cargo test --workspace
 # Integration tests (requires a running extenddb server; the variable says
 # where that server writes its backups, see the developer test guide)
 EXTENDDB_TEST_BACKUP_STORE=filesystem devtools/run-tests --extenddb --pytest
+
+# The backup store and job suites start servers of their own from the
+# release build named by EXTENDDB_BINARY, so they run by name (the S3 cells
+# need a moto server at EXTENDDB_TEST_S3_ENDPOINT; see the developer test
+# guide for the PostgreSQL and moto variables)
+EXTENDDB_BINARY=target/release/extenddb python3 -m pytest tests/test_backup_store_config.py tests/test_backup_job.py -v
 ```
 
+The MongoDB backup cells that read the store's root and the server's log
+(`EXTENDDB_TEST_BACKUP_ROOT`, `EXTENDDB_TEST_SERVER_LOG`) run through
+`devtools/run-mongodb-tests`, which exports both; its legacy LSI fallback
+cell needs the runner's `--backup-store none`. The cross-backend restore
+matrix is `devtools/backup-restore-matrix` (one cell per invocation, from the
+four per-backend release binaries; `pip install -r requirements-matrix.txt`
+for its MongoDB and Cassandra cells) and runs nightly from
+`.github/workflows/matrix-nightly.yml`. The developer test guide
+(`docs/manuals/06-developer-test-guide.md`, "Test Runner Script") has a
+paragraph on each.
+
 ### Code Style
 
 All contributions must pass: