[#154] Added a nightly push of the sanitized database to FTP or S3 with an opt-in timestamped copy.

.circleci/config.yml

@@ -363,6 +363,32 @@ jobs:
           no_output_timeout: 30m
       #;> DB_FETCH_SOURCE_CONTAINER_REGISTRY
 
+      #;< DB_PUSH
+      # Push the freshly-exported sanitized database dump to the database fetch
+      # source. No-op unless a fresh DB was fetched (the fetch semaphore) and the
+      # push was requested (VORTEX_PUSH_DB_PROCEED, in the script).
+      - run:
+          name: Push DB
+          command: |
+            [ ! -f /tmp/fetch-db-fresh ] && echo "==> No freshly fetched database. DB push will not proceed." && exit 0
+            ./vendor/bin/vortex-push-db
+            #;< VORTEX_DEV
+            if [ -n "${VORTEX_PUSH_DB_FTP_HOST:-}" ]; then VORTEX_FETCH_DB_SOURCE=ftp VORTEX_REFRESH_DB_SOURCE=url ./vendor/bin/vortex-push-db; fi
+            #;> VORTEX_DEV
+          no_output_timeout: 30m
+      #;> DB_PUSH
+
+      #;< MIGRATION_DB_PUSH
+      # Push the freshly-fetched migration database dump to its fetch source,
+      # next to the primary dump. No-op unless the push was requested.
+      - run:
+          name: Push migration DB
+          command: |
+            [ ! -f /tmp/fetch-db-fresh ] && echo "==> No freshly fetched database. DB push will not proceed." && exit 0
+            VORTEX_DB_INDEX=2 ./vendor/bin/vortex-push-db
+          no_output_timeout: 30m
+      #;> MIGRATION_DB_PUSH
+
       - save_cache:
           # Save cache per default branch and the timestamp.
           # The cache will not be saved if it already exists.
@@ -387,6 +413,14 @@ jobs:
       # Deploy container image.
       VORTEX_EXPORT_DB_CONTAINER_REGISTRY_PUSH_PROCEED: 1
       #;> DB_IN_IMAGE
+      # Refresh the database dumps in the fetch sources that are dump stores.
+      VORTEX_PUSH_DB_PROCEED: 1
+      #;< VORTEX_DEV
+      # Tell this provider's demo database dumps apart on the test FTP server,
+      # with a timestamped copy of each push.
+      VORTEX_PUSH_DB_FILE_SUFFIX: _circleci
+      VORTEX_PUSH_DB_TIMESTAMPED_COPY: 1
+      #;> VORTEX_DEV
       # Do not build the Drupal front-end.
       VORTEX_FRONTEND_BUILD_SKIP: 1
   #;> !PROVISION_TYPE_PROFILE

.circleci/vortex-test-common.yml

@@ -183,6 +183,32 @@ jobs:
           no_output_timeout: 30m
       #;> DB_FETCH_SOURCE_CONTAINER_REGISTRY
 
+      #;< DB_PUSH
+      # Push the freshly-exported sanitized database dump to the database fetch
+      # source. No-op unless a fresh DB was fetched (the fetch semaphore) and the
+      # push was requested (VORTEX_PUSH_DB_PROCEED, in the script).
+      - run:
+          name: Push DB
+          command: |
+            [ ! -f /tmp/fetch-db-fresh ] && echo "==> No freshly fetched database. DB push will not proceed." && exit 0
+            ./vendor/bin/vortex-push-db
+            #;< VORTEX_DEV
+            if [ -n "${VORTEX_PUSH_DB_FTP_HOST:-}" ]; then VORTEX_FETCH_DB_SOURCE=ftp VORTEX_REFRESH_DB_SOURCE=url ./vendor/bin/vortex-push-db; fi
+            #;> VORTEX_DEV
+          no_output_timeout: 30m
+      #;> DB_PUSH
+
+      #;< MIGRATION_DB_PUSH
+      # Push the freshly-fetched migration database dump to its fetch source,
+      # next to the primary dump. No-op unless the push was requested.
+      - run:
+          name: Push migration DB
+          command: |
+            [ ! -f /tmp/fetch-db-fresh ] && echo "==> No freshly fetched database. DB push will not proceed." && exit 0
+            VORTEX_DB_INDEX=2 ./vendor/bin/vortex-push-db
+          no_output_timeout: 30m
+      #;> MIGRATION_DB_PUSH
+
       - save_cache:
           # Save cache per default branch and the timestamp.
           # The cache will not be saved if it already exists.

.env

@@ -258,6 +258,21 @@ VORTEX_FETCH_DB_S3_BUCKET=
 VORTEX_FETCH_DB_S3_REGION=ap-southeast-2
 #;> DB_FETCH_SOURCE_S3
 
+#;< DB_PUSH
+# Upstream source of the database dump file, such as url, acquia or lagoon.
+#
+# Uncomment to make the database fetch source a dump store: the nightly CI
+# job fetches the dump from this source, sanitizes it and pushes it to the
+# fetch source as the latest dump. The fetch script of this source reads its
+# own variables, such as VORTEX_FETCH_DB_URL for url. Add them, and keep
+# secrets in CI.
+# VORTEX_REFRESH_DB_SOURCE=url
+
+# Also push a copy of each dump named with a UTC timestamp, such as
+# db_20261111-180000.sql, to keep a history of the pushed dumps.
+# VORTEX_PUSH_DB_TIMESTAMPED_COPY=1
+#;> DB_PUSH
+
 # Environment to fetch the database from.
 #
 # Applies to hosting environments.
@@ -333,6 +348,15 @@ VORTEX_FETCH_DB2_S3_BUCKET=
 VORTEX_FETCH_DB2_S3_REGION=ap-southeast-2
 #;> MIGRATION_DB_FETCH_SOURCE_S3
 
+#;< MIGRATION_DB_PUSH
+# Upstream source of the second database dump file, such as url or lagoon.
+#
+# Uncomment to make the second database fetch source a dump store: the
+# nightly CI job fetches the dump from this source and pushes it, as fetched,
+# to the fetch source, next to the primary dump.
+# VORTEX_REFRESH_DB2_SOURCE=url
+#;> MIGRATION_DB_PUSH
+
 # Environment to fetch the second database from.
 #
 # Applies to hosting environments.

.github/workflows/build-test-deploy.yml

@@ -225,6 +225,34 @@ jobs:
         VORTEX_CI_DB_CACHE_FALLBACK: "yes"
         # Which branch to use as a source of DB caches.
         VORTEX_CI_DB_CACHE_BRANCH: "develop"
+        #;< DB_FETCH_SOURCE_FTP
+        VORTEX_FETCH_DB_FTP_USER: ${{ secrets.VORTEX_FETCH_DB_FTP_USER }}
+        VORTEX_FETCH_DB_FTP_PASS: ${{ secrets.VORTEX_FETCH_DB_FTP_PASS }}
+        #;> DB_FETCH_SOURCE_FTP
+        #;< DB_FETCH_SOURCE_S3
+        VORTEX_FETCH_DB_S3_ACCESS_KEY: ${{ secrets.VORTEX_FETCH_DB_S3_ACCESS_KEY }}
+        VORTEX_FETCH_DB_S3_SECRET_KEY: ${{ secrets.VORTEX_FETCH_DB_S3_SECRET_KEY }}
+        #;> DB_FETCH_SOURCE_S3
+        #;< MIGRATION_DB_FETCH_SOURCE_FTP
+        VORTEX_FETCH_DB2_FTP_USER: ${{ secrets.VORTEX_FETCH_DB2_FTP_USER }}
+        VORTEX_FETCH_DB2_FTP_PASS: ${{ secrets.VORTEX_FETCH_DB2_FTP_PASS }}
+        #;> MIGRATION_DB_FETCH_SOURCE_FTP
+        #;< MIGRATION_DB_FETCH_SOURCE_S3
+        VORTEX_FETCH_DB2_S3_ACCESS_KEY: ${{ secrets.VORTEX_FETCH_DB2_S3_ACCESS_KEY }}
+        VORTEX_FETCH_DB2_S3_SECRET_KEY: ${{ secrets.VORTEX_FETCH_DB2_S3_SECRET_KEY }}
+        #;> MIGRATION_DB_FETCH_SOURCE_S3
+        #;< VORTEX_DEV
+        # FTP server that receives the demo database on the nightly run, under
+        # a suffix that tells this provider's dumps apart, with a timestamped
+        # copy of each push.
+        VORTEX_PUSH_DB_FTP_HOST: ${{ secrets.VORTEX_PUSH_DB_FTP_HOST }}
+        VORTEX_PUSH_DB_FTP_PORT: ${{ secrets.VORTEX_PUSH_DB_FTP_PORT }}
+        VORTEX_PUSH_DB_FTP_USER: ${{ secrets.VORTEX_PUSH_DB_FTP_USER }}
+        VORTEX_PUSH_DB_FTP_PASS: ${{ secrets.VORTEX_PUSH_DB_FTP_PASS }}
+        VORTEX_PUSH_DB_FTP_FILE: ${{ vars.VORTEX_PUSH_DB_FTP_FILE || 'db.sql' }}
+        VORTEX_PUSH_DB_FILE_SUFFIX: _gha
+        VORTEX_PUSH_DB_TIMESTAMPED_COPY: 1
+        #;> VORTEX_DEV
 
     steps:
       # Frees disk space by removing preinstalled toolchains unused by this project. Steps run in a container, so the host is reached via the Docker socket.
@@ -281,6 +309,8 @@ jobs:
             # Deploy container image.
             echo "VORTEX_EXPORT_DB_CONTAINER_REGISTRY_PUSH_PROCEED=1"
             #;> DB_IN_IMAGE
+            # Refresh the database dumps in the fetch sources that are dump stores.
+            echo "VORTEX_PUSH_DB_PROCEED=1"
           } >> "$GITHUB_ENV"
 
       - name: Create cache keys files for database caching
@@ -343,6 +373,30 @@ jobs:
         timeout-minutes: 30
       #;> DB_FETCH_SOURCE_CONTAINER_REGISTRY
 
+      #;< DB_PUSH
+      # Push the freshly-exported sanitized database dump to the database fetch
+      # source. No-op unless a fresh DB was fetched (the fetch semaphore) and the
+      # push was requested (VORTEX_PUSH_DB_PROCEED, in the script).
+      - name: Push DB
+        run: |
+          if [ ! -f /tmp/fetch-db-fresh ]; then echo "==> No freshly fetched database. DB push will not proceed."; exit 0; fi
+          ./vendor/bin/vortex-push-db
+          #;< VORTEX_DEV
+          if [ -n "${VORTEX_PUSH_DB_FTP_HOST:-}" ]; then VORTEX_FETCH_DB_SOURCE=ftp VORTEX_REFRESH_DB_SOURCE=url ./vendor/bin/vortex-push-db; fi
+          #;> VORTEX_DEV
+        timeout-minutes: 30
+      #;> DB_PUSH
+
+      #;< MIGRATION_DB_PUSH
+      # Push the freshly-fetched migration database dump to its fetch source,
+      # next to the primary dump. No-op unless the push was requested.
+      - name: Push migration DB
+        run: |
+          if [ ! -f /tmp/fetch-db-fresh ]; then echo "==> No freshly fetched database. DB push will not proceed."; exit 0; fi
+          VORTEX_DB_INDEX=2 ./vendor/bin/vortex-push-db
+        timeout-minutes: 30
+      #;> MIGRATION_DB_PUSH
+
       # Save cache per default branch and the timestamp.
       # The cache will not be saved if it already exists.
       # Note that the cache fallback flag is enabled for this case in order

.github/workflows/vortex-test-didi.yml

@@ -64,6 +64,34 @@ jobs:
         VORTEX_CI_DB_CACHE_FALLBACK: "yes"
         # Which branch to use as a source of DB caches.
         VORTEX_CI_DB_CACHE_BRANCH: vortex-dev-didi-fi
+        #;< DB_FETCH_SOURCE_FTP
+        VORTEX_FETCH_DB_FTP_USER: ${{ secrets.VORTEX_FETCH_DB_FTP_USER }}
+        VORTEX_FETCH_DB_FTP_PASS: ${{ secrets.VORTEX_FETCH_DB_FTP_PASS }}
+        #;> DB_FETCH_SOURCE_FTP
+        #;< DB_FETCH_SOURCE_S3
+        VORTEX_FETCH_DB_S3_ACCESS_KEY: ${{ secrets.VORTEX_FETCH_DB_S3_ACCESS_KEY }}
+        VORTEX_FETCH_DB_S3_SECRET_KEY: ${{ secrets.VORTEX_FETCH_DB_S3_SECRET_KEY }}
+        #;> DB_FETCH_SOURCE_S3
+        #;< MIGRATION_DB_FETCH_SOURCE_FTP
+        VORTEX_FETCH_DB2_FTP_USER: ${{ secrets.VORTEX_FETCH_DB2_FTP_USER }}
+        VORTEX_FETCH_DB2_FTP_PASS: ${{ secrets.VORTEX_FETCH_DB2_FTP_PASS }}
+        #;> MIGRATION_DB_FETCH_SOURCE_FTP
+        #;< MIGRATION_DB_FETCH_SOURCE_S3
+        VORTEX_FETCH_DB2_S3_ACCESS_KEY: ${{ secrets.VORTEX_FETCH_DB2_S3_ACCESS_KEY }}
+        VORTEX_FETCH_DB2_S3_SECRET_KEY: ${{ secrets.VORTEX_FETCH_DB2_S3_SECRET_KEY }}
+        #;> MIGRATION_DB_FETCH_SOURCE_S3
+        #;< VORTEX_DEV
+        # FTP server that receives the demo database on the nightly run, under
+        # a suffix that tells this provider's dumps apart, with a timestamped
+        # copy of each push.
+        VORTEX_PUSH_DB_FTP_HOST: ${{ secrets.VORTEX_PUSH_DB_FTP_HOST }}
+        VORTEX_PUSH_DB_FTP_PORT: ${{ secrets.VORTEX_PUSH_DB_FTP_PORT }}
+        VORTEX_PUSH_DB_FTP_USER: ${{ secrets.VORTEX_PUSH_DB_FTP_USER }}
+        VORTEX_PUSH_DB_FTP_PASS: ${{ secrets.VORTEX_PUSH_DB_FTP_PASS }}
+        VORTEX_PUSH_DB_FTP_FILE: ${{ vars.VORTEX_PUSH_DB_FTP_FILE || 'db.sql' }}
+        VORTEX_PUSH_DB_FILE_SUFFIX: _gha
+        VORTEX_PUSH_DB_TIMESTAMPED_COPY: 1
+        #;> VORTEX_DEV
         VORTEX_FETCH_DB_SOURCE: url
         VORTEX_FETCH_DB_FORCE: 1
         VORTEX_DB_IMAGE: drevops/vortex-dev-mariadb-drupal-data-demo-destination-11.x
@@ -129,6 +157,8 @@ jobs:
             # Deploy container image.
             echo "VORTEX_EXPORT_DB_CONTAINER_REGISTRY_PUSH_PROCEED=1"
             #;> DB_IN_IMAGE
+            # Refresh the database dumps in the fetch sources that are dump stores.
+            echo "VORTEX_PUSH_DB_PROCEED=1"
           } >> "$GITHUB_ENV"
 
       - name: Create cache keys files for database caching
@@ -191,6 +221,30 @@ jobs:
         timeout-minutes: 30
       #;> DB_FETCH_SOURCE_CONTAINER_REGISTRY
 
+      #;< DB_PUSH
+      # Push the freshly-exported sanitized database dump to the database fetch
+      # source. No-op unless a fresh DB was fetched (the fetch semaphore) and the
+      # push was requested (VORTEX_PUSH_DB_PROCEED, in the script).
+      - name: Push DB
+        run: |
+          if [ ! -f /tmp/fetch-db-fresh ]; then echo "==> No freshly fetched database. DB push will not proceed."; exit 0; fi
+          ./vendor/bin/vortex-push-db
+          #;< VORTEX_DEV
+          if [ -n "${VORTEX_PUSH_DB_FTP_HOST:-}" ]; then VORTEX_FETCH_DB_SOURCE=ftp VORTEX_REFRESH_DB_SOURCE=url ./vendor/bin/vortex-push-db; fi
+          #;> VORTEX_DEV
+        timeout-minutes: 30
+      #;> DB_PUSH
+
+      #;< MIGRATION_DB_PUSH
+      # Push the freshly-fetched migration database dump to its fetch source,
+      # next to the primary dump. No-op unless the push was requested.
+      - name: Push migration DB
+        run: |
+          if [ ! -f /tmp/fetch-db-fresh ]; then echo "==> No freshly fetched database. DB push will not proceed."; exit 0; fi
+          VORTEX_DB_INDEX=2 ./vendor/bin/vortex-push-db
+        timeout-minutes: 30
+      #;> MIGRATION_DB_PUSH
+
       # Save cache per default branch and the timestamp.
       # The cache will not be saved if it already exists.
       # Note that the cache fallback flag is enabled for this case in order
@@ -570,6 +624,34 @@ jobs:
         VORTEX_CI_DB_CACHE_FALLBACK: "yes"
         # Which branch to use as a source of DB caches.
         VORTEX_CI_DB_CACHE_BRANCH: vortex-dev-didi-ii
+        #;< DB_FETCH_SOURCE_FTP
+        VORTEX_FETCH_DB_FTP_USER: ${{ secrets.VORTEX_FETCH_DB_FTP_USER }}
+        VORTEX_FETCH_DB_FTP_PASS: ${{ secrets.VORTEX_FETCH_DB_FTP_PASS }}
+        #;> DB_FETCH_SOURCE_FTP
+        #;< DB_FETCH_SOURCE_S3
+        VORTEX_FETCH_DB_S3_ACCESS_KEY: ${{ secrets.VORTEX_FETCH_DB_S3_ACCESS_KEY }}
+        VORTEX_FETCH_DB_S3_SECRET_KEY: ${{ secrets.VORTEX_FETCH_DB_S3_SECRET_KEY }}
+        #;> DB_FETCH_SOURCE_S3
+        #;< MIGRATION_DB_FETCH_SOURCE_FTP
+        VORTEX_FETCH_DB2_FTP_USER: ${{ secrets.VORTEX_FETCH_DB2_FTP_USER }}
+        VORTEX_FETCH_DB2_FTP_PASS: ${{ secrets.VORTEX_FETCH_DB2_FTP_PASS }}
+        #;> MIGRATION_DB_FETCH_SOURCE_FTP
+        #;< MIGRATION_DB_FETCH_SOURCE_S3
+        VORTEX_FETCH_DB2_S3_ACCESS_KEY: ${{ secrets.VORTEX_FETCH_DB2_S3_ACCESS_KEY }}
+        VORTEX_FETCH_DB2_S3_SECRET_KEY: ${{ secrets.VORTEX_FETCH_DB2_S3_SECRET_KEY }}
+        #;> MIGRATION_DB_FETCH_SOURCE_S3
+        #;< VORTEX_DEV
+        # FTP server that receives the demo database on the nightly run, under
+        # a suffix that tells this provider's dumps apart, with a timestamped
+        # copy of each push.
+        VORTEX_PUSH_DB_FTP_HOST: ${{ secrets.VORTEX_PUSH_DB_FTP_HOST }}
+        VORTEX_PUSH_DB_FTP_PORT: ${{ secrets.VORTEX_PUSH_DB_FTP_PORT }}
+        VORTEX_PUSH_DB_FTP_USER: ${{ secrets.VORTEX_PUSH_DB_FTP_USER }}
+        VORTEX_PUSH_DB_FTP_PASS: ${{ secrets.VORTEX_PUSH_DB_FTP_PASS }}
+        VORTEX_PUSH_DB_FTP_FILE: ${{ vars.VORTEX_PUSH_DB_FTP_FILE || 'db.sql' }}
+        VORTEX_PUSH_DB_FILE_SUFFIX: _gha
+        VORTEX_PUSH_DB_TIMESTAMPED_COPY: 1
+        #;> VORTEX_DEV
         VORTEX_FETCH_DB_SOURCE: container_registry
         VORTEX_FETCH_DB_FORCE: 1
         VORTEX_DB_IMAGE: drevops/vortex-dev-mariadb-drupal-data-demo-destination-11.x
@@ -634,6 +716,8 @@ jobs:
             # Deploy container image.
             echo "VORTEX_EXPORT_DB_CONTAINER_REGISTRY_PUSH_PROCEED=1"
             #;> DB_IN_IMAGE
+            # Refresh the database dumps in the fetch sources that are dump stores.
+            echo "VORTEX_PUSH_DB_PROCEED=1"
           } >> "$GITHUB_ENV"
 
       - name: Create cache keys files for database caching
@@ -715,6 +799,30 @@ jobs:
         timeout-minutes: 30
       #;> DB_FETCH_SOURCE_CONTAINER_REGISTRY
 
+      #;< DB_PUSH
+      # Push the freshly-exported sanitized database dump to the database fetch
+      # source. No-op unless a fresh DB was fetched (the fetch semaphore) and the
+      # push was requested (VORTEX_PUSH_DB_PROCEED, in the script).
+      - name: Push DB
+        run: |
+          if [ ! -f /tmp/fetch-db-fresh ]; then echo "==> No freshly fetched database. DB push will not proceed."; exit 0; fi
+          ./vendor/bin/vortex-push-db
+          #;< VORTEX_DEV
+          if [ -n "${VORTEX_PUSH_DB_FTP_HOST:-}" ]; then VORTEX_FETCH_DB_SOURCE=ftp VORTEX_REFRESH_DB_SOURCE=url ./vendor/bin/vortex-push-db; fi
+          #;> VORTEX_DEV
+        timeout-minutes: 30
+      #;> DB_PUSH
+
+      #;< MIGRATION_DB_PUSH
+      # Push the freshly-fetched migration database dump to its fetch source,
+      # next to the primary dump. No-op unless the push was requested.
+      - name: Push migration DB
+        run: |
+          if [ ! -f /tmp/fetch-db-fresh ]; then echo "==> No freshly fetched database. DB push will not proceed."; exit 0; fi
+          VORTEX_DB_INDEX=2 ./vendor/bin/vortex-push-db
+        timeout-minutes: 30
+      #;> MIGRATION_DB_PUSH
+
       # Save cache per default branch and the timestamp.
       # The cache will not be saved if it already exists.
       # Note that the cache fallback flag is enabled for this case in order

.vortex/docs/content/continuous-integration/README.mdx

@@ -79,6 +79,9 @@ site from:
 - Caches the dump, so the `build` job and later runs skip the download
 - On the nightly run of a project that stores its database in a container
   image, pushes the refreshed image to the registry
+- On the nightly run of a project with a dump store, pushes the sanitized
+  dump to the FTP server or S3 bucket (see [Database
+  storage](../development/database-storage.mdx#fetch-from-a-dump-store))
 
 The job exists only when the site provisions from a database dump. With the
 `profile` provision type, the pipeline has no `database` job and no nightly