diff --git a/.circleci/config.yml b/.circleci/config.yml index 9a39ac1d10..23a1515596 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -363,6 +363,32 @@ jobs: no_output_timeout: 30m #;> DB_FETCH_SOURCE_CONTAINER_REGISTRY + #;< DB_PUSH + # Push the freshly-exported sanitized database dump to the database fetch + # source. No-op unless a fresh DB was fetched (the fetch semaphore) and the + # push was requested (VORTEX_PUSH_DB_PROCEED, in the script). + - run: + name: Push DB + command: | + [ ! -f /tmp/fetch-db-fresh ] && echo "==> No freshly fetched database. DB push will not proceed." && exit 0 + ./vendor/bin/vortex-push-db + #;< VORTEX_DEV + if [ -n "${VORTEX_PUSH_DB_FTP_HOST:-}" ]; then VORTEX_FETCH_DB_SOURCE=ftp VORTEX_REFRESH_DB_SOURCE=url ./vendor/bin/vortex-push-db; fi + #;> VORTEX_DEV + no_output_timeout: 30m + #;> DB_PUSH + + #;< MIGRATION_DB_PUSH + # Push the freshly-fetched migration database dump to its fetch source, + # next to the primary dump. No-op unless the push was requested. + - run: + name: Push migration DB + command: | + [ ! -f /tmp/fetch-db-fresh ] && echo "==> No freshly fetched database. DB push will not proceed." && exit 0 + VORTEX_DB_INDEX=2 ./vendor/bin/vortex-push-db + no_output_timeout: 30m + #;> MIGRATION_DB_PUSH + - save_cache: # Save cache per default branch and the timestamp. # The cache will not be saved if it already exists. @@ -387,6 +413,14 @@ jobs: # Deploy container image. VORTEX_EXPORT_DB_CONTAINER_REGISTRY_PUSH_PROCEED: 1 #;> DB_IN_IMAGE + # Refresh the database dumps in the fetch sources that are dump stores. + VORTEX_PUSH_DB_PROCEED: 1 + #;< VORTEX_DEV + # Tell this provider's demo database dumps apart on the test FTP server, + # with a timestamped copy of each push. + VORTEX_PUSH_DB_FILE_SUFFIX: _circleci + VORTEX_PUSH_DB_TIMESTAMPED_COPY: 1 + #;> VORTEX_DEV # Do not build the Drupal front-end. VORTEX_FRONTEND_BUILD_SKIP: 1 #;> !PROVISION_TYPE_PROFILE diff --git a/.circleci/vortex-test-common.yml b/.circleci/vortex-test-common.yml index 371f8c610a..a901d33adc 100644 --- a/.circleci/vortex-test-common.yml +++ b/.circleci/vortex-test-common.yml @@ -183,6 +183,32 @@ jobs: no_output_timeout: 30m #;> DB_FETCH_SOURCE_CONTAINER_REGISTRY + #;< DB_PUSH + # Push the freshly-exported sanitized database dump to the database fetch + # source. No-op unless a fresh DB was fetched (the fetch semaphore) and the + # push was requested (VORTEX_PUSH_DB_PROCEED, in the script). + - run: + name: Push DB + command: | + [ ! -f /tmp/fetch-db-fresh ] && echo "==> No freshly fetched database. DB push will not proceed." && exit 0 + ./vendor/bin/vortex-push-db + #;< VORTEX_DEV + if [ -n "${VORTEX_PUSH_DB_FTP_HOST:-}" ]; then VORTEX_FETCH_DB_SOURCE=ftp VORTEX_REFRESH_DB_SOURCE=url ./vendor/bin/vortex-push-db; fi + #;> VORTEX_DEV + no_output_timeout: 30m + #;> DB_PUSH + + #;< MIGRATION_DB_PUSH + # Push the freshly-fetched migration database dump to its fetch source, + # next to the primary dump. No-op unless the push was requested. + - run: + name: Push migration DB + command: | + [ ! -f /tmp/fetch-db-fresh ] && echo "==> No freshly fetched database. DB push will not proceed." && exit 0 + VORTEX_DB_INDEX=2 ./vendor/bin/vortex-push-db + no_output_timeout: 30m + #;> MIGRATION_DB_PUSH + - save_cache: # Save cache per default branch and the timestamp. # The cache will not be saved if it already exists. diff --git a/.env b/.env index 94f358f542..661a9a946b 100644 --- a/.env +++ b/.env @@ -258,6 +258,21 @@ VORTEX_FETCH_DB_S3_BUCKET= VORTEX_FETCH_DB_S3_REGION=ap-southeast-2 #;> DB_FETCH_SOURCE_S3 +#;< DB_PUSH +# Upstream source of the database dump file, such as url, acquia or lagoon. +# +# Uncomment to make the database fetch source a dump store: the nightly CI +# job fetches the dump from this source, sanitizes it and pushes it to the +# fetch source as the latest dump. The fetch script of this source reads its +# own variables, such as VORTEX_FETCH_DB_URL for url. Add them, and keep +# secrets in CI. +# VORTEX_REFRESH_DB_SOURCE=url + +# Also push a copy of each dump named with a UTC timestamp, such as +# db_20261111-180000.sql, to keep a history of the pushed dumps. +# VORTEX_PUSH_DB_TIMESTAMPED_COPY=1 +#;> DB_PUSH + # Environment to fetch the database from. # # Applies to hosting environments. @@ -333,6 +348,15 @@ VORTEX_FETCH_DB2_S3_BUCKET= VORTEX_FETCH_DB2_S3_REGION=ap-southeast-2 #;> MIGRATION_DB_FETCH_SOURCE_S3 +#;< MIGRATION_DB_PUSH +# Upstream source of the second database dump file, such as url or lagoon. +# +# Uncomment to make the second database fetch source a dump store: the +# nightly CI job fetches the dump from this source and pushes it, as fetched, +# to the fetch source, next to the primary dump. +# VORTEX_REFRESH_DB2_SOURCE=url +#;> MIGRATION_DB_PUSH + # Environment to fetch the second database from. # # Applies to hosting environments. diff --git a/.github/workflows/build-test-deploy.yml b/.github/workflows/build-test-deploy.yml index b9daa0d69d..00e0d1be9b 100644 --- a/.github/workflows/build-test-deploy.yml +++ b/.github/workflows/build-test-deploy.yml @@ -225,6 +225,34 @@ jobs: VORTEX_CI_DB_CACHE_FALLBACK: "yes" # Which branch to use as a source of DB caches. VORTEX_CI_DB_CACHE_BRANCH: "develop" + #;< DB_FETCH_SOURCE_FTP + VORTEX_FETCH_DB_FTP_USER: ${{ secrets.VORTEX_FETCH_DB_FTP_USER }} + VORTEX_FETCH_DB_FTP_PASS: ${{ secrets.VORTEX_FETCH_DB_FTP_PASS }} + #;> DB_FETCH_SOURCE_FTP + #;< DB_FETCH_SOURCE_S3 + VORTEX_FETCH_DB_S3_ACCESS_KEY: ${{ secrets.VORTEX_FETCH_DB_S3_ACCESS_KEY }} + VORTEX_FETCH_DB_S3_SECRET_KEY: ${{ secrets.VORTEX_FETCH_DB_S3_SECRET_KEY }} + #;> DB_FETCH_SOURCE_S3 + #;< MIGRATION_DB_FETCH_SOURCE_FTP + VORTEX_FETCH_DB2_FTP_USER: ${{ secrets.VORTEX_FETCH_DB2_FTP_USER }} + VORTEX_FETCH_DB2_FTP_PASS: ${{ secrets.VORTEX_FETCH_DB2_FTP_PASS }} + #;> MIGRATION_DB_FETCH_SOURCE_FTP + #;< MIGRATION_DB_FETCH_SOURCE_S3 + VORTEX_FETCH_DB2_S3_ACCESS_KEY: ${{ secrets.VORTEX_FETCH_DB2_S3_ACCESS_KEY }} + VORTEX_FETCH_DB2_S3_SECRET_KEY: ${{ secrets.VORTEX_FETCH_DB2_S3_SECRET_KEY }} + #;> MIGRATION_DB_FETCH_SOURCE_S3 + #;< VORTEX_DEV + # FTP server that receives the demo database on the nightly run, under + # a suffix that tells this provider's dumps apart, with a timestamped + # copy of each push. + VORTEX_PUSH_DB_FTP_HOST: ${{ secrets.VORTEX_PUSH_DB_FTP_HOST }} + VORTEX_PUSH_DB_FTP_PORT: ${{ secrets.VORTEX_PUSH_DB_FTP_PORT }} + VORTEX_PUSH_DB_FTP_USER: ${{ secrets.VORTEX_PUSH_DB_FTP_USER }} + VORTEX_PUSH_DB_FTP_PASS: ${{ secrets.VORTEX_PUSH_DB_FTP_PASS }} + VORTEX_PUSH_DB_FTP_FILE: ${{ vars.VORTEX_PUSH_DB_FTP_FILE || 'db.sql' }} + VORTEX_PUSH_DB_FILE_SUFFIX: _gha + VORTEX_PUSH_DB_TIMESTAMPED_COPY: 1 + #;> VORTEX_DEV steps: # Frees disk space by removing preinstalled toolchains unused by this project. Steps run in a container, so the host is reached via the Docker socket. @@ -281,6 +309,8 @@ jobs: # Deploy container image. echo "VORTEX_EXPORT_DB_CONTAINER_REGISTRY_PUSH_PROCEED=1" #;> DB_IN_IMAGE + # Refresh the database dumps in the fetch sources that are dump stores. + echo "VORTEX_PUSH_DB_PROCEED=1" } >> "$GITHUB_ENV" - name: Create cache keys files for database caching @@ -343,6 +373,30 @@ jobs: timeout-minutes: 30 #;> DB_FETCH_SOURCE_CONTAINER_REGISTRY + #;< DB_PUSH + # Push the freshly-exported sanitized database dump to the database fetch + # source. No-op unless a fresh DB was fetched (the fetch semaphore) and the + # push was requested (VORTEX_PUSH_DB_PROCEED, in the script). + - name: Push DB + run: | + if [ ! -f /tmp/fetch-db-fresh ]; then echo "==> No freshly fetched database. DB push will not proceed."; exit 0; fi + ./vendor/bin/vortex-push-db + #;< VORTEX_DEV + if [ -n "${VORTEX_PUSH_DB_FTP_HOST:-}" ]; then VORTEX_FETCH_DB_SOURCE=ftp VORTEX_REFRESH_DB_SOURCE=url ./vendor/bin/vortex-push-db; fi + #;> VORTEX_DEV + timeout-minutes: 30 + #;> DB_PUSH + + #;< MIGRATION_DB_PUSH + # Push the freshly-fetched migration database dump to its fetch source, + # next to the primary dump. No-op unless the push was requested. + - name: Push migration DB + run: | + if [ ! -f /tmp/fetch-db-fresh ]; then echo "==> No freshly fetched database. DB push will not proceed."; exit 0; fi + VORTEX_DB_INDEX=2 ./vendor/bin/vortex-push-db + timeout-minutes: 30 + #;> MIGRATION_DB_PUSH + # Save cache per default branch and the timestamp. # The cache will not be saved if it already exists. # Note that the cache fallback flag is enabled for this case in order diff --git a/.github/workflows/vortex-test-didi.yml b/.github/workflows/vortex-test-didi.yml index 4e8807d301..f2a7401592 100644 --- a/.github/workflows/vortex-test-didi.yml +++ b/.github/workflows/vortex-test-didi.yml @@ -64,6 +64,34 @@ jobs: VORTEX_CI_DB_CACHE_FALLBACK: "yes" # Which branch to use as a source of DB caches. VORTEX_CI_DB_CACHE_BRANCH: vortex-dev-didi-fi + #;< DB_FETCH_SOURCE_FTP + VORTEX_FETCH_DB_FTP_USER: ${{ secrets.VORTEX_FETCH_DB_FTP_USER }} + VORTEX_FETCH_DB_FTP_PASS: ${{ secrets.VORTEX_FETCH_DB_FTP_PASS }} + #;> DB_FETCH_SOURCE_FTP + #;< DB_FETCH_SOURCE_S3 + VORTEX_FETCH_DB_S3_ACCESS_KEY: ${{ secrets.VORTEX_FETCH_DB_S3_ACCESS_KEY }} + VORTEX_FETCH_DB_S3_SECRET_KEY: ${{ secrets.VORTEX_FETCH_DB_S3_SECRET_KEY }} + #;> DB_FETCH_SOURCE_S3 + #;< MIGRATION_DB_FETCH_SOURCE_FTP + VORTEX_FETCH_DB2_FTP_USER: ${{ secrets.VORTEX_FETCH_DB2_FTP_USER }} + VORTEX_FETCH_DB2_FTP_PASS: ${{ secrets.VORTEX_FETCH_DB2_FTP_PASS }} + #;> MIGRATION_DB_FETCH_SOURCE_FTP + #;< MIGRATION_DB_FETCH_SOURCE_S3 + VORTEX_FETCH_DB2_S3_ACCESS_KEY: ${{ secrets.VORTEX_FETCH_DB2_S3_ACCESS_KEY }} + VORTEX_FETCH_DB2_S3_SECRET_KEY: ${{ secrets.VORTEX_FETCH_DB2_S3_SECRET_KEY }} + #;> MIGRATION_DB_FETCH_SOURCE_S3 + #;< VORTEX_DEV + # FTP server that receives the demo database on the nightly run, under + # a suffix that tells this provider's dumps apart, with a timestamped + # copy of each push. + VORTEX_PUSH_DB_FTP_HOST: ${{ secrets.VORTEX_PUSH_DB_FTP_HOST }} + VORTEX_PUSH_DB_FTP_PORT: ${{ secrets.VORTEX_PUSH_DB_FTP_PORT }} + VORTEX_PUSH_DB_FTP_USER: ${{ secrets.VORTEX_PUSH_DB_FTP_USER }} + VORTEX_PUSH_DB_FTP_PASS: ${{ secrets.VORTEX_PUSH_DB_FTP_PASS }} + VORTEX_PUSH_DB_FTP_FILE: ${{ vars.VORTEX_PUSH_DB_FTP_FILE || 'db.sql' }} + VORTEX_PUSH_DB_FILE_SUFFIX: _gha + VORTEX_PUSH_DB_TIMESTAMPED_COPY: 1 + #;> VORTEX_DEV VORTEX_FETCH_DB_SOURCE: url VORTEX_FETCH_DB_FORCE: 1 VORTEX_DB_IMAGE: drevops/vortex-dev-mariadb-drupal-data-demo-destination-11.x @@ -129,6 +157,8 @@ jobs: # Deploy container image. echo "VORTEX_EXPORT_DB_CONTAINER_REGISTRY_PUSH_PROCEED=1" #;> DB_IN_IMAGE + # Refresh the database dumps in the fetch sources that are dump stores. + echo "VORTEX_PUSH_DB_PROCEED=1" } >> "$GITHUB_ENV" - name: Create cache keys files for database caching @@ -191,6 +221,30 @@ jobs: timeout-minutes: 30 #;> DB_FETCH_SOURCE_CONTAINER_REGISTRY + #;< DB_PUSH + # Push the freshly-exported sanitized database dump to the database fetch + # source. No-op unless a fresh DB was fetched (the fetch semaphore) and the + # push was requested (VORTEX_PUSH_DB_PROCEED, in the script). + - name: Push DB + run: | + if [ ! -f /tmp/fetch-db-fresh ]; then echo "==> No freshly fetched database. DB push will not proceed."; exit 0; fi + ./vendor/bin/vortex-push-db + #;< VORTEX_DEV + if [ -n "${VORTEX_PUSH_DB_FTP_HOST:-}" ]; then VORTEX_FETCH_DB_SOURCE=ftp VORTEX_REFRESH_DB_SOURCE=url ./vendor/bin/vortex-push-db; fi + #;> VORTEX_DEV + timeout-minutes: 30 + #;> DB_PUSH + + #;< MIGRATION_DB_PUSH + # Push the freshly-fetched migration database dump to its fetch source, + # next to the primary dump. No-op unless the push was requested. + - name: Push migration DB + run: | + if [ ! -f /tmp/fetch-db-fresh ]; then echo "==> No freshly fetched database. DB push will not proceed."; exit 0; fi + VORTEX_DB_INDEX=2 ./vendor/bin/vortex-push-db + timeout-minutes: 30 + #;> MIGRATION_DB_PUSH + # Save cache per default branch and the timestamp. # The cache will not be saved if it already exists. # Note that the cache fallback flag is enabled for this case in order @@ -570,6 +624,34 @@ jobs: VORTEX_CI_DB_CACHE_FALLBACK: "yes" # Which branch to use as a source of DB caches. VORTEX_CI_DB_CACHE_BRANCH: vortex-dev-didi-ii + #;< DB_FETCH_SOURCE_FTP + VORTEX_FETCH_DB_FTP_USER: ${{ secrets.VORTEX_FETCH_DB_FTP_USER }} + VORTEX_FETCH_DB_FTP_PASS: ${{ secrets.VORTEX_FETCH_DB_FTP_PASS }} + #;> DB_FETCH_SOURCE_FTP + #;< DB_FETCH_SOURCE_S3 + VORTEX_FETCH_DB_S3_ACCESS_KEY: ${{ secrets.VORTEX_FETCH_DB_S3_ACCESS_KEY }} + VORTEX_FETCH_DB_S3_SECRET_KEY: ${{ secrets.VORTEX_FETCH_DB_S3_SECRET_KEY }} + #;> DB_FETCH_SOURCE_S3 + #;< MIGRATION_DB_FETCH_SOURCE_FTP + VORTEX_FETCH_DB2_FTP_USER: ${{ secrets.VORTEX_FETCH_DB2_FTP_USER }} + VORTEX_FETCH_DB2_FTP_PASS: ${{ secrets.VORTEX_FETCH_DB2_FTP_PASS }} + #;> MIGRATION_DB_FETCH_SOURCE_FTP + #;< MIGRATION_DB_FETCH_SOURCE_S3 + VORTEX_FETCH_DB2_S3_ACCESS_KEY: ${{ secrets.VORTEX_FETCH_DB2_S3_ACCESS_KEY }} + VORTEX_FETCH_DB2_S3_SECRET_KEY: ${{ secrets.VORTEX_FETCH_DB2_S3_SECRET_KEY }} + #;> MIGRATION_DB_FETCH_SOURCE_S3 + #;< VORTEX_DEV + # FTP server that receives the demo database on the nightly run, under + # a suffix that tells this provider's dumps apart, with a timestamped + # copy of each push. + VORTEX_PUSH_DB_FTP_HOST: ${{ secrets.VORTEX_PUSH_DB_FTP_HOST }} + VORTEX_PUSH_DB_FTP_PORT: ${{ secrets.VORTEX_PUSH_DB_FTP_PORT }} + VORTEX_PUSH_DB_FTP_USER: ${{ secrets.VORTEX_PUSH_DB_FTP_USER }} + VORTEX_PUSH_DB_FTP_PASS: ${{ secrets.VORTEX_PUSH_DB_FTP_PASS }} + VORTEX_PUSH_DB_FTP_FILE: ${{ vars.VORTEX_PUSH_DB_FTP_FILE || 'db.sql' }} + VORTEX_PUSH_DB_FILE_SUFFIX: _gha + VORTEX_PUSH_DB_TIMESTAMPED_COPY: 1 + #;> VORTEX_DEV VORTEX_FETCH_DB_SOURCE: container_registry VORTEX_FETCH_DB_FORCE: 1 VORTEX_DB_IMAGE: drevops/vortex-dev-mariadb-drupal-data-demo-destination-11.x @@ -634,6 +716,8 @@ jobs: # Deploy container image. echo "VORTEX_EXPORT_DB_CONTAINER_REGISTRY_PUSH_PROCEED=1" #;> DB_IN_IMAGE + # Refresh the database dumps in the fetch sources that are dump stores. + echo "VORTEX_PUSH_DB_PROCEED=1" } >> "$GITHUB_ENV" - name: Create cache keys files for database caching @@ -715,6 +799,30 @@ jobs: timeout-minutes: 30 #;> DB_FETCH_SOURCE_CONTAINER_REGISTRY + #;< DB_PUSH + # Push the freshly-exported sanitized database dump to the database fetch + # source. No-op unless a fresh DB was fetched (the fetch semaphore) and the + # push was requested (VORTEX_PUSH_DB_PROCEED, in the script). + - name: Push DB + run: | + if [ ! -f /tmp/fetch-db-fresh ]; then echo "==> No freshly fetched database. DB push will not proceed."; exit 0; fi + ./vendor/bin/vortex-push-db + #;< VORTEX_DEV + if [ -n "${VORTEX_PUSH_DB_FTP_HOST:-}" ]; then VORTEX_FETCH_DB_SOURCE=ftp VORTEX_REFRESH_DB_SOURCE=url ./vendor/bin/vortex-push-db; fi + #;> VORTEX_DEV + timeout-minutes: 30 + #;> DB_PUSH + + #;< MIGRATION_DB_PUSH + # Push the freshly-fetched migration database dump to its fetch source, + # next to the primary dump. No-op unless the push was requested. + - name: Push migration DB + run: | + if [ ! -f /tmp/fetch-db-fresh ]; then echo "==> No freshly fetched database. DB push will not proceed."; exit 0; fi + VORTEX_DB_INDEX=2 ./vendor/bin/vortex-push-db + timeout-minutes: 30 + #;> MIGRATION_DB_PUSH + # Save cache per default branch and the timestamp. # The cache will not be saved if it already exists. # Note that the cache fallback flag is enabled for this case in order diff --git a/.vortex/docs/content/continuous-integration/README.mdx b/.vortex/docs/content/continuous-integration/README.mdx index 75f6286b9f..3668dbaabd 100644 --- a/.vortex/docs/content/continuous-integration/README.mdx +++ b/.vortex/docs/content/continuous-integration/README.mdx @@ -79,6 +79,9 @@ site from: - Caches the dump, so the `build` job and later runs skip the download - On the nightly run of a project that stores its database in a container image, pushes the refreshed image to the registry +- On the nightly run of a project with a dump store, pushes the sanitized + dump to the FTP server or S3 bucket (see [Database + storage](../development/database-storage.mdx#fetch-from-a-dump-store)) The job exists only when the site provisions from a database dump. With the `profile` provision type, the pipeline has no `database` job and no nightly diff --git a/.vortex/docs/content/development/database-storage.mdx b/.vortex/docs/content/development/database-storage.mdx new file mode 100644 index 0000000000..9b3102336c --- /dev/null +++ b/.vortex/docs/content/development/database-storage.mdx @@ -0,0 +1,239 @@ +--- +sidebar_label: Database storage +sidebar_position: 6 +--- + +# Database storage + +Your site's database reaches a developer's machine and CI in 1 of 3 ways. +The fetch source in `VORTEX_FETCH_DB_SOURCE` picks the way. This page +shows how fetching, refreshing and pushing the database work in each case. +It also covers setting up an FTP server or an S3 bucket as a dump store, +so developers get a sanitized dump without access to your hosting. + +The page covers these commands: + +- `ahoy fetch-db` (`vortex-fetch-db`) downloads a dump into `.data` +- `vortex-refresh-db` provisions a site from the fetched dump in CI and + exports it again +- `vortex-push-db` uploads the exported dump to a dump store on the + nightly CI run + +## Fetch from the source + +With the `url`, `ftp`, `acquia`, `lagoon` or `s3` source, every +environment downloads the dump from that source. Developers need +credentials for it. + +```text + Developer CI (every run with no cached dump) + ───────── ────────────────────────────────── + ahoy fetch-db Fetch DB + │ │ + ▼ ▼ + .data/db.sql ◀── source ──▶ .data/db.sql + │ │ + ▼ ▼ + ahoy provision Export DB (vortex-refresh-db) + │ provisions the dump and exports + │ it as fetched + ▼ + cache ──▶ build job +``` + +## Fetch a database image + +With the `container_registry` source, the database lives in a container +image. Developers and CI pull the image. The nightly CI run rebuilds the +image from a dump file and pushes it back. See [Dump file source for a +database +image](../continuous-integration/README.mdx#dump-file-source-for-a-database-image). + +```text + Nightly CI + ────────── + Fetch DB ── VORTEX_REFRESH_DB_SOURCE (for example, lagoon) + │ + ▼ + Export DB (vortex-refresh-db) + │ imports the dump into a clean base image + ▼ + Deploy DB image (vortex-push-db-image) ──▶ container registry +``` + +## Fetch from a dump store + +A dump store is an `ftp` or `s3` source with an upstream source set in +`VORTEX_REFRESH_DB_SOURCE`, such as `lagoon`. Developers and branch builds +fetch from the store and need no hosting access. The nightly CI run +refreshes the store from the upstream source: + +```text + Nightly CI (VORTEX_PUSH_DB_PROCEED=1) + ───────────────────────────────────── + Fetch DB + │ fetches from VORTEX_REFRESH_DB_SOURCE in place of the store + ▼ + Export DB (vortex-refresh-db) + │ provisions the dump, sanitizes it and exports it + ▼ + Push DB (vortex-push-db) + │ 1. replaces db.sql with the new dump + │ 2. with VORTEX_PUSH_DB_TIMESTAMPED_COPY=1, adds a timestamped copy + ▼ + FTP server or S3 bucket + ├── db.sql ◀── ahoy fetch-db, branch builds + ├── db_20261110-180000.sql timestamped copies, opt-in + └── db_20261111-180000.sql +``` + +`db.sql` always holds the latest dump, and fetching downloads it. With a +file suffix, the latest dump is the suffixed file, such as `db_gha.sql` +(see [Tell pushes apart](#tell-pushes-apart)). By default, it's the only +file a push creates. To keep a history of the +pushed dumps, set `VORTEX_PUSH_DB_TIMESTAMPED_COPY=1` in `.env`. Each push +then also creates a copy named with its UTC time, identical to `db.sql` +at that moment. Nothing removes the copies, so clear out the old ones +yourself. + +Every other CI run fetches from the store, so only the nightly run needs +the credentials of the upstream source. + +The pushed dump leaves your hosting and CI, so it's always sanitized. The +nightly run stops with an error when `VORTEX_PROVISION_SANITIZE_DB_SKIP` +is `1`. + +### Multiple databases + +The migration database pushes its own dump file to its own fetch source. +Its files sit next to the primary dump in the same store: + +```text + FTP server or S3 bucket + ├── db.sql primary database (VORTEX_FETCH_DB_FTP_FILE) + ├── db_20261111-180000.sql + ├── db2.sql migration database (VORTEX_FETCH_DB2_FTP_FILE) + └── db2_20261111-180000.sql +``` + +To make the migration source a dump store, set +`VORTEX_REFRESH_DB2_SOURCE` to its upstream source. The +**Push migration DB** step pushes the migration dump as fetched, because +**Vortex** sanitizes only the Drupal database. + +## Set up an FTP dump store + +Before you begin, create an FTP account that can write to a directory on +the server. The push uploads the dump under a temporary name, then renames +it into place, so a failed upload keeps the latest dump. The server must +allow the `DELE`, `RNFR` and `RNTO` commands, which standard FTP servers +do. + +1. In `.env`, set the fetch source and the server: + + ```shell + VORTEX_FETCH_DB_SOURCE=ftp + VORTEX_FETCH_DB_FTP_HOST=ftp.example.com + VORTEX_FETCH_DB_FTP_PORT=21 + VORTEX_FETCH_DB_FTP_FILE=your_site/db.sql + ``` + +2. In `.env`, uncomment `VORTEX_REFRESH_DB_SOURCE` and set it to your + upstream source, for example `lagoon`. Add the variables and secrets of + that source from the table in [Dump file source for a database + image](../continuous-integration/README.mdx#dump-file-source-for-a-database-image). +3. Add `VORTEX_FETCH_DB_FTP_USER` and `VORTEX_FETCH_DB_FTP_PASS` as CI + secrets. GitHub Actions passes them to the `database` job. CircleCI + reads them from the project environment variables. +4. Ask each developer to set `VORTEX_FETCH_DB_FTP_USER` and + `VORTEX_FETCH_DB_FTP_PASS` in `.env.local`. +5. After the next nightly run, confirm that the **Push DB** step ends with + `Finished database push.` and that the server holds `db.sql`. + +To give CI an account with write access and developers a read-only one, +set the CI account in `VORTEX_PUSH_DB_FTP_USER` and +`VORTEX_PUSH_DB_FTP_PASS`. On GitHub Actions, map those 2 secrets in the +`env` block of the `database` job. + +FTP sends the credentials and the dump unencrypted. Use a server on a +network you trust, or the S3 store. + +## Set up an S3 dump store + +Before you begin, create a private S3 bucket with public access blocked. + +1. Create an IAM user for CI with this policy. Replace `your-bucket` with + your bucket name: + + ```json + { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": ["s3:GetObject", "s3:PutObject"], + "Resource": "arn:aws:s3:::your-bucket/*" + } + ] + } + ``` + +2. In `.env`, set the fetch source and the bucket: + + ```shell + VORTEX_FETCH_DB_SOURCE=s3 + VORTEX_FETCH_DB_S3_BUCKET=your-bucket + VORTEX_FETCH_DB_S3_REGION=ap-southeast-2 + VORTEX_FETCH_DB_S3_PREFIX=your_site + ``` + +3. In `.env`, uncomment `VORTEX_REFRESH_DB_SOURCE` and set it to your + upstream source, and add the variables and secrets of that source. +4. Add the access key of the IAM user as the + `VORTEX_FETCH_DB_S3_ACCESS_KEY` and `VORTEX_FETCH_DB_S3_SECRET_KEY` CI + secrets. +5. Create a second IAM user for developers with `s3:GetObject` only. Ask + each developer to set its keys in `.env.local`. +6. After the next nightly run, confirm that the **Push DB** step ends with + `Finished database push.` and that the bucket holds `db.sql`. + +S3 makes the timestamped copy on its side, so the dump is uploaded once. +S3 copies a dump of up to 5 GB in 1 request, so the timestamped copy +fails for a larger dump. On FTP, the timestamped copy is a second upload. + +## Tell pushes apart + +`VORTEX_PUSH_DB_FILE_SUFFIX` adds a suffix before the extension of the +pushed files, such as `db_gha.sql` and `db_gha_20261111-180000.sql`. Use a +different suffix in each place that pushes to the same store. + +The push takes its remote filename from the fetch, then adds the suffix. To +fetch a suffixed dump from an FTP store, set both filenames, so the suffix +appears once: + +```shell +# The push adds the suffix to this name. +VORTEX_PUSH_DB_FTP_FILE=your_site/db.sql +# The fetch reads the suffixed name. +VORTEX_FETCH_DB_FTP_FILE=your_site/db_gha.sql +``` + +The S3 fetch names the local dump after the remote file. To fetch a +suffixed dump from an S3 store, rename the dump everywhere with +`VORTEX_DB_FILE`, which the fetch, the refresh and the import all read. +Then set the push remote filename to the name without the suffix: + +```shell +# The fetch, the refresh and the import use the suffixed name. +VORTEX_DB_FILE=db_gha.sql +# The push adds the suffix to this name. +VORTEX_PUSH_DB_S3_REMOTE_FILE=db.sql +``` + +## What's next + +- [Variables reference](variables.mdx) lists every `VORTEX_PUSH_DB_` + variable +- [Provisioning](provision.mdx) covers importing the fetched dump +- [Continuous integration](../continuous-integration/README.mdx) covers + the `database` job and its caching diff --git a/.vortex/docs/content/development/provision.mdx b/.vortex/docs/content/development/provision.mdx index ec35e263e2..4117778be9 100644 --- a/.vortex/docs/content/development/provision.mdx +++ b/.vortex/docs/content/development/provision.mdx @@ -379,6 +379,10 @@ following values: Any other value stops the command with an error that lists these values. +An `ftp` or `s3` source can also be a dump store that the nightly CI run +refreshes with a sanitized dump. See [Database +storage](database-storage.mdx). + Each source reads its own settings, which the [Variables reference](variables.mdx) lists. The hosting sources copy the environment named in `VORTEX_FETCH_DB_ENVIRONMENT`. `.env` ships with diff --git a/.vortex/docs/content/development/variables.mdx b/.vortex/docs/content/development/variables.mdx index 680f9ab2ac..cade9da4c0 100644 --- a/.vortex/docs/content/development/variables.mdx +++ b/.vortex/docs/content/development/variables.mdx @@ -367,7 +367,7 @@ The list below is automatically generated with [Shellvar](https://github.com/ale | `VORTEX_FETCH_DB_S3_PREFIX` | S3 prefix (path within the bucket). | `${S3_PREFIX}` | `.vortex/tooling/src/vortex-fetch-db-s3` | | `VORTEX_FETCH_DB_S3_REGION` | AWS S3 region. | `ap-southeast-2` | `.env`, `.vortex/tooling/src/vortex-fetch-db-s3` | | `VORTEX_FETCH_DB_S3_SECRET_KEY` | AWS secret key for S3 database fetch. | `UNDEFINED` | `.env.local.example`, `.vortex/tooling/src/vortex-fetch-db-s3` | -| `VORTEX_FETCH_DB_SOURCE` | Database fetch source. | `url` | `.env`, `.vortex/tooling/src/vortex-fetch-db` | +| `VORTEX_FETCH_DB_SOURCE` | Database fetch source. | `url` | `.env`, `.vortex/tooling/src/vortex-fetch-db`, `.vortex/tooling/src/vortex-push-db` | | `VORTEX_FETCH_DB_SSH_FILE` | Default SSH file used if custom fingerprint is not provided. | `${HOME}/.ssh/id_rsa` | `.vortex/tooling/src/vortex-fetch-db-lagoon` | | `VORTEX_FETCH_DB_SSH_FINGERPRINT` | SSH key fingerprint used to connect to a remote. | `UNDEFINED` | `.vortex/tooling/src/vortex-fetch-db-lagoon` | | `VORTEX_FETCH_DB_UNZIP_PASSWORD` | Password for unzipping password-protected zip files. | `UNDEFINED` | `.vortex/tooling/src/vortex-fetch-db-url` | @@ -536,17 +536,27 @@ The list below is automatically generated with [Shellvar](https://github.com/ale | `VORTEX_PUSH_CONTAINER_REGISTRY_MAP` | Comma-separated map of container services and images to push in format "service1=org/image1,service2=org/image2". | `UNDEFINED` | `.vortex/tooling/src/vortex-push-container-registry`, `.vortex/tooling/src/vortex-push-db-image` | | `VORTEX_PUSH_CONTAINER_REGISTRY_PASS` | The password to login into the container registry. | `${VORTEX_CONTAINER_REGISTRY_PASS}` | `.vortex/tooling/src/vortex-push-container-registry` | | `VORTEX_PUSH_CONTAINER_REGISTRY_USER` | The username to login into the container registry. | `${VORTEX_CONTAINER_REGISTRY_USER}` | `.vortex/tooling/src/vortex-push-container-registry` | +| `VORTEX_PUSH_DB_FILE_SUFFIX` | Suffix for the names of the pushed database dump files.

Goes before the extension, so 'db.sql' becomes 'db_gha.sql' with '_gha'. Use a different suffix in each place that pushes to the same storage. A fetch reads the suffixed name only when its remote file name includes it. | `UNDEFINED` | `.vortex/tooling/src/vortex-push-db-ftp`, `.vortex/tooling/src/vortex-push-db-s3` | +| `VORTEX_PUSH_DB_FTP_DB_DIR` | Directory with database dump file. | `./.data` | `.vortex/tooling/src/vortex-push-db-ftp` | +| `VORTEX_PUSH_DB_FTP_DB_FILE` | Database dump file name.

Defaults to the name the database is fetched and refreshed with. | `db.sql` | `.vortex/tooling/src/vortex-push-db-ftp` | +| `VORTEX_PUSH_DB_FTP_FILE` | The remote file name, including any directories. | `UNDEFINED` | `.vortex/tooling/src/vortex-push-db-ftp` | +| `VORTEX_PUSH_DB_FTP_HOST` | The FTP host. | `UNDEFINED` | `.vortex/tooling/src/vortex-push-db-ftp` | +| `VORTEX_PUSH_DB_FTP_PASS` | The FTP password. | `UNDEFINED` | `.vortex/tooling/src/vortex-push-db-ftp` | +| `VORTEX_PUSH_DB_FTP_PORT` | The FTP port. | `21` | `.vortex/tooling/src/vortex-push-db-ftp` | +| `VORTEX_PUSH_DB_FTP_USER` | The FTP user. | `UNDEFINED` | `.vortex/tooling/src/vortex-push-db-ftp` | +| `VORTEX_PUSH_DB_PROCEED` | Proceed with push.

Set to `1` in the nightly CI job only, so the storage is refreshed once a day and never from a feature branch build. | `UNDEFINED` | `.vortex/tooling/src/vortex-push-db` | | `VORTEX_PUSH_DB_S3_ACCESS_KEY` | AWS access key. | `${S3_ACCESS_KEY}` | `.vortex/tooling/src/vortex-push-db-s3` | | `VORTEX_PUSH_DB_S3_BUCKET` | S3 bucket name. | `${S3_BUCKET}` | `.vortex/tooling/src/vortex-push-db-s3` | | `VORTEX_PUSH_DB_S3_DB_DIR` | Directory with database dump file. | `./.data` | `.vortex/tooling/src/vortex-push-db-s3` | -| `VORTEX_PUSH_DB_S3_DB_FILE` | Database dump file name. | `db.sql` | `.vortex/tooling/src/vortex-push-db-s3` | +| `VORTEX_PUSH_DB_S3_DB_FILE` | Database dump file name.

Defaults to the name the database is fetched and refreshed with. | `db.sql` | `.vortex/tooling/src/vortex-push-db-s3` | | `VORTEX_PUSH_DB_S3_PREFIX` | S3 prefix (path within the bucket). | `${S3_PREFIX}` | `.vortex/tooling/src/vortex-push-db-s3` | | `VORTEX_PUSH_DB_S3_REGION` | S3 region. | `${S3_REGION}` | `.vortex/tooling/src/vortex-push-db-s3` | -| `VORTEX_PUSH_DB_S3_REMOTE_FILE` | Remote database dump file name. | `db.sql` | `.vortex/tooling/src/vortex-push-db-s3` | +| `VORTEX_PUSH_DB_S3_REMOTE_FILE` | Remote database dump file name.

Defaults to the remote file name the database is fetched from, then to the database dump file name. | `${VORTEX_PUSH_DB_S3_DB_FILE}` | `.vortex/tooling/src/vortex-push-db-s3` | | `VORTEX_PUSH_DB_S3_SECRET_KEY` | AWS secret key. | `${S3_SECRET_KEY}` | `.vortex/tooling/src/vortex-push-db-s3` | | `VORTEX_PUSH_DB_S3_STORAGE_CLASS` | S3 storage class. | `STANDARD` | `.vortex/tooling/src/vortex-push-db-s3` | +| `VORTEX_PUSH_DB_TIMESTAMPED_COPY` | Also push a copy of the database dump file named with a UTC timestamp.

Set to `1` to keep a history of the pushed dumps. The fetch never reads the timestamped copies. | `UNDEFINED` | `.vortex/tooling/src/vortex-push-db-ftp`, `.vortex/tooling/src/vortex-push-db-s3` | | `VORTEX_REFRESH_DB_PROCEED` | Proceed with the refresh of an indexed database.

The primary database is always refreshed. An indexed database, such as the migration database with VORTEX_DB_INDEX=`2`, is refreshed only when VORTEX_REFRESH_DB2_PROCEED is set to `1`. | `UNDEFINED` | `.vortex/tooling/src/vortex-refresh-db` | -| `VORTEX_REFRESH_DB_SOURCE` | Fetch source of the database dump file to import into the base image.

Required when the fetch source is 'container_registry' and a base image is set: the database image is built from a dump file, so it needs a dump file source such as 'url', 'ftp', 'acquia', 'lagoon' or 's3'. An indexed database uses VORTEX_REFRESH_DB2_SOURCE. | `UNDEFINED` | `.vortex/tooling/src/vortex-refresh-db` | +| `VORTEX_REFRESH_DB_SOURCE` | Upstream source of the pushed database dump file. | `UNDEFINED` | `.vortex/tooling/src/vortex-push-db`, `.vortex/tooling/src/vortex-refresh-db` | | `VORTEX_RELEASE_VERSION_SCHEME` | Versioning scheme used for releases.

Can be one of: calver, semver, other @see https://www.vortextemplate.com/docs/releasing | `calver` | `.env` | | `VORTEX_SHOW_LOGIN` | Show one-time login link. | `UNDEFINED` | `.vortex/tooling/src/vortex-info` | | `VORTEX_SSH_DISABLE_STRICT_HOST_KEY_CHECKING` | Disable strict host key checking in SSH. | `0` | `.vortex/tooling/src/vortex-setup-ssh` | diff --git a/.vortex/docs/content/features.mdx b/.vortex/docs/content/features.mdx index a30b487bdf..860fafd525 100644 --- a/.vortex/docs/content/features.mdx +++ b/.vortex/docs/content/features.mdx @@ -169,7 +169,8 @@ import { - _Lint_ the code without provisioning a site - _Fetch the database_ from the hosting provider and cache it for the - day, with a nightly refresh of the database container image + day, with a nightly refresh of the database image or of a sanitized + dump on FTP or S3 - _Build_ and test a fully provisioned site, with [Behat](https://docs.behat.org/en/latest/) tests split across parallel containers diff --git a/.vortex/docs/cspell.json b/.vortex/docs/cspell.json index 8a3db57a56..681610a5d9 100644 --- a/.vortex/docs/cspell.json +++ b/.vortex/docs/cspell.json @@ -18,6 +18,8 @@ "MAILHUB", "NRAK", "Olivero", + "RNFR", + "RNTO", "REDIS", "Runsheet", "Runsheets", diff --git a/.vortex/installer/src/Prompts/Handlers/DatabaseFetchSource.php b/.vortex/installer/src/Prompts/Handlers/DatabaseFetchSource.php index 57c5f1841a..a1d628e4ad 100644 --- a/.vortex/installer/src/Prompts/Handlers/DatabaseFetchSource.php +++ b/.vortex/installer/src/Prompts/Handlers/DatabaseFetchSource.php @@ -147,6 +147,12 @@ public function process(): void { if ($v !== self::CONTAINER_REGISTRY && !$is_migration_in_image) { File::removeTokenAsync('DB_IN_IMAGE'); } + + // DB_PUSH wraps the nightly push of the sanitized dump, which only an + // 'ftp' or 's3' source accepts. + if (!in_array($v, [self::FTP, self::S3], TRUE)) { + File::removeTokenAsync('DB_PUSH'); + } } } diff --git a/.vortex/installer/src/Prompts/Handlers/MigrationFetchSource.php b/.vortex/installer/src/Prompts/Handlers/MigrationFetchSource.php index 8c87b6a1ed..e0be98f19d 100644 --- a/.vortex/installer/src/Prompts/Handlers/MigrationFetchSource.php +++ b/.vortex/installer/src/Prompts/Handlers/MigrationFetchSource.php @@ -141,6 +141,12 @@ public function process(): void { File::removeTokenAsync($token); } } + + // MIGRATION_DB_PUSH wraps the nightly push of the migration dump, which + // only an 'ftp' or 's3' source accepts. + if (!in_array($v, [self::FTP, self::S3], TRUE)) { + File::removeTokenAsync('MIGRATION_DB_PUSH'); + } } } diff --git a/.vortex/installer/tests/Fixtures/handler_process/_baseline/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/_baseline/.github/workflows/build-test-deploy.yml index b7059b0f25..3580a1e345 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/_baseline/.github/workflows/build-test-deploy.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/_baseline/.github/workflows/build-test-deploy.yml @@ -241,6 +241,8 @@ jobs: { echo "VORTEX_CI_DB_CACHE_FALLBACK=no" echo "VORTEX_FRONTEND_BUILD_SKIP=1" + # Refresh the database dumps in the fetch sources that are dump stores. + echo "VORTEX_PUSH_DB_PROCEED=1" } >> "$GITHUB_ENV" - name: Create cache keys files for database caching diff --git a/.vortex/installer/tests/Fixtures/handler_process/ciprovider_circleci/.circleci/config.yml b/.vortex/installer/tests/Fixtures/handler_process/ciprovider_circleci/.circleci/config.yml index f56bdf626c..b59dde79d2 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/ciprovider_circleci/.circleci/config.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/ciprovider_circleci/.circleci/config.yml @@ -307,6 +307,8 @@ jobs: VORTEX_DEPLOY_SSH_FINGERPRINT: *deploy_ssh_fingerprint # Enforce fresh DB build (do not rely on fallback caches). VORTEX_CI_DB_CACHE_FALLBACK: 'no' + # Refresh the database dumps in the fetch sources that are dump stores. + VORTEX_PUSH_DB_PROCEED: 1 # Do not build the Drupal front-end. VORTEX_FRONTEND_BUILD_SKIP: 1 diff --git a/.vortex/installer/tests/Fixtures/handler_process/code_coverage_provider_codecov/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/code_coverage_provider_codecov/.github/workflows/build-test-deploy.yml index 65a0cc3f96..613d58ed05 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/code_coverage_provider_codecov/.github/workflows/build-test-deploy.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/code_coverage_provider_codecov/.github/workflows/build-test-deploy.yml @@ -1,4 +1,4 @@ -@@ -490,6 +490,17 @@ +@@ -492,6 +492,17 @@ hide_and_recreate: true diff --git a/.vortex/installer/tests/Fixtures/handler_process/code_coverage_provider_codecov_circleci/.circleci/config.yml b/.vortex/installer/tests/Fixtures/handler_process/code_coverage_provider_codecov_circleci/.circleci/config.yml index 095f6fc249..ea65c0eaad 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/code_coverage_provider_codecov_circleci/.circleci/config.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/code_coverage_provider_codecov_circleci/.circleci/config.yml @@ -307,6 +307,8 @@ jobs: VORTEX_DEPLOY_SSH_FINGERPRINT: *deploy_ssh_fingerprint # Enforce fresh DB build (do not rely on fallback caches). VORTEX_CI_DB_CACHE_FALLBACK: 'no' + # Refresh the database dumps in the fetch sources that are dump stores. + VORTEX_PUSH_DB_PROCEED: 1 # Do not build the Drupal front-end. VORTEX_FRONTEND_BUILD_SKIP: 1 diff --git a/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_acquia/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_acquia/.github/workflows/build-test-deploy.yml index c4971de6bd..a4f3384e34 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_acquia/.github/workflows/build-test-deploy.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_acquia/.github/workflows/build-test-deploy.yml @@ -1,4 +1,4 @@ -@@ -269,6 +269,9 @@ +@@ -271,6 +271,9 @@ VORTEX_FETCH_DB_SEMAPHORE=/tmp/fetch-db-fresh ./vendor/bin/vortex-fetch-db echo "db_hash=${{ hashFiles('.data') }}" >> "$GITHUB_ENV" timeout-minutes: 30 diff --git a/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_container_registry/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_container_registry/.github/workflows/build-test-deploy.yml index 5a2acf73b9..3086561937 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_container_registry/.github/workflows/build-test-deploy.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_container_registry/.github/workflows/build-test-deploy.yml @@ -6,10 +6,10 @@ + echo "VORTEX_DB_IMAGE_BASE=drevops/db-data-mysql-8.4:__VERSION__" + # Deploy container image. + echo "VORTEX_EXPORT_DB_CONTAINER_REGISTRY_PUSH_PROCEED=1" + # Refresh the database dumps in the fetch sources that are dump stores. + echo "VORTEX_PUSH_DB_PROCEED=1" } >> "$GITHUB_ENV" - - - name: Create cache keys files for database caching -@@ -274,6 +278,15 @@ +@@ -276,6 +280,15 @@ run: | if [ ! -f /tmp/fetch-db-fresh ]; then echo "==> Database fetch semaphore file is missing. DB export will not proceed."; exit 0; fi ./vendor/bin/vortex-refresh-db diff --git a/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_ftp/.env b/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_ftp/.env index 74fe89268f..8d266c9412 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_ftp/.env +++ b/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_ftp/.env @@ -1,4 +1,4 @@ -@@ -180,13 +180,19 @@ +@@ -180,13 +180,32 @@ VORTEX_DB_FILE=db.sql # Database fetch source. @@ -6,11 +6,8 @@ +VORTEX_FETCH_DB_SOURCE=ftp -# Database dump file sourced from a URL. --# --# HTTP Basic Authentication credentials should be embedded into the value. --VORTEX_FETCH_DB_URL= +# Database dump file sourced from FTP. - ++ +# Database dump FTP host. +VORTEX_FETCH_DB_FTP_HOST= + @@ -19,11 +16,26 @@ + +# Database dump FTP file name. +VORTEX_FETCH_DB_FTP_FILE=db.sql ++ ++# Upstream source of the database dump file, such as url, acquia or lagoon. + # +-# HTTP Basic Authentication credentials should be embedded into the value. +-VORTEX_FETCH_DB_URL= ++# Uncomment to make the database fetch source a dump store: the nightly CI ++# job fetches the dump from this source, sanitizes it and pushes it to the ++# fetch source as the latest dump. The fetch script of this source reads its ++# own variables, such as VORTEX_FETCH_DB_URL for url. Add them, and keep ++# secrets in CI. ++# VORTEX_REFRESH_DB_SOURCE=url + ++# Also push a copy of each dump named with a UTC timestamp, such as ++# db_20261111-180000.sql, to keep a history of the pushed dumps. ++# VORTEX_PUSH_DB_TIMESTAMPED_COPY=1 + # Environment to fetch the database from. # # Applies to hosting environments. -@@ -254,17 +260,3 @@ +@@ -254,17 +273,3 @@ # with optional names in the format "email|name". # Example: "to1@example.com|Jane Doe, to2@example.com|John Doe" VORTEX_NOTIFY_EMAIL_RECIPIENTS="webmaster@star-wars.com|Webmaster" diff --git a/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_ftp/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_ftp/.github/workflows/build-test-deploy.yml new file mode 100644 index 0000000000..99ebadce06 --- /dev/null +++ b/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_ftp/.github/workflows/build-test-deploy.yml @@ -0,0 +1,25 @@ +@@ -191,6 +191,8 @@ + VORTEX_CI_DB_CACHE_FALLBACK: "yes" + # Which branch to use as a source of DB caches. + VORTEX_CI_DB_CACHE_BRANCH: "develop" ++ VORTEX_FETCH_DB_FTP_USER: ${{ secrets.VORTEX_FETCH_DB_FTP_USER }} ++ VORTEX_FETCH_DB_FTP_PASS: ${{ secrets.VORTEX_FETCH_DB_FTP_PASS }} + + steps: + # Frees disk space by removing preinstalled toolchains unused by this project. Steps run in a container, so the host is reached via the Docker socket. +@@ -276,6 +278,15 @@ + run: | + if [ ! -f /tmp/fetch-db-fresh ]; then echo "==> Database fetch semaphore file is missing. DB export will not proceed."; exit 0; fi + ./vendor/bin/vortex-refresh-db ++ timeout-minutes: 30 ++ ++ # Push the freshly-exported sanitized database dump to the database fetch ++ # source. No-op unless a fresh DB was fetched (the fetch semaphore) and the ++ # push was requested (VORTEX_PUSH_DB_PROCEED, in the script). ++ - name: Push DB ++ run: | ++ if [ ! -f /tmp/fetch-db-fresh ]; then echo "==> No freshly fetched database. DB push will not proceed."; exit 0; fi ++ ./vendor/bin/vortex-push-db + timeout-minutes: 30 + + # Save cache per default branch and the timestamp. diff --git a/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_s3/.env b/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_s3/.env index 5d9d274041..aa442ec9c3 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_s3/.env +++ b/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_s3/.env @@ -1,4 +1,4 @@ -@@ -180,13 +180,16 @@ +@@ -180,13 +180,29 @@ VORTEX_DB_FILE=db.sql # Database fetch source. @@ -6,21 +6,33 @@ +VORTEX_FETCH_DB_SOURCE=s3 -# Database dump file sourced from a URL. --# --# HTTP Basic Authentication credentials should be embedded into the value. --VORTEX_FETCH_DB_URL= +# Database dump file sourced from S3. - ++ +# AWS S3 bucket name for database fetch. +VORTEX_FETCH_DB_S3_BUCKET= + +# AWS S3 region. +VORTEX_FETCH_DB_S3_REGION=ap-southeast-2 ++ ++# Upstream source of the database dump file, such as url, acquia or lagoon. + # +-# HTTP Basic Authentication credentials should be embedded into the value. +-VORTEX_FETCH_DB_URL= ++# Uncomment to make the database fetch source a dump store: the nightly CI ++# job fetches the dump from this source, sanitizes it and pushes it to the ++# fetch source as the latest dump. The fetch script of this source reads its ++# own variables, such as VORTEX_FETCH_DB_URL for url. Add them, and keep ++# secrets in CI. ++# VORTEX_REFRESH_DB_SOURCE=url + ++# Also push a copy of each dump named with a UTC timestamp, such as ++# db_20261111-180000.sql, to keep a history of the pushed dumps. ++# VORTEX_PUSH_DB_TIMESTAMPED_COPY=1 + # Environment to fetch the database from. # # Applies to hosting environments. -@@ -254,17 +257,3 @@ +@@ -254,17 +270,3 @@ # with optional names in the format "email|name". # Example: "to1@example.com|Jane Doe, to2@example.com|John Doe" VORTEX_NOTIFY_EMAIL_RECIPIENTS="webmaster@star-wars.com|Webmaster" diff --git a/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_s3/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_s3/.github/workflows/build-test-deploy.yml new file mode 100644 index 0000000000..b95ee1f3c8 --- /dev/null +++ b/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_s3/.github/workflows/build-test-deploy.yml @@ -0,0 +1,25 @@ +@@ -191,6 +191,8 @@ + VORTEX_CI_DB_CACHE_FALLBACK: "yes" + # Which branch to use as a source of DB caches. + VORTEX_CI_DB_CACHE_BRANCH: "develop" ++ VORTEX_FETCH_DB_S3_ACCESS_KEY: ${{ secrets.VORTEX_FETCH_DB_S3_ACCESS_KEY }} ++ VORTEX_FETCH_DB_S3_SECRET_KEY: ${{ secrets.VORTEX_FETCH_DB_S3_SECRET_KEY }} + + steps: + # Frees disk space by removing preinstalled toolchains unused by this project. Steps run in a container, so the host is reached via the Docker socket. +@@ -276,6 +278,15 @@ + run: | + if [ ! -f /tmp/fetch-db-fresh ]; then echo "==> Database fetch semaphore file is missing. DB export will not proceed."; exit 0; fi + ./vendor/bin/vortex-refresh-db ++ timeout-minutes: 30 ++ ++ # Push the freshly-exported sanitized database dump to the database fetch ++ # source. No-op unless a fresh DB was fetched (the fetch semaphore) and the ++ # push was requested (VORTEX_PUSH_DB_PROCEED, in the script). ++ - name: Push DB ++ run: | ++ if [ ! -f /tmp/fetch-db-fresh ]; then echo "==> No freshly fetched database. DB push will not proceed."; exit 0; fi ++ ./vendor/bin/vortex-push-db + timeout-minutes: 30 + + # Save cache per default branch and the timestamp. diff --git a/.vortex/installer/tests/Fixtures/handler_process/deploy_types_all_circleci/.circleci/config.yml b/.vortex/installer/tests/Fixtures/handler_process/deploy_types_all_circleci/.circleci/config.yml index f56bdf626c..b59dde79d2 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/deploy_types_all_circleci/.circleci/config.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/deploy_types_all_circleci/.circleci/config.yml @@ -307,6 +307,8 @@ jobs: VORTEX_DEPLOY_SSH_FINGERPRINT: *deploy_ssh_fingerprint # Enforce fresh DB build (do not rely on fallback caches). VORTEX_CI_DB_CACHE_FALLBACK: 'no' + # Refresh the database dumps in the fetch sources that are dump stores. + VORTEX_PUSH_DB_PROCEED: 1 # Do not build the Drupal front-end. VORTEX_FRONTEND_BUILD_SKIP: 1 diff --git a/.vortex/installer/tests/Fixtures/handler_process/deploy_types_none_circleci/.circleci/config.yml b/.vortex/installer/tests/Fixtures/handler_process/deploy_types_none_circleci/.circleci/config.yml index 70ccf44837..6b2526e489 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/deploy_types_none_circleci/.circleci/config.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/deploy_types_none_circleci/.circleci/config.yml @@ -307,6 +307,8 @@ jobs: VORTEX_DEPLOY_SSH_FINGERPRINT: *deploy_ssh_fingerprint # Enforce fresh DB build (do not rely on fallback caches). VORTEX_CI_DB_CACHE_FALLBACK: 'no' + # Refresh the database dumps in the fetch sources that are dump stores. + VORTEX_PUSH_DB_PROCEED: 1 # Do not build the Drupal front-end. VORTEX_FRONTEND_BUILD_SKIP: 1 diff --git a/.vortex/installer/tests/Fixtures/handler_process/deploy_types_none_gha/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/deploy_types_none_gha/.github/workflows/build-test-deploy.yml index 24407e1e7d..d0b2146bf0 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/deploy_types_none_gha/.github/workflows/build-test-deploy.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/deploy_types_none_gha/.github/workflows/build-test-deploy.yml @@ -1,4 +1,4 @@ -@@ -573,104 +573,3 @@ +@@ -575,104 +575,3 @@ timeout-minutes: 120 # Cancel the action after 120 minutes, regardless of whether a connection has been established. with: detached: true diff --git a/.vortex/installer/tests/Fixtures/handler_process/deps_updates_provider_ci_circleci/.circleci/config.yml b/.vortex/installer/tests/Fixtures/handler_process/deps_updates_provider_ci_circleci/.circleci/config.yml index f56bdf626c..b59dde79d2 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/deps_updates_provider_ci_circleci/.circleci/config.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/deps_updates_provider_ci_circleci/.circleci/config.yml @@ -307,6 +307,8 @@ jobs: VORTEX_DEPLOY_SSH_FINGERPRINT: *deploy_ssh_fingerprint # Enforce fresh DB build (do not rely on fallback caches). VORTEX_CI_DB_CACHE_FALLBACK: 'no' + # Refresh the database dumps in the fetch sources that are dump stores. + VORTEX_PUSH_DB_PROCEED: 1 # Do not build the Drupal front-end. VORTEX_FRONTEND_BUILD_SKIP: 1 diff --git a/.vortex/installer/tests/Fixtures/handler_process/hosting_acquia/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/hosting_acquia/.github/workflows/build-test-deploy.yml index c4971de6bd..a4f3384e34 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/hosting_acquia/.github/workflows/build-test-deploy.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/hosting_acquia/.github/workflows/build-test-deploy.yml @@ -1,4 +1,4 @@ -@@ -269,6 +269,9 @@ +@@ -271,6 +271,9 @@ VORTEX_FETCH_DB_SEMAPHORE=/tmp/fetch-db-fresh ./vendor/bin/vortex-fetch-db echo "db_hash=${{ hashFiles('.data') }}" >> "$GITHUB_ENV" timeout-minutes: 30 diff --git a/.vortex/installer/tests/Fixtures/handler_process/hosting_project_name___acquia/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/hosting_project_name___acquia/.github/workflows/build-test-deploy.yml index c4971de6bd..a4f3384e34 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/hosting_project_name___acquia/.github/workflows/build-test-deploy.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/hosting_project_name___acquia/.github/workflows/build-test-deploy.yml @@ -1,4 +1,4 @@ -@@ -269,6 +269,9 @@ +@@ -271,6 +271,9 @@ VORTEX_FETCH_DB_SEMAPHORE=/tmp/fetch-db-fresh ./vendor/bin/vortex-fetch-db echo "db_hash=${{ hashFiles('.data') }}" >> "$GITHUB_ENV" timeout-minutes: 30 diff --git a/.vortex/installer/tests/Fixtures/handler_process/migration_disabled_circleci/.circleci/config.yml b/.vortex/installer/tests/Fixtures/handler_process/migration_disabled_circleci/.circleci/config.yml index f56bdf626c..b59dde79d2 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/migration_disabled_circleci/.circleci/config.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/migration_disabled_circleci/.circleci/config.yml @@ -307,6 +307,8 @@ jobs: VORTEX_DEPLOY_SSH_FINGERPRINT: *deploy_ssh_fingerprint # Enforce fresh DB build (do not rely on fallback caches). VORTEX_CI_DB_CACHE_FALLBACK: 'no' + # Refresh the database dumps in the fetch sources that are dump stores. + VORTEX_PUSH_DB_PROCEED: 1 # Do not build the Drupal front-end. VORTEX_FRONTEND_BUILD_SKIP: 1 diff --git a/.vortex/installer/tests/Fixtures/handler_process/migration_enabled/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/migration_enabled/.github/workflows/build-test-deploy.yml index 4292b4f549..460b6bc35f 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/migration_enabled/.github/workflows/build-test-deploy.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/migration_enabled/.github/workflows/build-test-deploy.yml @@ -1,4 +1,4 @@ -@@ -270,6 +270,9 @@ +@@ -272,6 +272,9 @@ echo "db_hash=${{ hashFiles('.data') }}" >> "$GITHUB_ENV" timeout-minutes: 30 @@ -8,7 +8,7 @@ - name: Export DB run: | if [ ! -f /tmp/fetch-db-fresh ]; then echo "==> Database fetch semaphore file is missing. DB export will not proceed."; exit 0; fi -@@ -431,6 +434,11 @@ +@@ -433,6 +436,11 @@ docker compose exec cli mkdir -p .data docker compose --progress quiet cp -L .data/db.sql cli:/app/.data/db.sql rm -f .data/db.sql diff --git a/.vortex/installer/tests/Fixtures/handler_process/migration_enabled_circleci/.circleci/config.yml b/.vortex/installer/tests/Fixtures/handler_process/migration_enabled_circleci/.circleci/config.yml index 69a8a20830..657e41c70f 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/migration_enabled_circleci/.circleci/config.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/migration_enabled_circleci/.circleci/config.yml @@ -312,6 +312,8 @@ jobs: VORTEX_DEPLOY_SSH_FINGERPRINT: *deploy_ssh_fingerprint # Enforce fresh DB build (do not rely on fallback caches). VORTEX_CI_DB_CACHE_FALLBACK: 'no' + # Refresh the database dumps in the fetch sources that are dump stores. + VORTEX_PUSH_DB_PROCEED: 1 # Do not build the Drupal front-end. VORTEX_FRONTEND_BUILD_SKIP: 1 diff --git a/.vortex/installer/tests/Fixtures/handler_process/migration_enabled_lagoon/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/migration_enabled_lagoon/.github/workflows/build-test-deploy.yml index 4292b4f549..460b6bc35f 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/migration_enabled_lagoon/.github/workflows/build-test-deploy.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/migration_enabled_lagoon/.github/workflows/build-test-deploy.yml @@ -1,4 +1,4 @@ -@@ -270,6 +270,9 @@ +@@ -272,6 +272,9 @@ echo "db_hash=${{ hashFiles('.data') }}" >> "$GITHUB_ENV" timeout-minutes: 30 @@ -8,7 +8,7 @@ - name: Export DB run: | if [ ! -f /tmp/fetch-db-fresh ]; then echo "==> Database fetch semaphore file is missing. DB export will not proceed."; exit 0; fi -@@ -431,6 +434,11 @@ +@@ -433,6 +436,11 @@ docker compose exec cli mkdir -p .data docker compose --progress quiet cp -L .data/db.sql cli:/app/.data/db.sql rm -f .data/db.sql diff --git a/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_acquia/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_acquia/.github/workflows/build-test-deploy.yml index 4292b4f549..460b6bc35f 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_acquia/.github/workflows/build-test-deploy.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_acquia/.github/workflows/build-test-deploy.yml @@ -1,4 +1,4 @@ -@@ -270,6 +270,9 @@ +@@ -272,6 +272,9 @@ echo "db_hash=${{ hashFiles('.data') }}" >> "$GITHUB_ENV" timeout-minutes: 30 @@ -8,7 +8,7 @@ - name: Export DB run: | if [ ! -f /tmp/fetch-db-fresh ]; then echo "==> Database fetch semaphore file is missing. DB export will not proceed."; exit 0; fi -@@ -431,6 +434,11 @@ +@@ -433,6 +436,11 @@ docker compose exec cli mkdir -p .data docker compose --progress quiet cp -L .data/db.sql cli:/app/.data/db.sql rm -f .data/db.sql diff --git a/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_container_registry/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_container_registry/.github/workflows/build-test-deploy.yml index 975da88adb..e819850e04 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_container_registry/.github/workflows/build-test-deploy.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_container_registry/.github/workflows/build-test-deploy.yml @@ -6,10 +6,10 @@ + echo "VORTEX_DB_IMAGE_BASE=drevops/db-data-mysql-8.4:__VERSION__" + # Deploy container image. + echo "VORTEX_EXPORT_DB_CONTAINER_REGISTRY_PUSH_PROCEED=1" + # Refresh the database dumps in the fetch sources that are dump stores. + echo "VORTEX_PUSH_DB_PROCEED=1" } >> "$GITHUB_ENV" - - - name: Create cache keys files for database caching -@@ -270,9 +274,13 @@ +@@ -272,9 +276,13 @@ echo "db_hash=${{ hashFiles('.data') }}" >> "$GITHUB_ENV" timeout-minutes: 30 @@ -23,7 +23,7 @@ ./vendor/bin/vortex-refresh-db timeout-minutes: 30 -@@ -431,6 +439,11 @@ +@@ -433,6 +441,11 @@ docker compose exec cli mkdir -p .data docker compose --progress quiet cp -L .data/db.sql cli:/app/.data/db.sql rm -f .data/db.sql diff --git a/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_ftp/.env b/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_ftp/.env index 7f2e72048c..ae7d4bb2fe 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_ftp/.env +++ b/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_ftp/.env @@ -1,4 +1,4 @@ -@@ -195,6 +195,35 @@ +@@ -195,6 +195,42 @@ VORTEX_FETCH_DB_ENVIRONMENT=prod ################################################################################ @@ -25,6 +25,13 @@ +# Second database dump FTP file name. +VORTEX_FETCH_DB2_FTP_FILE=db2.sql + ++# Upstream source of the second database dump file, such as url or lagoon. ++# ++# Uncomment to make the second database fetch source a dump store: the ++# nightly CI job fetches the dump from this source and pushes it, as fetched, ++# to the fetch source, next to the primary dump. ++# VORTEX_REFRESH_DB2_SOURCE=url ++ +# Environment to fetch the second database from. +# +# Applies to hosting environments. diff --git a/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_ftp/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_ftp/.github/workflows/build-test-deploy.yml index 4292b4f549..bf29fc7cbc 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_ftp/.github/workflows/build-test-deploy.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_ftp/.github/workflows/build-test-deploy.yml @@ -1,4 +1,13 @@ -@@ -270,6 +270,9 @@ +@@ -191,6 +191,8 @@ + VORTEX_CI_DB_CACHE_FALLBACK: "yes" + # Which branch to use as a source of DB caches. + VORTEX_CI_DB_CACHE_BRANCH: "develop" ++ VORTEX_FETCH_DB2_FTP_USER: ${{ secrets.VORTEX_FETCH_DB2_FTP_USER }} ++ VORTEX_FETCH_DB2_FTP_PASS: ${{ secrets.VORTEX_FETCH_DB2_FTP_PASS }} + + steps: + # Frees disk space by removing preinstalled toolchains unused by this project. Steps run in a container, so the host is reached via the Docker socket. +@@ -272,6 +274,9 @@ echo "db_hash=${{ hashFiles('.data') }}" >> "$GITHUB_ENV" timeout-minutes: 30 @@ -8,7 +17,22 @@ - name: Export DB run: | if [ ! -f /tmp/fetch-db-fresh ]; then echo "==> Database fetch semaphore file is missing. DB export will not proceed."; exit 0; fi -@@ -431,6 +434,11 @@ +@@ -278,6 +283,14 @@ + ./vendor/bin/vortex-refresh-db + timeout-minutes: 30 + ++ # Push the freshly-fetched migration database dump to its fetch source, ++ # next to the primary dump. No-op unless the push was requested. ++ - name: Push migration DB ++ run: | ++ if [ ! -f /tmp/fetch-db-fresh ]; then echo "==> No freshly fetched database. DB push will not proceed."; exit 0; fi ++ VORTEX_DB_INDEX=2 ./vendor/bin/vortex-push-db ++ timeout-minutes: 30 ++ + # Save cache per default branch and the timestamp. + # The cache will not be saved if it already exists. + # Note that the cache fallback flag is enabled for this case in order +@@ -433,6 +446,11 @@ docker compose exec cli mkdir -p .data docker compose --progress quiet cp -L .data/db.sql cli:/app/.data/db.sql rm -f .data/db.sql diff --git a/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_lagoon/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_lagoon/.github/workflows/build-test-deploy.yml index 4292b4f549..460b6bc35f 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_lagoon/.github/workflows/build-test-deploy.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_lagoon/.github/workflows/build-test-deploy.yml @@ -1,4 +1,4 @@ -@@ -270,6 +270,9 @@ +@@ -272,6 +272,9 @@ echo "db_hash=${{ hashFiles('.data') }}" >> "$GITHUB_ENV" timeout-minutes: 30 @@ -8,7 +8,7 @@ - name: Export DB run: | if [ ! -f /tmp/fetch-db-fresh ]; then echo "==> Database fetch semaphore file is missing. DB export will not proceed."; exit 0; fi -@@ -431,6 +434,11 @@ +@@ -433,6 +436,11 @@ docker compose exec cli mkdir -p .data docker compose --progress quiet cp -L .data/db.sql cli:/app/.data/db.sql rm -f .data/db.sql diff --git a/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_s3/.env b/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_s3/.env index cddcb5610a..b1aec4e0f5 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_s3/.env +++ b/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_s3/.env @@ -1,4 +1,4 @@ -@@ -195,6 +195,32 @@ +@@ -195,6 +195,39 @@ VORTEX_FETCH_DB_ENVIRONMENT=prod ################################################################################ @@ -22,6 +22,13 @@ +# AWS S3 region. +VORTEX_FETCH_DB2_S3_REGION=ap-southeast-2 + ++# Upstream source of the second database dump file, such as url or lagoon. ++# ++# Uncomment to make the second database fetch source a dump store: the ++# nightly CI job fetches the dump from this source and pushes it, as fetched, ++# to the fetch source, next to the primary dump. ++# VORTEX_REFRESH_DB2_SOURCE=url ++ +# Environment to fetch the second database from. +# +# Applies to hosting environments. diff --git a/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_s3/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_s3/.github/workflows/build-test-deploy.yml index 4292b4f549..283b4f1f72 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_s3/.github/workflows/build-test-deploy.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_s3/.github/workflows/build-test-deploy.yml @@ -1,4 +1,13 @@ -@@ -270,6 +270,9 @@ +@@ -191,6 +191,8 @@ + VORTEX_CI_DB_CACHE_FALLBACK: "yes" + # Which branch to use as a source of DB caches. + VORTEX_CI_DB_CACHE_BRANCH: "develop" ++ VORTEX_FETCH_DB2_S3_ACCESS_KEY: ${{ secrets.VORTEX_FETCH_DB2_S3_ACCESS_KEY }} ++ VORTEX_FETCH_DB2_S3_SECRET_KEY: ${{ secrets.VORTEX_FETCH_DB2_S3_SECRET_KEY }} + + steps: + # Frees disk space by removing preinstalled toolchains unused by this project. Steps run in a container, so the host is reached via the Docker socket. +@@ -272,6 +274,9 @@ echo "db_hash=${{ hashFiles('.data') }}" >> "$GITHUB_ENV" timeout-minutes: 30 @@ -8,7 +17,22 @@ - name: Export DB run: | if [ ! -f /tmp/fetch-db-fresh ]; then echo "==> Database fetch semaphore file is missing. DB export will not proceed."; exit 0; fi -@@ -431,6 +434,11 @@ +@@ -278,6 +283,14 @@ + ./vendor/bin/vortex-refresh-db + timeout-minutes: 30 + ++ # Push the freshly-fetched migration database dump to its fetch source, ++ # next to the primary dump. No-op unless the push was requested. ++ - name: Push migration DB ++ run: | ++ if [ ! -f /tmp/fetch-db-fresh ]; then echo "==> No freshly fetched database. DB push will not proceed."; exit 0; fi ++ VORTEX_DB_INDEX=2 ./vendor/bin/vortex-push-db ++ timeout-minutes: 30 ++ + # Save cache per default branch and the timestamp. + # The cache will not be saved if it already exists. + # Note that the cache fallback flag is enabled for this case in order +@@ -433,6 +446,11 @@ docker compose exec cli mkdir -p .data docker compose --progress quiet cp -L .data/db.sql cli:/app/.data/db.sql rm -f .data/db.sql diff --git a/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_url/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_url/.github/workflows/build-test-deploy.yml index 4292b4f549..460b6bc35f 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_url/.github/workflows/build-test-deploy.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_url/.github/workflows/build-test-deploy.yml @@ -1,4 +1,4 @@ -@@ -270,6 +270,9 @@ +@@ -272,6 +272,9 @@ echo "db_hash=${{ hashFiles('.data') }}" >> "$GITHUB_ENV" timeout-minutes: 30 @@ -8,7 +8,7 @@ - name: Export DB run: | if [ ! -f /tmp/fetch-db-fresh ]; then echo "==> Database fetch semaphore file is missing. DB export will not proceed."; exit 0; fi -@@ -431,6 +434,11 @@ +@@ -433,6 +436,11 @@ docker compose exec cli mkdir -p .data docker compose --progress quiet cp -L .data/db.sql cli:/app/.data/db.sql rm -f .data/db.sql diff --git a/.vortex/installer/tests/Fixtures/handler_process/modules_no_dev_modules_storybook/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/modules_no_dev_modules_storybook/.github/workflows/build-test-deploy.yml index c0b7f62c2c..ab7610ac1b 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/modules_no_dev_modules_storybook/.github/workflows/build-test-deploy.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/modules_no_dev_modules_storybook/.github/workflows/build-test-deploy.yml @@ -1,4 +1,4 @@ -@@ -312,7 +312,6 @@ +@@ -314,7 +314,6 @@ VORTEX_CI_RUNNER_TOTAL: ${{ strategy.job-total }} VORTEX_CI_IS_JEST_RUNNER: ${{ matrix.instance == 0 || strategy.job-total == 1 }} VORTEX_CI_IS_PHPUNIT_RUNNER: ${{ matrix.instance == 0 || strategy.job-total == 1 }} @@ -6,7 +6,7 @@ # Runs on every instance, using the `p` profile. VORTEX_CI_IS_BEHAT_RUNNER: true # Subtracted from the instance index to derive the profile number. Raise -@@ -499,18 +498,6 @@ +@@ -501,18 +500,6 @@ fi env: VORTEX_CI_CODE_COVERAGE_THRESHOLD: ${{ vars.VORTEX_CI_CODE_COVERAGE_THRESHOLD || '90' }} diff --git a/.vortex/installer/tests/Fixtures/handler_process/modules_no_devel_sdc_devel/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/modules_no_devel_sdc_devel/.github/workflows/build-test-deploy.yml index c0b7f62c2c..ab7610ac1b 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/modules_no_devel_sdc_devel/.github/workflows/build-test-deploy.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/modules_no_devel_sdc_devel/.github/workflows/build-test-deploy.yml @@ -1,4 +1,4 @@ -@@ -312,7 +312,6 @@ +@@ -314,7 +314,6 @@ VORTEX_CI_RUNNER_TOTAL: ${{ strategy.job-total }} VORTEX_CI_IS_JEST_RUNNER: ${{ matrix.instance == 0 || strategy.job-total == 1 }} VORTEX_CI_IS_PHPUNIT_RUNNER: ${{ matrix.instance == 0 || strategy.job-total == 1 }} @@ -6,7 +6,7 @@ # Runs on every instance, using the `p` profile. VORTEX_CI_IS_BEHAT_RUNNER: true # Subtracted from the instance index to derive the profile number. Raise -@@ -499,18 +498,6 @@ +@@ -501,18 +500,6 @@ fi env: VORTEX_CI_CODE_COVERAGE_THRESHOLD: ${{ vars.VORTEX_CI_CODE_COVERAGE_THRESHOLD || '90' }} diff --git a/.vortex/installer/tests/Fixtures/handler_process/modules_no_devel_sdc_devel_generated_content/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/modules_no_devel_sdc_devel_generated_content/.github/workflows/build-test-deploy.yml index c0b7f62c2c..ab7610ac1b 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/modules_no_devel_sdc_devel_generated_content/.github/workflows/build-test-deploy.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/modules_no_devel_sdc_devel_generated_content/.github/workflows/build-test-deploy.yml @@ -1,4 +1,4 @@ -@@ -312,7 +312,6 @@ +@@ -314,7 +314,6 @@ VORTEX_CI_RUNNER_TOTAL: ${{ strategy.job-total }} VORTEX_CI_IS_JEST_RUNNER: ${{ matrix.instance == 0 || strategy.job-total == 1 }} VORTEX_CI_IS_PHPUNIT_RUNNER: ${{ matrix.instance == 0 || strategy.job-total == 1 }} @@ -6,7 +6,7 @@ # Runs on every instance, using the `p` profile. VORTEX_CI_IS_BEHAT_RUNNER: true # Subtracted from the instance index to derive the profile number. Raise -@@ -499,18 +498,6 @@ +@@ -501,18 +500,6 @@ fi env: VORTEX_CI_CODE_COVERAGE_THRESHOLD: ${{ vars.VORTEX_CI_CODE_COVERAGE_THRESHOLD || '90' }} diff --git a/.vortex/installer/tests/Fixtures/handler_process/modules_no_devel_sdc_devel_generated_content_testmode/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/modules_no_devel_sdc_devel_generated_content_testmode/.github/workflows/build-test-deploy.yml index c0b7f62c2c..ab7610ac1b 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/modules_no_devel_sdc_devel_generated_content_testmode/.github/workflows/build-test-deploy.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/modules_no_devel_sdc_devel_generated_content_testmode/.github/workflows/build-test-deploy.yml @@ -1,4 +1,4 @@ -@@ -312,7 +312,6 @@ +@@ -314,7 +314,6 @@ VORTEX_CI_RUNNER_TOTAL: ${{ strategy.job-total }} VORTEX_CI_IS_JEST_RUNNER: ${{ matrix.instance == 0 || strategy.job-total == 1 }} VORTEX_CI_IS_PHPUNIT_RUNNER: ${{ matrix.instance == 0 || strategy.job-total == 1 }} @@ -6,7 +6,7 @@ # Runs on every instance, using the `p` profile. VORTEX_CI_IS_BEHAT_RUNNER: true # Subtracted from the instance index to derive the profile number. Raise -@@ -499,18 +498,6 @@ +@@ -501,18 +500,6 @@ fi env: VORTEX_CI_CODE_COVERAGE_THRESHOLD: ${{ vars.VORTEX_CI_CODE_COVERAGE_THRESHOLD || '90' }} diff --git a/.vortex/installer/tests/Fixtures/handler_process/modules_no_devel_sdc_devel_generated_content_testmode_reroute_email/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/modules_no_devel_sdc_devel_generated_content_testmode_reroute_email/.github/workflows/build-test-deploy.yml index c0b7f62c2c..ab7610ac1b 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/modules_no_devel_sdc_devel_generated_content_testmode_reroute_email/.github/workflows/build-test-deploy.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/modules_no_devel_sdc_devel_generated_content_testmode_reroute_email/.github/workflows/build-test-deploy.yml @@ -1,4 +1,4 @@ -@@ -312,7 +312,6 @@ +@@ -314,7 +314,6 @@ VORTEX_CI_RUNNER_TOTAL: ${{ strategy.job-total }} VORTEX_CI_IS_JEST_RUNNER: ${{ matrix.instance == 0 || strategy.job-total == 1 }} VORTEX_CI_IS_PHPUNIT_RUNNER: ${{ matrix.instance == 0 || strategy.job-total == 1 }} @@ -6,7 +6,7 @@ # Runs on every instance, using the `p` profile. VORTEX_CI_IS_BEHAT_RUNNER: true # Subtracted from the instance index to derive the profile number. Raise -@@ -499,18 +498,6 @@ +@@ -501,18 +500,6 @@ fi env: VORTEX_CI_CODE_COVERAGE_THRESHOLD: ${{ vars.VORTEX_CI_CODE_COVERAGE_THRESHOLD || '90' }} diff --git a/.vortex/installer/tests/Fixtures/handler_process/modules_no_sdc_devel/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/modules_no_sdc_devel/.github/workflows/build-test-deploy.yml index c0b7f62c2c..ab7610ac1b 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/modules_no_sdc_devel/.github/workflows/build-test-deploy.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/modules_no_sdc_devel/.github/workflows/build-test-deploy.yml @@ -1,4 +1,4 @@ -@@ -312,7 +312,6 @@ +@@ -314,7 +314,6 @@ VORTEX_CI_RUNNER_TOTAL: ${{ strategy.job-total }} VORTEX_CI_IS_JEST_RUNNER: ${{ matrix.instance == 0 || strategy.job-total == 1 }} VORTEX_CI_IS_PHPUNIT_RUNNER: ${{ matrix.instance == 0 || strategy.job-total == 1 }} @@ -6,7 +6,7 @@ # Runs on every instance, using the `p` profile. VORTEX_CI_IS_BEHAT_RUNNER: true # Subtracted from the instance index to derive the profile number. Raise -@@ -499,18 +498,6 @@ +@@ -501,18 +500,6 @@ fi env: VORTEX_CI_CODE_COVERAGE_THRESHOLD: ${{ vars.VORTEX_CI_CODE_COVERAGE_THRESHOLD || '90' }} diff --git a/.vortex/installer/tests/Fixtures/handler_process/modules_none/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/modules_none/.github/workflows/build-test-deploy.yml index c0b7f62c2c..ab7610ac1b 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/modules_none/.github/workflows/build-test-deploy.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/modules_none/.github/workflows/build-test-deploy.yml @@ -1,4 +1,4 @@ -@@ -312,7 +312,6 @@ +@@ -314,7 +314,6 @@ VORTEX_CI_RUNNER_TOTAL: ${{ strategy.job-total }} VORTEX_CI_IS_JEST_RUNNER: ${{ matrix.instance == 0 || strategy.job-total == 1 }} VORTEX_CI_IS_PHPUNIT_RUNNER: ${{ matrix.instance == 0 || strategy.job-total == 1 }} @@ -6,7 +6,7 @@ # Runs on every instance, using the `p` profile. VORTEX_CI_IS_BEHAT_RUNNER: true # Subtracted from the instance index to derive the profile number. Raise -@@ -499,18 +498,6 @@ +@@ -501,18 +500,6 @@ fi env: VORTEX_CI_CODE_COVERAGE_THRESHOLD: ${{ vars.VORTEX_CI_CODE_COVERAGE_THRESHOLD || '90' }} diff --git a/.vortex/installer/tests/Fixtures/handler_process/provision_profile/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/provision_profile/.github/workflows/build-test-deploy.yml index 042d1c6b55..0dfba85dcf 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/provision_profile/.github/workflows/build-test-deploy.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/provision_profile/.github/workflows/build-test-deploy.yml @@ -15,7 +15,7 @@ container: # https://hub.docker.com/r/drevops/ci-runner -@@ -167,130 +164,8 @@ +@@ -167,132 +164,8 @@ run: docker compose exec -T cli bash -c "npm --prefix=\${WEBROOT}/themes/custom/\${DRUPAL_THEME} run lint" continue-on-error: ${{ vars.VORTEX_CI_NODEJS_LINT_IGNORE_FAILURE == '1' }} @@ -93,6 +93,8 @@ - { - echo "VORTEX_CI_DB_CACHE_FALLBACK=no" - echo "VORTEX_FRONTEND_BUILD_SKIP=1" +- # Refresh the database dumps in the fetch sources that are dump stores. +- echo "VORTEX_PUSH_DB_PROCEED=1" - } >> "$GITHUB_ENV" - - - name: Create cache keys files for database caching @@ -146,7 +148,7 @@ permissions: contents: read -@@ -333,14 +208,6 @@ +@@ -335,14 +208,6 @@ VORTEX_SSH_DISABLE_STRICT_HOST_KEY_CHECKING: "1" VORTEX_SSH_REMOVE_ALL_KEYS: "1" VORTEX_DEBUG: ${{ vars.VORTEX_DEBUG }} @@ -161,7 +163,7 @@ steps: # Frees disk space by removing preinstalled toolchains unused by this project. Steps run in a container, so the host is reached via the Docker socket. -@@ -379,29 +246,6 @@ +@@ -381,29 +246,6 @@ - name: Install Vortex tooling run: ./scripts/vortex-tooling.sh @@ -191,7 +193,7 @@ - name: Login to container registry run: ./vendor/bin/vortex-login-container-registry -@@ -577,7 +421,6 @@ +@@ -579,7 +421,6 @@ deploy: runs-on: ubuntu-latest needs: [build, lint] diff --git a/.vortex/installer/tests/Fixtures/handler_process/theme_claro/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/theme_claro/.github/workflows/build-test-deploy.yml index da2e5188f3..0cd1802313 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/theme_claro/.github/workflows/build-test-deploy.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/theme_claro/.github/workflows/build-test-deploy.yml @@ -10,7 +10,7 @@ database: runs-on: ubuntu-latest if: ${{ !inputs.deploy_target && (github.event_name == 'push' || !startsWith(github.head_ref, 'project/')) }} -@@ -312,7 +307,6 @@ +@@ -314,7 +309,6 @@ VORTEX_CI_RUNNER_TOTAL: ${{ strategy.job-total }} VORTEX_CI_IS_JEST_RUNNER: ${{ matrix.instance == 0 || strategy.job-total == 1 }} VORTEX_CI_IS_PHPUNIT_RUNNER: ${{ matrix.instance == 0 || strategy.job-total == 1 }} @@ -18,7 +18,7 @@ # Runs on every instance, using the `p` profile. VORTEX_CI_IS_BEHAT_RUNNER: true # Subtracted from the instance index to derive the profile number. Raise -@@ -499,18 +493,6 @@ +@@ -501,18 +495,6 @@ fi env: VORTEX_CI_CODE_COVERAGE_THRESHOLD: ${{ vars.VORTEX_CI_CODE_COVERAGE_THRESHOLD || '90' }} diff --git a/.vortex/installer/tests/Fixtures/handler_process/theme_olivero/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/theme_olivero/.github/workflows/build-test-deploy.yml index da2e5188f3..0cd1802313 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/theme_olivero/.github/workflows/build-test-deploy.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/theme_olivero/.github/workflows/build-test-deploy.yml @@ -10,7 +10,7 @@ database: runs-on: ubuntu-latest if: ${{ !inputs.deploy_target && (github.event_name == 'push' || !startsWith(github.head_ref, 'project/')) }} -@@ -312,7 +307,6 @@ +@@ -314,7 +309,6 @@ VORTEX_CI_RUNNER_TOTAL: ${{ strategy.job-total }} VORTEX_CI_IS_JEST_RUNNER: ${{ matrix.instance == 0 || strategy.job-total == 1 }} VORTEX_CI_IS_PHPUNIT_RUNNER: ${{ matrix.instance == 0 || strategy.job-total == 1 }} @@ -18,7 +18,7 @@ # Runs on every instance, using the `p` profile. VORTEX_CI_IS_BEHAT_RUNNER: true # Subtracted from the instance index to derive the profile number. Raise -@@ -499,18 +493,6 @@ +@@ -501,18 +495,6 @@ fi env: VORTEX_CI_CODE_COVERAGE_THRESHOLD: ${{ vars.VORTEX_CI_CODE_COVERAGE_THRESHOLD || '90' }} diff --git a/.vortex/installer/tests/Fixtures/handler_process/theme_stark/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/theme_stark/.github/workflows/build-test-deploy.yml index da2e5188f3..0cd1802313 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/theme_stark/.github/workflows/build-test-deploy.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/theme_stark/.github/workflows/build-test-deploy.yml @@ -10,7 +10,7 @@ database: runs-on: ubuntu-latest if: ${{ !inputs.deploy_target && (github.event_name == 'push' || !startsWith(github.head_ref, 'project/')) }} -@@ -312,7 +307,6 @@ +@@ -314,7 +309,6 @@ VORTEX_CI_RUNNER_TOTAL: ${{ strategy.job-total }} VORTEX_CI_IS_JEST_RUNNER: ${{ matrix.instance == 0 || strategy.job-total == 1 }} VORTEX_CI_IS_PHPUNIT_RUNNER: ${{ matrix.instance == 0 || strategy.job-total == 1 }} @@ -18,7 +18,7 @@ # Runs on every instance, using the `p` profile. VORTEX_CI_IS_BEHAT_RUNNER: true # Subtracted from the instance index to derive the profile number. Raise -@@ -499,18 +493,6 @@ +@@ -501,18 +495,6 @@ fi env: VORTEX_CI_CODE_COVERAGE_THRESHOLD: ${{ vars.VORTEX_CI_CODE_COVERAGE_THRESHOLD || '90' }} diff --git a/.vortex/installer/tests/Fixtures/handler_process/timezone_circleci/.circleci/config.yml b/.vortex/installer/tests/Fixtures/handler_process/timezone_circleci/.circleci/config.yml index f56bdf626c..b59dde79d2 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/timezone_circleci/.circleci/config.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/timezone_circleci/.circleci/config.yml @@ -307,6 +307,8 @@ jobs: VORTEX_DEPLOY_SSH_FINGERPRINT: *deploy_ssh_fingerprint # Enforce fresh DB build (do not rely on fallback caches). VORTEX_CI_DB_CACHE_FALLBACK: 'no' + # Refresh the database dumps in the fetch sources that are dump stores. + VORTEX_PUSH_DB_PROCEED: 1 # Do not build the Drupal front-end. VORTEX_FRONTEND_BUILD_SKIP: 1 diff --git a/.vortex/installer/tests/Fixtures/handler_process/tools_groups_no_be_lint_circleci/.circleci/config.yml b/.vortex/installer/tests/Fixtures/handler_process/tools_groups_no_be_lint_circleci/.circleci/config.yml index 962162638f..56e766a652 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/tools_groups_no_be_lint_circleci/.circleci/config.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/tools_groups_no_be_lint_circleci/.circleci/config.yml @@ -295,6 +295,8 @@ jobs: VORTEX_DEPLOY_SSH_FINGERPRINT: *deploy_ssh_fingerprint # Enforce fresh DB build (do not rely on fallback caches). VORTEX_CI_DB_CACHE_FALLBACK: 'no' + # Refresh the database dumps in the fetch sources that are dump stores. + VORTEX_PUSH_DB_PROCEED: 1 # Do not build the Drupal front-end. VORTEX_FRONTEND_BUILD_SKIP: 1 diff --git a/.vortex/installer/tests/Fixtures/handler_process/tools_groups_no_be_tests/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/tools_groups_no_be_tests/.github/workflows/build-test-deploy.yml index 2f4b0324b6..e5104a12f6 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/tools_groups_no_be_tests/.github/workflows/build-test-deploy.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/tools_groups_no_be_tests/.github/workflows/build-test-deploy.yml @@ -9,7 +9,7 @@ - name: Lint module code with NodeJS linters run: docker compose exec -T cli bash -c "npm run lint" continue-on-error: ${{ vars.VORTEX_CI_NODEJS_LINT_IGNORE_FAILURE == '1' }} -@@ -311,14 +307,7 @@ +@@ -313,14 +309,7 @@ VORTEX_CI_RUNNER_INDEX: ${{ strategy.job-index }} VORTEX_CI_RUNNER_TOTAL: ${{ strategy.job-total }} VORTEX_CI_IS_JEST_RUNNER: ${{ matrix.instance == 0 || strategy.job-total == 1 }} @@ -24,7 +24,7 @@ container: # https://hub.docker.com/r/drevops/ci-runner -@@ -440,66 +429,6 @@ +@@ -442,66 +431,6 @@ run: docker compose exec -T cli bash -c "npm test" continue-on-error: ${{ vars.VORTEX_CI_JEST_IGNORE_FAILURE == '1' }} @@ -91,7 +91,7 @@ - name: Validate Single Directory Components if: ${{ env.VORTEX_CI_IS_SDC_DEVEL_RUNNER == 'true' }} run: | -@@ -512,19 +441,6 @@ +@@ -514,19 +443,6 @@ fi continue-on-error: ${{ vars.VORTEX_CI_SDC_DEVEL_IGNORE_FAILURE == '1' }} @@ -111,7 +111,7 @@ - name: Process test logs and artifacts if: always() run: | -@@ -541,16 +457,6 @@ +@@ -543,16 +459,6 @@ path: .logs include-hidden-files: true if-no-files-found: error diff --git a/.vortex/installer/tests/Fixtures/handler_process/tools_groups_no_be_tests_circleci/.circleci/config.yml b/.vortex/installer/tests/Fixtures/handler_process/tools_groups_no_be_tests_circleci/.circleci/config.yml index 0b83288c75..4125037c62 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/tools_groups_no_be_tests_circleci/.circleci/config.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/tools_groups_no_be_tests_circleci/.circleci/config.yml @@ -303,6 +303,8 @@ jobs: VORTEX_DEPLOY_SSH_FINGERPRINT: *deploy_ssh_fingerprint # Enforce fresh DB build (do not rely on fallback caches). VORTEX_CI_DB_CACHE_FALLBACK: 'no' + # Refresh the database dumps in the fetch sources that are dump stores. + VORTEX_PUSH_DB_PROCEED: 1 # Do not build the Drupal front-end. VORTEX_FRONTEND_BUILD_SKIP: 1 diff --git a/.vortex/installer/tests/Fixtures/handler_process/tools_groups_no_fe_lint/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/tools_groups_no_fe_lint/.github/workflows/build-test-deploy.yml index 452179b6d7..4d1ca474b4 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/tools_groups_no_fe_lint/.github/workflows/build-test-deploy.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/tools_groups_no_fe_lint/.github/workflows/build-test-deploy.yml @@ -22,7 +22,7 @@ database: runs-on: ubuntu-latest if: ${{ !inputs.deploy_target && (github.event_name == 'push' || !startsWith(github.head_ref, 'project/')) }} -@@ -310,7 +300,6 @@ +@@ -312,7 +302,6 @@ # https://www.vortextemplate.com/docs/continuous-integration#test-parallelism VORTEX_CI_RUNNER_INDEX: ${{ strategy.job-index }} VORTEX_CI_RUNNER_TOTAL: ${{ strategy.job-total }} @@ -30,7 +30,7 @@ VORTEX_CI_IS_PHPUNIT_RUNNER: ${{ matrix.instance == 0 || strategy.job-total == 1 }} VORTEX_CI_IS_SDC_DEVEL_RUNNER: ${{ matrix.instance == 0 || strategy.job-total == 1 }} # Runs on every instance, using the `p` profile. -@@ -421,7 +410,6 @@ +@@ -423,7 +412,6 @@ docker compose exec $(env | cut -f1 -d= | sed 's/^/-e /') -T cli bash -c " \ if [ -n \"\${PACKAGE_TOKEN:-}\" ]; then composer config --global --auth github-oauth.github.com \"\${PACKAGE_TOKEN}\"; fi && \ COMPOSER_MEMORY_LIMIT=-1 composer --ansi install --prefer-dist" @@ -38,7 +38,7 @@ - name: Provision site run: | -@@ -434,11 +422,6 @@ +@@ -436,11 +424,6 @@ fi docker compose exec $(env | cut -f1 -d= | sed 's/^/-e /') -T cli ./vendor/bin/vortex-provision timeout-minutes: 30 diff --git a/.vortex/installer/tests/Fixtures/handler_process/tools_groups_no_fe_lint_circleci/.circleci/config.yml b/.vortex/installer/tests/Fixtures/handler_process/tools_groups_no_fe_lint_circleci/.circleci/config.yml index 0d5dd2cae9..d21bef58a2 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/tools_groups_no_fe_lint_circleci/.circleci/config.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/tools_groups_no_fe_lint_circleci/.circleci/config.yml @@ -286,6 +286,8 @@ jobs: VORTEX_DEPLOY_SSH_FINGERPRINT: *deploy_ssh_fingerprint # Enforce fresh DB build (do not rely on fallback caches). VORTEX_CI_DB_CACHE_FALLBACK: 'no' + # Refresh the database dumps in the fetch sources that are dump stores. + VORTEX_PUSH_DB_PROCEED: 1 # Do not build the Drupal front-end. VORTEX_FRONTEND_BUILD_SKIP: 1 diff --git a/.vortex/installer/tests/Fixtures/handler_process/tools_groups_no_fe_lint_no_theme/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/tools_groups_no_fe_lint_no_theme/.github/workflows/build-test-deploy.yml index 8ffe668f17..3245313f2f 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/tools_groups_no_fe_lint_no_theme/.github/workflows/build-test-deploy.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/tools_groups_no_fe_lint_no_theme/.github/workflows/build-test-deploy.yml @@ -22,7 +22,7 @@ database: runs-on: ubuntu-latest if: ${{ !inputs.deploy_target && (github.event_name == 'push' || !startsWith(github.head_ref, 'project/')) }} -@@ -310,9 +300,7 @@ +@@ -312,9 +302,7 @@ # https://www.vortextemplate.com/docs/continuous-integration#test-parallelism VORTEX_CI_RUNNER_INDEX: ${{ strategy.job-index }} VORTEX_CI_RUNNER_TOTAL: ${{ strategy.job-total }} @@ -32,7 +32,7 @@ # Runs on every instance, using the `p` profile. VORTEX_CI_IS_BEHAT_RUNNER: true # Subtracted from the instance index to derive the profile number. Raise -@@ -421,7 +409,6 @@ +@@ -423,7 +411,6 @@ docker compose exec $(env | cut -f1 -d= | sed 's/^/-e /') -T cli bash -c " \ if [ -n \"\${PACKAGE_TOKEN:-}\" ]; then composer config --global --auth github-oauth.github.com \"\${PACKAGE_TOKEN}\"; fi && \ COMPOSER_MEMORY_LIMIT=-1 composer --ansi install --prefer-dist" @@ -40,7 +40,7 @@ - name: Provision site run: | -@@ -435,11 +422,6 @@ +@@ -437,11 +424,6 @@ docker compose exec $(env | cut -f1 -d= | sed 's/^/-e /') -T cli ./vendor/bin/vortex-provision timeout-minutes: 30 @@ -52,7 +52,7 @@ - name: Test with PHPUnit if: ${{ env.VORTEX_CI_IS_PHPUNIT_RUNNER == 'true' }} run: docker compose exec -T cli vendor/bin/phpunit -@@ -499,18 +481,6 @@ +@@ -501,18 +483,6 @@ fi env: VORTEX_CI_CODE_COVERAGE_THRESHOLD: ${{ vars.VORTEX_CI_CODE_COVERAGE_THRESHOLD || '90' }} diff --git a/.vortex/installer/tests/Fixtures/handler_process/tools_groups_no_fe_lint_no_theme_circleci/.circleci/config.yml b/.vortex/installer/tests/Fixtures/handler_process/tools_groups_no_fe_lint_no_theme_circleci/.circleci/config.yml index 87d8397465..8e015f51d7 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/tools_groups_no_fe_lint_no_theme_circleci/.circleci/config.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/tools_groups_no_fe_lint_no_theme_circleci/.circleci/config.yml @@ -277,6 +277,8 @@ jobs: VORTEX_DEPLOY_SSH_FINGERPRINT: *deploy_ssh_fingerprint # Enforce fresh DB build (do not rely on fallback caches). VORTEX_CI_DB_CACHE_FALLBACK: 'no' + # Refresh the database dumps in the fetch sources that are dump stores. + VORTEX_PUSH_DB_PROCEED: 1 # Do not build the Drupal front-end. VORTEX_FRONTEND_BUILD_SKIP: 1 diff --git a/.vortex/installer/tests/Fixtures/handler_process/tools_no_behat/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/tools_no_behat/.github/workflows/build-test-deploy.yml index 0b32b5b70d..fd50aad45b 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/tools_no_behat/.github/workflows/build-test-deploy.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/tools_no_behat/.github/workflows/build-test-deploy.yml @@ -9,7 +9,7 @@ - name: Lint module code with NodeJS linters run: docker compose exec -T cli bash -c "npm run lint" continue-on-error: ${{ vars.VORTEX_CI_NODEJS_LINT_IGNORE_FAILURE == '1' }} -@@ -313,12 +309,6 @@ +@@ -315,12 +311,6 @@ VORTEX_CI_IS_JEST_RUNNER: ${{ matrix.instance == 0 || strategy.job-total == 1 }} VORTEX_CI_IS_PHPUNIT_RUNNER: ${{ matrix.instance == 0 || strategy.job-total == 1 }} VORTEX_CI_IS_SDC_DEVEL_RUNNER: ${{ matrix.instance == 0 || strategy.job-total == 1 }} @@ -22,7 +22,7 @@ container: # https://hub.docker.com/r/drevops/ci-runner -@@ -511,19 +501,6 @@ +@@ -513,19 +503,6 @@ exit 1 fi continue-on-error: ${{ vars.VORTEX_CI_SDC_DEVEL_IGNORE_FAILURE == '1' }} diff --git a/.vortex/installer/tests/Fixtures/handler_process/tools_no_behat_circleci/.circleci/config.yml b/.vortex/installer/tests/Fixtures/handler_process/tools_no_behat_circleci/.circleci/config.yml index 194ac3218b..4735766d37 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/tools_no_behat_circleci/.circleci/config.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/tools_no_behat_circleci/.circleci/config.yml @@ -303,6 +303,8 @@ jobs: VORTEX_DEPLOY_SSH_FINGERPRINT: *deploy_ssh_fingerprint # Enforce fresh DB build (do not rely on fallback caches). VORTEX_CI_DB_CACHE_FALLBACK: 'no' + # Refresh the database dumps in the fetch sources that are dump stores. + VORTEX_PUSH_DB_PROCEED: 1 # Do not build the Drupal front-end. VORTEX_FRONTEND_BUILD_SKIP: 1 diff --git a/.vortex/installer/tests/Fixtures/handler_process/tools_no_dclint_circleci/.circleci/config.yml b/.vortex/installer/tests/Fixtures/handler_process/tools_no_dclint_circleci/.circleci/config.yml index 3cf240711d..54dc4477e3 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/tools_no_dclint_circleci/.circleci/config.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/tools_no_dclint_circleci/.circleci/config.yml @@ -298,6 +298,8 @@ jobs: VORTEX_DEPLOY_SSH_FINGERPRINT: *deploy_ssh_fingerprint # Enforce fresh DB build (do not rely on fallback caches). VORTEX_CI_DB_CACHE_FALLBACK: 'no' + # Refresh the database dumps in the fetch sources that are dump stores. + VORTEX_PUSH_DB_PROCEED: 1 # Do not build the Drupal front-end. VORTEX_FRONTEND_BUILD_SKIP: 1 diff --git a/.vortex/installer/tests/Fixtures/handler_process/tools_no_dclint_migration/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/tools_no_dclint_migration/.github/workflows/build-test-deploy.yml index f4e7a01167..03afc8216b 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/tools_no_dclint_migration/.github/workflows/build-test-deploy.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/tools_no_dclint_migration/.github/workflows/build-test-deploy.yml @@ -9,7 +9,7 @@ - name: Build stack run: docker compose up --no-deps --detach cli -@@ -270,6 +266,9 @@ +@@ -272,6 +268,9 @@ echo "db_hash=${{ hashFiles('.data') }}" >> "$GITHUB_ENV" timeout-minutes: 30 @@ -19,7 +19,7 @@ - name: Export DB run: | if [ ! -f /tmp/fetch-db-fresh ]; then echo "==> Database fetch semaphore file is missing. DB export will not proceed."; exit 0; fi -@@ -431,6 +430,11 @@ +@@ -433,6 +432,11 @@ docker compose exec cli mkdir -p .data docker compose --progress quiet cp -L .data/db.sql cli:/app/.data/db.sql rm -f .data/db.sql diff --git a/.vortex/installer/tests/Fixtures/handler_process/tools_no_docker_linters_circleci/.circleci/config.yml b/.vortex/installer/tests/Fixtures/handler_process/tools_no_docker_linters_circleci/.circleci/config.yml index 64bcd3e9ab..fffe322f5d 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/tools_no_docker_linters_circleci/.circleci/config.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/tools_no_docker_linters_circleci/.circleci/config.yml @@ -291,6 +291,8 @@ jobs: VORTEX_DEPLOY_SSH_FINGERPRINT: *deploy_ssh_fingerprint # Enforce fresh DB build (do not rely on fallback caches). VORTEX_CI_DB_CACHE_FALLBACK: 'no' + # Refresh the database dumps in the fetch sources that are dump stores. + VORTEX_PUSH_DB_PROCEED: 1 # Do not build the Drupal front-end. VORTEX_FRONTEND_BUILD_SKIP: 1 diff --git a/.vortex/installer/tests/Fixtures/handler_process/tools_no_eslint_circleci/.circleci/config.yml b/.vortex/installer/tests/Fixtures/handler_process/tools_no_eslint_circleci/.circleci/config.yml index f56bdf626c..b59dde79d2 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/tools_no_eslint_circleci/.circleci/config.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/tools_no_eslint_circleci/.circleci/config.yml @@ -307,6 +307,8 @@ jobs: VORTEX_DEPLOY_SSH_FINGERPRINT: *deploy_ssh_fingerprint # Enforce fresh DB build (do not rely on fallback caches). VORTEX_CI_DB_CACHE_FALLBACK: 'no' + # Refresh the database dumps in the fetch sources that are dump stores. + VORTEX_PUSH_DB_PROCEED: 1 # Do not build the Drupal front-end. VORTEX_FRONTEND_BUILD_SKIP: 1 diff --git a/.vortex/installer/tests/Fixtures/handler_process/tools_no_eslint_no_theme/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/tools_no_eslint_no_theme/.github/workflows/build-test-deploy.yml index da2e5188f3..0cd1802313 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/tools_no_eslint_no_theme/.github/workflows/build-test-deploy.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/tools_no_eslint_no_theme/.github/workflows/build-test-deploy.yml @@ -10,7 +10,7 @@ database: runs-on: ubuntu-latest if: ${{ !inputs.deploy_target && (github.event_name == 'push' || !startsWith(github.head_ref, 'project/')) }} -@@ -312,7 +307,6 @@ +@@ -314,7 +309,6 @@ VORTEX_CI_RUNNER_TOTAL: ${{ strategy.job-total }} VORTEX_CI_IS_JEST_RUNNER: ${{ matrix.instance == 0 || strategy.job-total == 1 }} VORTEX_CI_IS_PHPUNIT_RUNNER: ${{ matrix.instance == 0 || strategy.job-total == 1 }} @@ -18,7 +18,7 @@ # Runs on every instance, using the `p` profile. VORTEX_CI_IS_BEHAT_RUNNER: true # Subtracted from the instance index to derive the profile number. Raise -@@ -499,18 +493,6 @@ +@@ -501,18 +495,6 @@ fi env: VORTEX_CI_CODE_COVERAGE_THRESHOLD: ${{ vars.VORTEX_CI_CODE_COVERAGE_THRESHOLD || '90' }} diff --git a/.vortex/installer/tests/Fixtures/handler_process/tools_no_hadolint_circleci/.circleci/config.yml b/.vortex/installer/tests/Fixtures/handler_process/tools_no_hadolint_circleci/.circleci/config.yml index 37281e8ff5..121b9f5477 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/tools_no_hadolint_circleci/.circleci/config.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/tools_no_hadolint_circleci/.circleci/config.yml @@ -300,6 +300,8 @@ jobs: VORTEX_DEPLOY_SSH_FINGERPRINT: *deploy_ssh_fingerprint # Enforce fresh DB build (do not rely on fallback caches). VORTEX_CI_DB_CACHE_FALLBACK: 'no' + # Refresh the database dumps in the fetch sources that are dump stores. + VORTEX_PUSH_DB_PROCEED: 1 # Do not build the Drupal front-end. VORTEX_FRONTEND_BUILD_SKIP: 1 diff --git a/.vortex/installer/tests/Fixtures/handler_process/tools_no_jest/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/tools_no_jest/.github/workflows/build-test-deploy.yml index a4847a4654..2963f464ca 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/tools_no_jest/.github/workflows/build-test-deploy.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/tools_no_jest/.github/workflows/build-test-deploy.yml @@ -1,4 +1,4 @@ -@@ -310,7 +310,6 @@ +@@ -312,7 +312,6 @@ # https://www.vortextemplate.com/docs/continuous-integration#test-parallelism VORTEX_CI_RUNNER_INDEX: ${{ strategy.job-index }} VORTEX_CI_RUNNER_TOTAL: ${{ strategy.job-total }} @@ -6,7 +6,7 @@ VORTEX_CI_IS_PHPUNIT_RUNNER: ${{ matrix.instance == 0 || strategy.job-total == 1 }} VORTEX_CI_IS_SDC_DEVEL_RUNNER: ${{ matrix.instance == 0 || strategy.job-total == 1 }} # Runs on every instance, using the `p` profile. -@@ -421,7 +420,6 @@ +@@ -423,7 +422,6 @@ docker compose exec $(env | cut -f1 -d= | sed 's/^/-e /') -T cli bash -c " \ if [ -n \"\${PACKAGE_TOKEN:-}\" ]; then composer config --global --auth github-oauth.github.com \"\${PACKAGE_TOKEN}\"; fi && \ COMPOSER_MEMORY_LIMIT=-1 composer --ansi install --prefer-dist" @@ -14,7 +14,7 @@ - name: Provision site run: | -@@ -434,11 +432,6 @@ +@@ -436,11 +434,6 @@ fi docker compose exec $(env | cut -f1 -d= | sed 's/^/-e /') -T cli ./vendor/bin/vortex-provision timeout-minutes: 30 diff --git a/.vortex/installer/tests/Fixtures/handler_process/tools_no_jest_circleci/.circleci/config.yml b/.vortex/installer/tests/Fixtures/handler_process/tools_no_jest_circleci/.circleci/config.yml index 52fdedfed1..8cbd50d31b 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/tools_no_jest_circleci/.circleci/config.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/tools_no_jest_circleci/.circleci/config.yml @@ -307,6 +307,8 @@ jobs: VORTEX_DEPLOY_SSH_FINGERPRINT: *deploy_ssh_fingerprint # Enforce fresh DB build (do not rely on fallback caches). VORTEX_CI_DB_CACHE_FALLBACK: 'no' + # Refresh the database dumps in the fetch sources that are dump stores. + VORTEX_PUSH_DB_PROCEED: 1 # Do not build the Drupal front-end. VORTEX_FRONTEND_BUILD_SKIP: 1 diff --git a/.vortex/installer/tests/Fixtures/handler_process/tools_no_phpcs_circleci/.circleci/config.yml b/.vortex/installer/tests/Fixtures/handler_process/tools_no_phpcs_circleci/.circleci/config.yml index f2aacd0e81..3262246832 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/tools_no_phpcs_circleci/.circleci/config.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/tools_no_phpcs_circleci/.circleci/config.yml @@ -303,6 +303,8 @@ jobs: VORTEX_DEPLOY_SSH_FINGERPRINT: *deploy_ssh_fingerprint # Enforce fresh DB build (do not rely on fallback caches). VORTEX_CI_DB_CACHE_FALLBACK: 'no' + # Refresh the database dumps in the fetch sources that are dump stores. + VORTEX_PUSH_DB_PROCEED: 1 # Do not build the Drupal front-end. VORTEX_FRONTEND_BUILD_SKIP: 1 diff --git a/.vortex/installer/tests/Fixtures/handler_process/tools_no_phpstan_circleci/.circleci/config.yml b/.vortex/installer/tests/Fixtures/handler_process/tools_no_phpstan_circleci/.circleci/config.yml index bf65b2a0ba..8fc685f729 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/tools_no_phpstan_circleci/.circleci/config.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/tools_no_phpstan_circleci/.circleci/config.yml @@ -303,6 +303,8 @@ jobs: VORTEX_DEPLOY_SSH_FINGERPRINT: *deploy_ssh_fingerprint # Enforce fresh DB build (do not rely on fallback caches). VORTEX_CI_DB_CACHE_FALLBACK: 'no' + # Refresh the database dumps in the fetch sources that are dump stores. + VORTEX_PUSH_DB_PROCEED: 1 # Do not build the Drupal front-end. VORTEX_FRONTEND_BUILD_SKIP: 1 diff --git a/.vortex/installer/tests/Fixtures/handler_process/tools_no_phpunit/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/tools_no_phpunit/.github/workflows/build-test-deploy.yml index 83fae1f090..4bf951f78a 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/tools_no_phpunit/.github/workflows/build-test-deploy.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/tools_no_phpunit/.github/workflows/build-test-deploy.yml @@ -1,4 +1,4 @@ -@@ -311,7 +311,6 @@ +@@ -313,7 +313,6 @@ VORTEX_CI_RUNNER_INDEX: ${{ strategy.job-index }} VORTEX_CI_RUNNER_TOTAL: ${{ strategy.job-total }} VORTEX_CI_IS_JEST_RUNNER: ${{ matrix.instance == 0 || strategy.job-total == 1 }} @@ -6,7 +6,7 @@ VORTEX_CI_IS_SDC_DEVEL_RUNNER: ${{ matrix.instance == 0 || strategy.job-total == 1 }} # Runs on every instance, using the `p` profile. VORTEX_CI_IS_BEHAT_RUNNER: true -@@ -439,66 +438,6 @@ +@@ -441,66 +440,6 @@ if: ${{ env.VORTEX_CI_IS_JEST_RUNNER == 'true' }} run: docker compose exec -T cli bash -c "npm test" continue-on-error: ${{ vars.VORTEX_CI_JEST_IGNORE_FAILURE == '1' }} diff --git a/.vortex/installer/tests/Fixtures/handler_process/tools_no_phpunit_circleci/.circleci/config.yml b/.vortex/installer/tests/Fixtures/handler_process/tools_no_phpunit_circleci/.circleci/config.yml index affec2423a..05b7fbe608 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/tools_no_phpunit_circleci/.circleci/config.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/tools_no_phpunit_circleci/.circleci/config.yml @@ -307,6 +307,8 @@ jobs: VORTEX_DEPLOY_SSH_FINGERPRINT: *deploy_ssh_fingerprint # Enforce fresh DB build (do not rely on fallback caches). VORTEX_CI_DB_CACHE_FALLBACK: 'no' + # Refresh the database dumps in the fetch sources that are dump stores. + VORTEX_PUSH_DB_PROCEED: 1 # Do not build the Drupal front-end. VORTEX_FRONTEND_BUILD_SKIP: 1 diff --git a/.vortex/installer/tests/Fixtures/handler_process/tools_no_rector_circleci/.circleci/config.yml b/.vortex/installer/tests/Fixtures/handler_process/tools_no_rector_circleci/.circleci/config.yml index 65af74315e..dfdbda3a2a 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/tools_no_rector_circleci/.circleci/config.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/tools_no_rector_circleci/.circleci/config.yml @@ -303,6 +303,8 @@ jobs: VORTEX_DEPLOY_SSH_FINGERPRINT: *deploy_ssh_fingerprint # Enforce fresh DB build (do not rely on fallback caches). VORTEX_CI_DB_CACHE_FALLBACK: 'no' + # Refresh the database dumps in the fetch sources that are dump stores. + VORTEX_PUSH_DB_PROCEED: 1 # Do not build the Drupal front-end. VORTEX_FRONTEND_BUILD_SKIP: 1 diff --git a/.vortex/installer/tests/Fixtures/handler_process/tools_no_stylelint_circleci/.circleci/config.yml b/.vortex/installer/tests/Fixtures/handler_process/tools_no_stylelint_circleci/.circleci/config.yml index f56bdf626c..b59dde79d2 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/tools_no_stylelint_circleci/.circleci/config.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/tools_no_stylelint_circleci/.circleci/config.yml @@ -307,6 +307,8 @@ jobs: VORTEX_DEPLOY_SSH_FINGERPRINT: *deploy_ssh_fingerprint # Enforce fresh DB build (do not rely on fallback caches). VORTEX_CI_DB_CACHE_FALLBACK: 'no' + # Refresh the database dumps in the fetch sources that are dump stores. + VORTEX_PUSH_DB_PROCEED: 1 # Do not build the Drupal front-end. VORTEX_FRONTEND_BUILD_SKIP: 1 diff --git a/.vortex/installer/tests/Fixtures/handler_process/tools_no_stylelint_no_theme/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/tools_no_stylelint_no_theme/.github/workflows/build-test-deploy.yml index da2e5188f3..0cd1802313 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/tools_no_stylelint_no_theme/.github/workflows/build-test-deploy.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/tools_no_stylelint_no_theme/.github/workflows/build-test-deploy.yml @@ -10,7 +10,7 @@ database: runs-on: ubuntu-latest if: ${{ !inputs.deploy_target && (github.event_name == 'push' || !startsWith(github.head_ref, 'project/')) }} -@@ -312,7 +307,6 @@ +@@ -314,7 +309,6 @@ VORTEX_CI_RUNNER_TOTAL: ${{ strategy.job-total }} VORTEX_CI_IS_JEST_RUNNER: ${{ matrix.instance == 0 || strategy.job-total == 1 }} VORTEX_CI_IS_PHPUNIT_RUNNER: ${{ matrix.instance == 0 || strategy.job-total == 1 }} @@ -18,7 +18,7 @@ # Runs on every instance, using the `p` profile. VORTEX_CI_IS_BEHAT_RUNNER: true # Subtracted from the instance index to derive the profile number. Raise -@@ -499,18 +493,6 @@ +@@ -501,18 +495,6 @@ fi env: VORTEX_CI_CODE_COVERAGE_THRESHOLD: ${{ vars.VORTEX_CI_CODE_COVERAGE_THRESHOLD || '90' }} diff --git a/.vortex/installer/tests/Fixtures/handler_process/tools_no_twig_circleci/.circleci/config.yml b/.vortex/installer/tests/Fixtures/handler_process/tools_no_twig_circleci/.circleci/config.yml index 5001ef54a4..f2890ba107 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/tools_no_twig_circleci/.circleci/config.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/tools_no_twig_circleci/.circleci/config.yml @@ -303,6 +303,8 @@ jobs: VORTEX_DEPLOY_SSH_FINGERPRINT: *deploy_ssh_fingerprint # Enforce fresh DB build (do not rely on fallback caches). VORTEX_CI_DB_CACHE_FALLBACK: 'no' + # Refresh the database dumps in the fetch sources that are dump stores. + VORTEX_PUSH_DB_PROCEED: 1 # Do not build the Drupal front-end. VORTEX_FRONTEND_BUILD_SKIP: 1 diff --git a/.vortex/installer/tests/Fixtures/handler_process/tools_none/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/tools_none/.github/workflows/build-test-deploy.yml index ecbb0ad695..e3c358bb75 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/tools_none/.github/workflows/build-test-deploy.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/tools_none/.github/workflows/build-test-deploy.yml @@ -58,7 +58,7 @@ database: runs-on: ubuntu-latest if: ${{ !inputs.deploy_target && (github.event_name == 'push' || !startsWith(github.head_ref, 'project/')) }} -@@ -310,15 +269,7 @@ +@@ -312,15 +271,7 @@ # https://www.vortextemplate.com/docs/continuous-integration#test-parallelism VORTEX_CI_RUNNER_INDEX: ${{ strategy.job-index }} VORTEX_CI_RUNNER_TOTAL: ${{ strategy.job-total }} @@ -74,7 +74,7 @@ container: # https://hub.docker.com/r/drevops/ci-runner -@@ -421,7 +372,6 @@ +@@ -423,7 +374,6 @@ docker compose exec $(env | cut -f1 -d= | sed 's/^/-e /') -T cli bash -c " \ if [ -n \"\${PACKAGE_TOKEN:-}\" ]; then composer config --global --auth github-oauth.github.com \"\${PACKAGE_TOKEN}\"; fi && \ COMPOSER_MEMORY_LIMIT=-1 composer --ansi install --prefer-dist" @@ -82,7 +82,7 @@ - name: Provision site run: | -@@ -435,71 +385,6 @@ +@@ -437,71 +387,6 @@ docker compose exec $(env | cut -f1 -d= | sed 's/^/-e /') -T cli ./vendor/bin/vortex-provision timeout-minutes: 30 @@ -154,7 +154,7 @@ - name: Validate Single Directory Components if: ${{ env.VORTEX_CI_IS_SDC_DEVEL_RUNNER == 'true' }} run: | -@@ -512,19 +397,6 @@ +@@ -514,19 +399,6 @@ fi continue-on-error: ${{ vars.VORTEX_CI_SDC_DEVEL_IGNORE_FAILURE == '1' }} @@ -174,7 +174,7 @@ - name: Process test logs and artifacts if: always() run: | -@@ -541,16 +413,6 @@ +@@ -543,16 +415,6 @@ path: .logs include-hidden-files: true if-no-files-found: error diff --git a/.vortex/installer/tests/Functional/Prompts/Handlers/MigrationFetchSourceHandlerProcessTest.php b/.vortex/installer/tests/Functional/Prompts/Handlers/MigrationFetchSourceHandlerProcessTest.php index 0c63da7a4c..9c128f4a8a 100644 --- a/.vortex/installer/tests/Functional/Prompts/Handlers/MigrationFetchSourceHandlerProcessTest.php +++ b/.vortex/installer/tests/Functional/Prompts/Handlers/MigrationFetchSourceHandlerProcessTest.php @@ -27,6 +27,7 @@ public static function dataProviderHandlerProcess(): \Iterator { $test->assertFileNotContainsString(static::$sut . '/.github/workflows/build-test-deploy.yml', 'VORTEX_DB_INDEX=2 ./vendor/bin/vortex-refresh-db'); $test->assertFileNotContainsString(static::$sut . '/.github/workflows/build-test-deploy.yml', 'VORTEX_DB_IMAGE_BASE'); $test->assertFileNotContainsString(static::$sut . '/.github/workflows/build-test-deploy.yml', 'Deploy DB image'); + $test->assertFileNotContainsString(static::$sut . '/.github/workflows/build-test-deploy.yml', 'VORTEX_DB_INDEX=2 ./vendor/bin/vortex-push-db'); }), ]; yield 'migration_fetch_source_ftp' => [ @@ -39,6 +40,7 @@ public static function dataProviderHandlerProcess(): \Iterator { $test->assertFileContainsString(static::$sut . '/.env', 'VORTEX_FETCH_DB2_FTP_HOST'); $test->assertFileNotContainsString(static::$sut . '/.env', 'VORTEX_FETCH_DB2_URL='); $test->assertFileNotContainsString(static::$sut . '/.env', 'VORTEX_FETCH_DB2_S3_BUCKET'); + $test->assertFileContainsString(static::$sut . '/.github/workflows/build-test-deploy.yml', 'VORTEX_DB_INDEX=2 ./vendor/bin/vortex-push-db'); }), ]; yield 'migration_fetch_source_acquia' => [ @@ -75,6 +77,7 @@ public static function dataProviderHandlerProcess(): \Iterator { $test->assertFileContainsString(static::$sut . '/.env', 'VORTEX_FETCH_DB2_S3_BUCKET'); $test->assertFileNotContainsString(static::$sut . '/.env', 'VORTEX_FETCH_DB2_URL='); $test->assertFileNotContainsString(static::$sut . '/.env', 'VORTEX_FETCH_DB2_FTP_HOST'); + $test->assertFileContainsString(static::$sut . '/.github/workflows/build-test-deploy.yml', 'VORTEX_DB_INDEX=2 ./vendor/bin/vortex-push-db'); }), ]; yield 'migration_fetch_source_container_registry' => [ diff --git a/.vortex/tooling/composer.json b/.vortex/tooling/composer.json index 32d784ea24..7ed3f155a3 100644 --- a/.vortex/tooling/composer.json +++ b/.vortex/tooling/composer.json @@ -34,6 +34,7 @@ "src/vortex-logout", "src/vortex-notify", "src/vortex-provision", + "src/vortex-push-db", "src/vortex-push-db-image", "src/vortex-refresh-db", "src/vortex-reset", diff --git a/.vortex/tooling/playground/try-push-db-ftp.sh b/.vortex/tooling/playground/try-push-db-ftp.sh new file mode 100755 index 0000000000..1716701ae0 --- /dev/null +++ b/.vortex/tooling/playground/try-push-db-ftp.sh @@ -0,0 +1,71 @@ +#!/usr/bin/env bash +## +# Manual test script for FTP database push. +# +# Uploads a database dump file to an FTP server using push-db-ftp. Run it +# twice to check that the second run replaces 'db.sql'. Set +# VORTEX_PUSH_DB_TIMESTAMPED_COPY=1 to also upload a timestamped copy. +# +# Usage: +# export FTP_HOST="ftp.example.com" +# export FTP_PORT="21" +# export FTP_USER="your-user" +# export FTP_PASS="your-password" +# export FTP_FILE="path/to/db.sql" +# ./try-push-db-ftp.sh + +set -eu + +# Source secrets if available. +if [ -f "$HOME/.profile.secrets" ]; then + # shellcheck disable=SC1091 + . "$HOME/.profile.secrets" +fi + +# Get the directory of this script and navigate to project root. +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_ROOT="$(cd "${SCRIPT_DIR}/../../.." && pwd)" + +# FTP credentials and settings. +FTP_HOST="${FTP_HOST:-}" +FTP_PORT="${FTP_PORT:-21}" +FTP_USER="${FTP_USER:-}" +FTP_PASS="${FTP_PASS:-}" +FTP_FILE="${FTP_FILE:-db.sql}" + +# Validate required variables. +for name in FTP_HOST FTP_USER FTP_PASS; do + if [ -z "${!name}" ]; then + echo "Error: ${name} environment variable is required" + echo "Usage: export ${name}=\"...\" && $0" + exit 1 + fi +done + +cd "${PROJECT_ROOT}" || exit 1 + +# Create a test dump file when none exists. +db_dir="${VORTEX_DB_DIR:-./.data}" +db_file="${VORTEX_DB_FILE:-db.sql}" +if [ ! -f "${db_dir}/${db_file}" ]; then + mkdir -p "${db_dir}" + echo "-- Test dump created by try-push-db-ftp.sh on $(date -u)" >"${db_dir}/${db_file}" +fi + +echo "Testing FTP database push..." +echo "" +echo "FTP host: ${FTP_HOST}:${FTP_PORT}" +echo "FTP file: ${FTP_FILE}" +echo "Local file: ${db_dir}/${db_file}" +echo "" + +export VORTEX_PUSH_DB_FTP_HOST="${FTP_HOST}" +export VORTEX_PUSH_DB_FTP_PORT="${FTP_PORT}" +export VORTEX_PUSH_DB_FTP_USER="${FTP_USER}" +export VORTEX_PUSH_DB_FTP_PASS="${FTP_PASS}" +export VORTEX_PUSH_DB_FTP_FILE="${FTP_FILE}" + +./vendor/drevops/vortex-tooling/src/vortex-push-db-ftp + +echo "" +echo "Push complete!" diff --git a/.vortex/tooling/src/vortex-fetch-db b/.vortex/tooling/src/vortex-fetch-db index 1cc7d7acf7..b3f93c2f6e 100755 --- a/.vortex/tooling/src/vortex-fetch-db +++ b/.vortex/tooling/src/vortex-fetch-db @@ -55,6 +55,24 @@ info "Started database${_db_index:+ ${_db_index}} fetch." [ "${VORTEX_FETCH_DB_PROCEED}" != "1" ] && pass "Skipped database fetch as VORTEX_FETCH_DB_PROCEED is not set to 1." && exit 0 +# An 'ftp' or 's3' source with an upstream source in VORTEX_REFRESH_DB_SOURCE +# is a dump store: it receives the dump that the nightly refresh pushes, so +# with a push requested the dump comes from the upstream source. The store +# may hold no dump yet on the first push. +_v="VORTEX_REFRESH_DB${_db_index}_SOURCE" +refresh_source="${!_v:-}" +if [ "${VORTEX_PUSH_DB_PROCEED:-}" = "1" ] && [ -n "${refresh_source}" ]; then + case "${VORTEX_FETCH_DB_SOURCE}" in + ftp | s3) + case "${refresh_source}" in + "${VORTEX_FETCH_DB_SOURCE}" | container_registry | none) fail "The database dump file pushed to ${VORTEX_FETCH_DB_SOURCE} is fetched from an upstream source, but VORTEX_REFRESH_DB${_db_index}_SOURCE is \"${refresh_source}\". Set it to another source of the dump file: url, ftp, acquia, lagoon or s3." ;; + esac + note "Fetching from the ${refresh_source} source in VORTEX_REFRESH_DB${_db_index}_SOURCE as VORTEX_PUSH_DB_PROCEED is set to 1." + VORTEX_FETCH_DB_SOURCE="${refresh_source}" + ;; + esac +fi + # The fetch script is resolved before the existing dump check, so an # unsupported source fails even when a cached dump would be reused. case "${VORTEX_FETCH_DB_SOURCE}" in diff --git a/.vortex/tooling/src/vortex-push-db b/.vortex/tooling/src/vortex-push-db new file mode 100755 index 0000000000..9a36c9629f --- /dev/null +++ b/.vortex/tooling/src/vortex-push-db @@ -0,0 +1,73 @@ +#!/usr/bin/env bash +## +# Push database dump. +# +# This is a router script to call relevant scripts based on type. +# +# The database dump file is pushed to the storage the database is fetched +# from, set in VORTEX_FETCH_DB_SOURCE. Only an 'ftp' or 's3' source with an +# upstream source in VORTEX_REFRESH_DB_SOURCE accepts a push (a dump store); +# the push is skipped for other sources. +# +# With VORTEX_DB_INDEX set (e.g., "2" for the migration database), the +# indexed database pushes its own dump file with the indexed variables, so +# several databases share 1 storage. +# +# IMPORTANT! This script runs outside the container on the host system. +# +# shellcheck disable=SC1090,SC1091 + +t=$(mktemp) && export -p >"${t}" && set -a && . ./.env && if [ -f ./.env.local ]; then . ./.env.local; fi && set +a && . "${t}" && rm "${t}" && unset t + +set -eu +[ "${VORTEX_DEBUG-}" = "1" ] && set -x + +# Database index suffix. When set (e.g., "2"), all DB-related variable lookups +# use the indexed variant. +_db_index="${VORTEX_DB_INDEX:-}" + +# Proceed with push. +# +# Set to 1 in the nightly CI job only, so the storage is refreshed once a day +# and never from a feature branch build. +VORTEX_PUSH_DB_PROCEED="${VORTEX_PUSH_DB_PROCEED:-}" + +# Database fetch source, which is also the push destination. +_v="VORTEX_FETCH_DB${_db_index}_SOURCE" +VORTEX_FETCH_DB_SOURCE="${!_v:-url}" + +# Upstream source of the pushed database dump file. +_v="VORTEX_REFRESH_DB${_db_index}_SOURCE" +VORTEX_REFRESH_DB_SOURCE="${!_v:-}" + +# ------------------------------------------------------------------------------ + +# @formatter:off +info() { [ "${TERM:-}" != "dumb" ] && tput colors >/dev/null 2>&1 && printf "\033[36m[INFO] %s\033[0m\n" "${1}" || printf "[INFO] %s\n" "${1}"; } +note() { printf " %s\n" "${1}"; } +task() { _TASK_START=$(date +%s); [ "${TERM:-}" != "dumb" ] && tput colors >/dev/null 2>&1 && printf "\033[34m[TASK] %s\033[0m\n" "${1}" || printf "[TASK] %s\n" "${1}"; } +pass() { _d=""; [ -n "${_TASK_START:-}" ] && _d=" ($(($(date +%s) - _TASK_START))s)" && unset _TASK_START; [ "${TERM:-}" != "dumb" ] && tput colors >/dev/null 2>&1 && printf "\033[32m[ OK ] %s%s\033[0m\n" "${1}" "${_d}" || printf "[ OK ] %s%s\n" "${1}" "${_d}"; } +fail() { [ "${TERM:-}" != "dumb" ] && tput colors >/dev/null 2>&1 && printf "\033[31m[FAIL] %s\033[0m\n" "${1}" || printf "[FAIL] %s\n" "${1}"; exit "${2:-1}"; } +# @formatter:on + +info "Started database${_db_index:+ ${_db_index}} push." + +[ "${VORTEX_PUSH_DB_PROCEED}" != "1" ] && pass "Skipped database push as VORTEX_PUSH_DB_PROCEED is not set to 1." && exit 0 + +case "${VORTEX_FETCH_DB_SOURCE}" in + ftp) push_script="vortex-push-db-ftp" ;; + s3) push_script="vortex-push-db-s3" ;; + *) pass "Skipped database push as the \"${VORTEX_FETCH_DB_SOURCE}\" database source in VORTEX_FETCH_DB${_db_index}_SOURCE does not accept a push. Use one of: ftp, s3." && exit 0 ;; +esac + +# Without an upstream source, the fetch source is where the dump comes from, +# and a push would replace that dump with its own export. +[ -z "${VORTEX_REFRESH_DB_SOURCE}" ] && pass "Skipped database push as VORTEX_REFRESH_DB${_db_index}_SOURCE is not set, so the ${VORTEX_FETCH_DB_SOURCE} source is not a dump store." && exit 0 + +# The pushed dump leaves the hosting and CI, so it must be sanitized. Only the +# primary database is sanitized: an indexed database is pushed as fetched. +[ -z "${_db_index}" ] && [ "${VORTEX_PROVISION_SANITIZE_DB_SKIP:-}" = "1" ] && fail "Refused to push the database as VORTEX_PROVISION_SANITIZE_DB_SKIP is set to 1 and the database dump is not sanitized." + +env VORTEX_DB_INDEX="${_db_index}" "$(dirname "${BASH_SOURCE[0]}")/${push_script}" + +pass "Finished database${_db_index:+ ${_db_index}} push." diff --git a/.vortex/tooling/src/vortex-push-db-ftp b/.vortex/tooling/src/vortex-push-db-ftp new file mode 100755 index 0000000000..f2abb91e23 --- /dev/null +++ b/.vortex/tooling/src/vortex-push-db-ftp @@ -0,0 +1,161 @@ +#!/usr/bin/env bash +## +# Push DB dump to FTP. +# +# Uploads the database dump file under the remote file name, such as +# 'db.sql', which always holds the latest dump that the fetch downloads. With +# VORTEX_PUSH_DB_TIMESTAMPED_COPY set to 1, the dump is also uploaded under a +# name with a UTC timestamp, such as 'db_20261111-180000.sql', to keep a +# history. +# +# Each file is uploaded under a temporary name and renamed into place only +# after the upload succeeds, so a failed upload keeps the existing file. +# +# IMPORTANT! This script runs outside the container on the host system. +# +# shellcheck disable=SC1090,SC1091 + +t=$(mktemp) && export -p >"${t}" && set -a && . ./.env && if [ -f ./.env.local ]; then . ./.env.local; fi && set +a && . "${t}" && rm "${t}" && unset t + +set -eu + +# Database index suffix. When set (e.g., "2"), all DB-related variable lookups +# use the indexed variant, so each database pushes its own dump file. +_db_index="${VORTEX_DB_INDEX:-}" + +# The FTP values fall back to the values the database is fetched with, so a +# server used as the fetch source is configured once. + +# The FTP user. +_v="VORTEX_PUSH_DB${_db_index}_FTP_USER" +_vs="VORTEX_FETCH_DB${_db_index}_FTP_USER" +VORTEX_PUSH_DB_FTP_USER="${!_v:-${!_vs:-}}" + +# The FTP password. +_v="VORTEX_PUSH_DB${_db_index}_FTP_PASS" +_vs="VORTEX_FETCH_DB${_db_index}_FTP_PASS" +VORTEX_PUSH_DB_FTP_PASS="${!_v:-${!_vs:-}}" + +# The FTP host. +_v="VORTEX_PUSH_DB${_db_index}_FTP_HOST" +_vs="VORTEX_FETCH_DB${_db_index}_FTP_HOST" +VORTEX_PUSH_DB_FTP_HOST="${!_v:-${!_vs:-}}" + +# The FTP port. +_v="VORTEX_PUSH_DB${_db_index}_FTP_PORT" +_vs="VORTEX_FETCH_DB${_db_index}_FTP_PORT" +VORTEX_PUSH_DB_FTP_PORT="${!_v:-${!_vs:-21}}" + +# The remote file name, including any directories. +_v="VORTEX_PUSH_DB${_db_index}_FTP_FILE" +_vs="VORTEX_FETCH_DB${_db_index}_FTP_FILE" +VORTEX_PUSH_DB_FTP_FILE="${!_v:-${!_vs:-}}" + +# Directory with database dump file. +_v="VORTEX_PUSH_DB${_db_index}_FTP_DB_DIR" +_vs="VORTEX_FETCH_DB${_db_index}_DIR" +_vss="VORTEX_DB${_db_index}_DIR" +VORTEX_PUSH_DB_FTP_DB_DIR="${!_v:-${!_vs:-${!_vss:-${VORTEX_DB_DIR:-./.data}}}}" + +# Database dump file name. +# +# Defaults to the name the database is fetched and refreshed with. +_v="VORTEX_PUSH_DB${_db_index}_FTP_DB_FILE" +_vs="VORTEX_FETCH_DB${_db_index}_FILE" +_vss="VORTEX_DB${_db_index}_FILE" +VORTEX_PUSH_DB_FTP_DB_FILE="${!_v:-${!_vs:-${!_vss:-${VORTEX_DB_FILE:-db.sql}}}}" + +# Suffix for the names of the pushed database dump files. +# +# Goes before the extension, so 'db.sql' becomes 'db_gha.sql' with '_gha'. +# Use a different suffix in each place that pushes to the same storage. A +# fetch reads the suffixed name only when its remote file name includes it. +VORTEX_PUSH_DB_FILE_SUFFIX="${VORTEX_PUSH_DB_FILE_SUFFIX:-}" + +# Also push a copy of the database dump file named with a UTC timestamp. +# +# Set to 1 to keep a history of the pushed dumps. The fetch never reads the +# timestamped copies. +VORTEX_PUSH_DB_TIMESTAMPED_COPY="${VORTEX_PUSH_DB_TIMESTAMPED_COPY:-}" + +# ------------------------------------------------------------------------------ + +# @formatter:off +info() { [ "${TERM:-}" != "dumb" ] && tput colors >/dev/null 2>&1 && printf "\033[36m[INFO] %s\033[0m\n" "${1}" || printf "[INFO] %s\n" "${1}"; } +note() { printf " %s\n" "${1}"; } +task() { _TASK_START=$(date +%s); [ "${TERM:-}" != "dumb" ] && tput colors >/dev/null 2>&1 && printf "\033[34m[TASK] %s\033[0m\n" "${1}" || printf "[TASK] %s\n" "${1}"; } +pass() { _d=""; [ -n "${_TASK_START:-}" ] && _d=" ($(($(date +%s) - _TASK_START))s)" && unset _TASK_START; [ "${TERM:-}" != "dumb" ] && tput colors >/dev/null 2>&1 && printf "\033[32m[ OK ] %s%s\033[0m\n" "${1}" "${_d}" || printf "[ OK ] %s%s\n" "${1}" "${_d}"; } +fail() { [ "${TERM:-}" != "dumb" ] && tput colors >/dev/null 2>&1 && printf "\033[31m[FAIL] %s\033[0m\n" "${1}" || printf "[FAIL] %s\n" "${1}"; exit "${2:-1}"; } +# @formatter:on + +# Print a value quoted for a curl config file. +curl_config_quote() { + local value="${1//\\/\\\\}" + printf '"%s"' "${value//\"/\\\"}" +} + +# Upload the database dump file to a name in the remote directory. +# +# The credentials go to curl on stdin, so they never appear in the process +# list. The commands after the upload run in the remote directory: the +# existing file is removed and the upload takes its name. +ftp_upload() { + local name="${1}" + local upload_name="${name}.uploading" + + echo "user = $(curl_config_quote "${VORTEX_PUSH_DB_FTP_USER}:${VORTEX_PUSH_DB_FTP_PASS}")" | curl --silent --show-error --config - --ftp-create-dirs --upload-file "${local_file}" \ + --quote "-*DELE ${name}" \ + --quote "-RNFR ${upload_name}" \ + --quote "-RNTO ${name}" \ + "ftp://${VORTEX_PUSH_DB_FTP_HOST}:${VORTEX_PUSH_DB_FTP_PORT}/${remote_dir}${upload_name}" +} + +# shellcheck disable=SC2043 +for cmd in curl; do command -v "${cmd}" >/dev/null || fail "Command ${cmd} is not available."; done + +[ -z "${VORTEX_PUSH_DB_FTP_USER}" ] && fail "Missing required value for VORTEX_PUSH_DB${_db_index}_FTP_USER or VORTEX_FETCH_DB${_db_index}_FTP_USER." +[ -z "${VORTEX_PUSH_DB_FTP_PASS}" ] && fail "Missing required value for VORTEX_PUSH_DB${_db_index}_FTP_PASS or VORTEX_FETCH_DB${_db_index}_FTP_PASS." +[ -z "${VORTEX_PUSH_DB_FTP_HOST}" ] && fail "Missing required value for VORTEX_PUSH_DB${_db_index}_FTP_HOST or VORTEX_FETCH_DB${_db_index}_FTP_HOST." +[ -z "${VORTEX_PUSH_DB_FTP_FILE}" ] && fail "Missing required value for VORTEX_PUSH_DB${_db_index}_FTP_FILE or VORTEX_FETCH_DB${_db_index}_FTP_FILE." + +local_file="${VORTEX_PUSH_DB_FTP_DB_DIR}/${VORTEX_PUSH_DB_FTP_DB_FILE}" +[ ! -f "${local_file}" ] && fail "Database dump file ${local_file} does not exist." + +remote_dir="" +remote_name="${VORTEX_PUSH_DB_FTP_FILE#/}" +if [[ ${remote_name} == */* ]]; then + remote_dir="${remote_name%/*}/" + remote_name="${remote_name##*/}" +fi + +base="${remote_name}" +ext="" +if [[ ${remote_name} == *.* ]]; then + base="${remote_name%.*}" + ext=".${remote_name##*.}" +fi +current_name="${base}${VORTEX_PUSH_DB_FILE_SUFFIX}${ext}" +timestamped_name="${base}${VORTEX_PUSH_DB_FILE_SUFFIX}_$(date -u +%Y%m%d-%H%M%S)${ext}" + +info "Started database${_db_index:+ ${_db_index}} dump push to FTP." + +note "Local file: ${local_file}" +note "Remote file: ${remote_dir}${current_name}" +note "FTP host: ${VORTEX_PUSH_DB_FTP_HOST}:${VORTEX_PUSH_DB_FTP_PORT}" + +task "Uploading the database dump file." +ftp_upload "${current_name}" || fail "Unable to push the database dump file to FTP." +pass "Uploaded the database dump file." + +if [ "${VORTEX_PUSH_DB_TIMESTAMPED_COPY}" = "1" ]; then + # FTP has no copy command, so the copy is a second upload. + task "Uploading the timestamped copy ${remote_dir}${timestamped_name}." + ftp_upload "${timestamped_name}" || fail "Unable to push the timestamped copy of the database dump file to FTP." + pass "Uploaded the timestamped copy ${remote_dir}${timestamped_name}." +fi + +# Enable xtrace only after the request so the credentials are never written +# to the trace output. +[ "${VORTEX_DEBUG-}" = "1" ] && set -x + +pass "Finished database${_db_index:+ ${_db_index}} dump push to FTP." diff --git a/.vortex/tooling/src/vortex-push-db-s3 b/.vortex/tooling/src/vortex-push-db-s3 index 93470ad6ed..0a9ce5ea95 100755 --- a/.vortex/tooling/src/vortex-push-db-s3 +++ b/.vortex/tooling/src/vortex-push-db-s3 @@ -4,6 +4,12 @@ # # Uses AWS Signature Version 4 with curl (no AWS CLI required). # +# Uploads the database dump file under the remote file name, such as +# 'db.sql', which always holds the latest dump that the fetch downloads. With +# VORTEX_PUSH_DB_TIMESTAMPED_COPY set to 1, the uploaded object is also copied +# to a name with a UTC timestamp, such as 'db_20261111-180000.sql', to keep a +# history. +# # IMPORTANT! This script runs outside the container on the host system. # # shellcheck disable=SC1090,SC1091 @@ -13,33 +19,76 @@ t=$(mktemp) && export -p >"${t}" && set -a && . ./.env && if [ -f ./.env.local ] set -eu set -o pipefail +# Database index suffix. When set (e.g., "2"), all DB-related variable lookups +# use the indexed variant, so each database pushes its own dump file. +_db_index="${VORTEX_DB_INDEX:-}" + +# The S3 values fall back to the values the database is fetched with, so a +# bucket used as the fetch source is configured once. + # AWS access key. -VORTEX_PUSH_DB_S3_ACCESS_KEY="${VORTEX_PUSH_DB_S3_ACCESS_KEY:-${S3_ACCESS_KEY:-}}" +_v="VORTEX_PUSH_DB${_db_index}_S3_ACCESS_KEY" +_vs="VORTEX_FETCH_DB${_db_index}_S3_ACCESS_KEY" +VORTEX_PUSH_DB_S3_ACCESS_KEY="${!_v:-${!_vs:-${S3_ACCESS_KEY:-}}}" # AWS secret key. -VORTEX_PUSH_DB_S3_SECRET_KEY="${VORTEX_PUSH_DB_S3_SECRET_KEY:-${S3_SECRET_KEY:-}}" +_v="VORTEX_PUSH_DB${_db_index}_S3_SECRET_KEY" +_vs="VORTEX_FETCH_DB${_db_index}_S3_SECRET_KEY" +VORTEX_PUSH_DB_S3_SECRET_KEY="${!_v:-${!_vs:-${S3_SECRET_KEY:-}}}" # S3 bucket name. -VORTEX_PUSH_DB_S3_BUCKET="${VORTEX_PUSH_DB_S3_BUCKET:-${S3_BUCKET:-}}" +_v="VORTEX_PUSH_DB${_db_index}_S3_BUCKET" +_vs="VORTEX_FETCH_DB${_db_index}_S3_BUCKET" +VORTEX_PUSH_DB_S3_BUCKET="${!_v:-${!_vs:-${S3_BUCKET:-}}}" # S3 region. -VORTEX_PUSH_DB_S3_REGION="${VORTEX_PUSH_DB_S3_REGION:-${S3_REGION:-}}" +_v="VORTEX_PUSH_DB${_db_index}_S3_REGION" +_vs="VORTEX_FETCH_DB${_db_index}_S3_REGION" +VORTEX_PUSH_DB_S3_REGION="${!_v:-${!_vs:-${S3_REGION:-}}}" # S3 prefix (path within the bucket). -VORTEX_PUSH_DB_S3_PREFIX="${VORTEX_PUSH_DB_S3_PREFIX:-${S3_PREFIX:-}}" +_v="VORTEX_PUSH_DB${_db_index}_S3_PREFIX" +_vs="VORTEX_FETCH_DB${_db_index}_S3_PREFIX" +VORTEX_PUSH_DB_S3_PREFIX="${!_v:-${!_vs:-${S3_PREFIX:-}}}" # Directory with database dump file. -VORTEX_PUSH_DB_S3_DB_DIR="${VORTEX_PUSH_DB_S3_DB_DIR:-${VORTEX_DB_DIR:-./.data}}" +_v="VORTEX_PUSH_DB${_db_index}_S3_DB_DIR" +_vs="VORTEX_FETCH_DB${_db_index}_DIR" +_vss="VORTEX_DB${_db_index}_DIR" +VORTEX_PUSH_DB_S3_DB_DIR="${!_v:-${!_vs:-${!_vss:-${VORTEX_DB_DIR:-./.data}}}}" # Database dump file name. -VORTEX_PUSH_DB_S3_DB_FILE="${VORTEX_PUSH_DB_S3_DB_FILE:-${VORTEX_DB_FILE:-db.sql}}" +# +# Defaults to the name the database is fetched and refreshed with. +_v="VORTEX_PUSH_DB${_db_index}_S3_DB_FILE" +_vs="VORTEX_FETCH_DB${_db_index}_FILE" +_vss="VORTEX_DB${_db_index}_FILE" +VORTEX_PUSH_DB_S3_DB_FILE="${!_v:-${!_vs:-${!_vss:-${VORTEX_DB_FILE:-db.sql}}}}" # Remote database dump file name. -VORTEX_PUSH_DB_S3_REMOTE_FILE="${VORTEX_PUSH_DB_S3_REMOTE_FILE:-db.sql}" +# +# Defaults to the remote file name the database is fetched from, then to the +# database dump file name. +_v="VORTEX_PUSH_DB${_db_index}_S3_REMOTE_FILE" +_vs="VORTEX_FETCH_DB${_db_index}_S3_DB_FILE" +VORTEX_PUSH_DB_S3_REMOTE_FILE="${!_v:-${!_vs:-${VORTEX_PUSH_DB_S3_DB_FILE}}}" # S3 storage class. VORTEX_PUSH_DB_S3_STORAGE_CLASS="${VORTEX_PUSH_DB_S3_STORAGE_CLASS:-STANDARD}" +# Suffix for the names of the pushed database dump files. +# +# Goes before the extension, so 'db.sql' becomes 'db_gha.sql' with '_gha'. +# Use a different suffix in each place that pushes to the same storage. A +# fetch reads the suffixed name only when its remote file name includes it. +VORTEX_PUSH_DB_FILE_SUFFIX="${VORTEX_PUSH_DB_FILE_SUFFIX:-}" + +# Also push a copy of the database dump file named with a UTC timestamp. +# +# Set to 1 to keep a history of the pushed dumps. The fetch never reads the +# timestamped copies. +VORTEX_PUSH_DB_TIMESTAMPED_COPY="${VORTEX_PUSH_DB_TIMESTAMPED_COPY:-}" + # ------------------------------------------------------------------------------ # @formatter:off @@ -86,32 +135,101 @@ aws_signature() { hmac_sha256 "hexkey:${signing_key}" "${string_to_sign}" } +# Send a signed PUT request for an object and print the response body +# followed by the HTTP status code on the last line. +# +# Arguments: the encoded object key, the payload hash, the file to upload +# (empty for none), then any extra 'name:value' headers to sign, with +# lowercase names. +s3_put() { + local key="${1}" + local payload_hash="${2}" + local upload_file="${3}" + shift 3 + + local host="${VORTEX_PUSH_DB_S3_BUCKET}.s3.${VORTEX_PUSH_DB_S3_REGION}.amazonaws.com" + local date_short date_long + date_short=$(date -u +'%Y%m%d') + date_long=$(date -u +'%Y%m%dT%H%M%SZ') + + local canonical_headers signed_headers + canonical_headers=$(printf '%s\n' "host:${host}" "x-amz-content-sha256:${payload_hash}" "x-amz-date:${date_long}" "x-amz-server-side-encryption:AES256" "x-amz-storage-class:${VORTEX_PUSH_DB_S3_STORAGE_CLASS}" "$@" | LC_ALL=C sort) + signed_headers=$(echo "${canonical_headers}" | cut -d: -f1 | paste -sd ';' -) + + local canonical_request="PUT +/${key} + +${canonical_headers} + +${signed_headers} +${payload_hash}" + + local canonical_request_hash + canonical_request_hash=$(hash_sha256 "${canonical_request}") + + local string_to_sign="AWS4-HMAC-SHA256 +${date_long} +${date_short}/${VORTEX_PUSH_DB_S3_REGION}/s3/aws4_request +${canonical_request_hash}" + + local signature + signature=$(aws_signature "${VORTEX_PUSH_DB_S3_SECRET_KEY}" "${date_short}" "${VORTEX_PUSH_DB_S3_REGION}" "s3" "${string_to_sign}") + + local args=(--silent --show-error --location --proto-redir "=https" --connect-timeout "${VORTEX_PUSH_DB_S3_CONNECT_TIMEOUT:-10}" --max-time "${VORTEX_PUSH_DB_S3_MAX_TIME:-300}" --request PUT --write-out "\n%{http_code}") + # curl sends no length for a PUT without a body, and S3 rejects such a + # request. + if [ -n "${upload_file}" ]; then + args+=(--upload-file "${upload_file}") + else + args+=(--header "Content-Length: 0") + fi + + local header + while IFS= read -r header; do + args+=(--header "${header%%:*}: ${header#*:}") + done <<<"${canonical_headers}" + args+=(--header "Authorization: AWS4-HMAC-SHA256 Credential=${VORTEX_PUSH_DB_S3_ACCESS_KEY}/${date_short}/${VORTEX_PUSH_DB_S3_REGION}/s3/aws4_request, SignedHeaders=${signed_headers}, Signature=${signature}") + + curl "${args[@]}" "https://${host}/${key}" +} + for cmd in curl openssl; do command -v "${cmd}" >/dev/null || fail "Command ${cmd} is not available."; done -[ -z "${VORTEX_PUSH_DB_S3_ACCESS_KEY}" ] && fail "Missing required value for VORTEX_PUSH_DB_S3_ACCESS_KEY." -[ -z "${VORTEX_PUSH_DB_S3_SECRET_KEY}" ] && fail "Missing required value for VORTEX_PUSH_DB_S3_SECRET_KEY." -[ -z "${VORTEX_PUSH_DB_S3_BUCKET}" ] && fail "Missing required value for VORTEX_PUSH_DB_S3_BUCKET." -[ -z "${VORTEX_PUSH_DB_S3_REGION}" ] && fail "Missing required value for VORTEX_PUSH_DB_S3_REGION." +[ -z "${VORTEX_PUSH_DB_S3_ACCESS_KEY}" ] && fail "Missing required value for VORTEX_PUSH_DB${_db_index}_S3_ACCESS_KEY." +[ -z "${VORTEX_PUSH_DB_S3_SECRET_KEY}" ] && fail "Missing required value for VORTEX_PUSH_DB${_db_index}_S3_SECRET_KEY." +[ -z "${VORTEX_PUSH_DB_S3_BUCKET}" ] && fail "Missing required value for VORTEX_PUSH_DB${_db_index}_S3_BUCKET." +[ -z "${VORTEX_PUSH_DB_S3_REGION}" ] && fail "Missing required value for VORTEX_PUSH_DB${_db_index}_S3_REGION." local_file="${VORTEX_PUSH_DB_S3_DB_DIR}/${VORTEX_PUSH_DB_S3_DB_FILE}" [ ! -f "${local_file}" ] && fail "Database dump file ${local_file} does not exist." -info "Started database dump push to S3." +info "Started database${_db_index:+ ${_db_index}} dump push to S3." [ -n "${VORTEX_PUSH_DB_S3_PREFIX}" ] && VORTEX_PUSH_DB_S3_PREFIX="${VORTEX_PUSH_DB_S3_PREFIX%/}/" -request_type="PUT" -auth_type="AWS4-HMAC-SHA256" -service="s3" -base_url=".${service}.${VORTEX_PUSH_DB_S3_REGION}.amazonaws.com" -date_short=$(date -u +'%Y%m%d') -date_long=$(date -u +'%Y%m%dT%H%M%SZ') +# The suffix and the timestamp apply to the file name, not to the directories +# in the remote file name. +remote_dir="" +remote_name="${VORTEX_PUSH_DB_S3_REMOTE_FILE}" +if [[ ${remote_name} == */* ]]; then + remote_dir="${remote_name%/*}/" + remote_name="${remote_name##*/}" +fi + +base="${remote_name}" +ext="" +if [[ ${remote_name} == *.* ]]; then + base="${remote_name%.*}" + ext=".${remote_name##*.}" +fi +current_file="${remote_dir}${base}${VORTEX_PUSH_DB_FILE_SUFFIX}${ext}" +timestamped_file="${remote_dir}${base}${VORTEX_PUSH_DB_FILE_SUFFIX}_$(date -u +%Y%m%d-%H%M%S)${ext}" -object_key="$(uri_encode_key "${VORTEX_PUSH_DB_S3_PREFIX}${VORTEX_PUSH_DB_S3_REMOTE_FILE}")" -object_url="https://${VORTEX_PUSH_DB_S3_BUCKET}${base_url}/${object_key}" +object_key="$(uri_encode_key "${VORTEX_PUSH_DB_S3_PREFIX}${current_file}")" +timestamped_key="$(uri_encode_key "${VORTEX_PUSH_DB_S3_PREFIX}${timestamped_file}")" note "Local file: ${local_file}" -note "Remote file: ${VORTEX_PUSH_DB_S3_PREFIX}${VORTEX_PUSH_DB_S3_REMOTE_FILE}" +note "Remote file: ${VORTEX_PUSH_DB_S3_PREFIX}${current_file}" note "S3 bucket: ${VORTEX_PUSH_DB_S3_BUCKET}" note "S3 region: ${VORTEX_PUSH_DB_S3_REGION}" [ -n "${VORTEX_PUSH_DB_S3_PREFIX}" ] && note "S3 prefix: ${VORTEX_PUSH_DB_S3_PREFIX}" @@ -125,53 +243,12 @@ fi payload_hash=$(openssl dgst -sha256 <"${local_file}" | sed 's/^.* //') -header_list='content-type;host;x-amz-content-sha256;x-amz-date;x-amz-server-side-encryption;x-amz-storage-class' - -canonical_request="\ -${request_type} -/${object_key} - -content-type:${content_type} -host:${VORTEX_PUSH_DB_S3_BUCKET}${base_url} -x-amz-content-sha256:${payload_hash} -x-amz-date:${date_long} -x-amz-server-side-encryption:AES256 -x-amz-storage-class:${VORTEX_PUSH_DB_S3_STORAGE_CLASS} - -${header_list} -${payload_hash}" - -canonical_request_hash=$(hash_sha256 "${canonical_request}") - -string_to_sign="\ -${auth_type} -${date_long} -${date_short}/${VORTEX_PUSH_DB_S3_REGION}/${service}/aws4_request -${canonical_request_hash}" - -signature=$(aws_signature "${VORTEX_PUSH_DB_S3_SECRET_KEY}" "${date_short}" "${VORTEX_PUSH_DB_S3_REGION}" "${service}" "${string_to_sign}") - +task "Uploading the database dump file." set +e -response=$(curl --silent --show-error --location --proto-redir =https \ - --connect-timeout "${VORTEX_PUSH_DB_S3_CONNECT_TIMEOUT:-10}" \ - --max-time "${VORTEX_PUSH_DB_S3_MAX_TIME:-300}" \ - --request "${request_type}" --upload-file "${local_file}" \ - --header "Content-Type: ${content_type}" \ - --header "Host: ${VORTEX_PUSH_DB_S3_BUCKET}${base_url}" \ - --header "X-Amz-Content-SHA256: ${payload_hash}" \ - --header "X-Amz-Date: ${date_long}" \ - --header "X-Amz-Server-Side-Encryption: AES256" \ - --header "X-Amz-Storage-Class: ${VORTEX_PUSH_DB_S3_STORAGE_CLASS}" \ - --header "Authorization: ${auth_type} Credential=${VORTEX_PUSH_DB_S3_ACCESS_KEY}/${date_short}/${VORTEX_PUSH_DB_S3_REGION}/${service}/aws4_request, SignedHeaders=${header_list}, Signature=${signature}" \ - --write-out "\n%{http_code}" \ - "${object_url}") +response=$(s3_put "${object_key}" "${payload_hash}" "${local_file}" "content-type:${content_type}") curl_status=$? set -e -# Enable xtrace only after the signed request so credentials and the -# signature are never written to the trace output. -[ "${VORTEX_DEBUG-}" = "1" ] && set -x - [ "${curl_status}" -ne 0 ] && fail "S3 push request failed with curl exit ${curl_status}." http_code=$(echo "${response}" | tail -1) @@ -184,5 +261,32 @@ case "${http_code}" in fail "Unable to push to S3: HTTP status ${http_code}." ;; esac +pass "Uploaded the database dump file." + +if [ "${VORTEX_PUSH_DB_TIMESTAMPED_COPY}" = "1" ]; then + # The copy runs on the S3 side, so the dump is uploaded only once. A copy + # can fail after S3 has sent a 200 status, so a success needs the copy + # result in the body. + task "Copying the database dump file to ${VORTEX_PUSH_DB_S3_PREFIX}${timestamped_file}." + set +e + response=$(s3_put "${timestamped_key}" "$(hash_sha256 "")" "" "x-amz-copy-source:/${VORTEX_PUSH_DB_S3_BUCKET}/${object_key}") + curl_status=$? + set -e + + [ "${curl_status}" -ne 0 ] && fail "S3 copy request failed with curl exit ${curl_status}." + + http_code=$(echo "${response}" | tail -1) + response_body=$(echo "${response}" | sed '$d') + + if [[ ${http_code} != 2* ]] || [[ ${response_body} != *"/dev/null } +@test "fetch-db: Fetch a dump store source from the refresh source with a push requested" { + pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 + + stub_fetch_scripts + + export VORTEX_FETCH_DB_PROCEED="1" + export VORTEX_PUSH_DB_PROCEED="1" + export VORTEX_REFRESH_DB_SOURCE="lagoon" + + for db_source in ftp s3; do + export VORTEX_FETCH_DB_SOURCE="${db_source}" + + run .vortex/tooling/src/vortex-fetch-db + assert_success + assert_output_contains "Fetching from the lagoon source in VORTEX_REFRESH_DB_SOURCE as VORTEX_PUSH_DB_PROCEED is set to 1." + assert_output_contains "Fetched from lagoon." + assert_output_not_contains "Fetched from ${db_source}." + assert_output_contains "Finished database fetch." + done + + popd >/dev/null +} + +@test "fetch-db: Fetch an indexed dump store source from its refresh source with a push requested" { + pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 + + stub_fetch_scripts + + export VORTEX_DB_INDEX=3 + export VORTEX_FETCH_DB3_SOURCE="ftp" + export VORTEX_FETCH_DB3_PROCEED="1" + export VORTEX_FETCH_DB3_DIR="${BUILD_DIR}/db3" + export VORTEX_PUSH_DB_PROCEED="1" + export VORTEX_REFRESH_DB3_SOURCE="url" + export VORTEX_REFRESH_DB_SOURCE="lagoon" + + run .vortex/tooling/src/vortex-fetch-db + assert_success + assert_output_contains "Fetching from the url source in VORTEX_REFRESH_DB3_SOURCE as VORTEX_PUSH_DB_PROCEED is set to 1." + assert_output_contains "Fetched from url." + assert_output_not_contains "Fetched from ftp." + assert_output_not_contains "Fetched from lagoon." + + popd >/dev/null +} + +@test "fetch-db: Fetch a dump store source from the store without a push requested" { + pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 + + stub_fetch_scripts + + export VORTEX_FETCH_DB_SOURCE="ftp" + export VORTEX_FETCH_DB_PROCEED="1" + export VORTEX_REFRESH_DB_SOURCE="lagoon" + unset VORTEX_PUSH_DB_PROCEED + + run .vortex/tooling/src/vortex-fetch-db + assert_success + assert_output_contains "Fetched from ftp." + assert_output_not_contains "Fetched from lagoon." + + popd >/dev/null +} + +@test "fetch-db: Fetch an ftp source without a refresh source as an upstream source" { + pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 + + stub_fetch_scripts + + export VORTEX_FETCH_DB_SOURCE="ftp" + export VORTEX_FETCH_DB_PROCEED="1" + export VORTEX_PUSH_DB_PROCEED="1" + export VORTEX_REFRESH_DB_SOURCE="" + + run .vortex/tooling/src/vortex-fetch-db + assert_success + assert_output_not_contains "Fetching from the" + assert_output_contains "Fetched from ftp." + + popd >/dev/null +} + +@test "fetch-db: Fail with an invalid upstream source for a dump store source with a push requested" { + pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 + + stub_fetch_scripts + + export VORTEX_FETCH_DB_SOURCE="ftp" + export VORTEX_FETCH_DB_PROCEED="1" + export VORTEX_PUSH_DB_PROCEED="1" + export VORTEX_FETCH_DB_SEMAPHORE="${BUILD_DIR}/fetch-db-fresh" + + for refresh_source in ftp container_registry none; do + export VORTEX_REFRESH_DB_SOURCE="${refresh_source}" + + run .vortex/tooling/src/vortex-fetch-db + assert_failure + assert_output_contains "[FAIL] The database dump file pushed to ftp is fetched from an upstream source, but VORTEX_REFRESH_DB_SOURCE is \"${refresh_source}\". Set it to another source of the dump file: url, ftp, acquia, lagoon or s3." + assert_output_not_contains "Fetched from ftp." + assert_output_not_contains "Finished database fetch." + assert_file_not_exists "${BUILD_DIR}/fetch-db-fresh" + done + + popd >/dev/null +} + @test "fetch-db: Fetch nothing with the none source" { pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 diff --git a/.vortex/tooling/tests/unit/push-db-ftp.bats b/.vortex/tooling/tests/unit/push-db-ftp.bats new file mode 100644 index 0000000000..efc6ad44eb --- /dev/null +++ b/.vortex/tooling/tests/unit/push-db-ftp.bats @@ -0,0 +1,262 @@ +#!/usr/bin/env bats +## +# Unit tests for the 'vortex-push-db-ftp' script. +# +# shellcheck disable=SC2030,SC2031,SC2034 + +load ../_helper.bash + +# Exports the FTP values the database is fetched with. +setup_ftp_fetch_values() { + export VORTEX_FETCH_DB_FTP_USER="fetch-user" + export VORTEX_FETCH_DB_FTP_PASS="fetch-pass" + export VORTEX_FETCH_DB_FTP_HOST="ftp.example.com" + export VORTEX_FETCH_DB_FTP_PORT="21" + export VORTEX_FETCH_DB_FTP_FILE="db.sql" + unset VORTEX_PUSH_DB_FTP_USER VORTEX_PUSH_DB_FTP_PASS VORTEX_PUSH_DB_FTP_HOST VORTEX_PUSH_DB_FTP_PORT VORTEX_PUSH_DB_FTP_FILE VORTEX_PUSH_DB_FILE_SUFFIX VORTEX_PUSH_DB_TIMESTAMPED_COPY +} + +@test "push-db-ftp: Upload only the latest dump by default" { + pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 + + mkdir -p .data + echo "CREATE TABLE test (id INT);" >.data/db.sql + + setup_ftp_fetch_values + + mock_curl=$(mock_command "curl") + mock_set_side_effect "${mock_curl}" "cat >${BUILD_DIR}/curl-config" 1 + + run .vortex/tooling/src/vortex-push-db-ftp + assert_success + assert_output_contains "[INFO] Started database dump push to FTP." + assert_output_contains "Local file: ./.data/db.sql" + assert_output_contains "Remote file: db.sql" + assert_output_contains "FTP host: ftp.example.com:21" + assert_output_contains "[ OK ] Uploaded the database dump file." + assert_output_not_contains "timestamped copy" + assert_output_contains "[ OK ] Finished database dump push to FTP." + assert_output_not_contains "fetch-pass" + + assert_equal "1" "$(mock_get_call_num "${mock_curl}")" + + args="$(mock_get_call_args "${mock_curl}" 1)" + assert_string_contains "${args}" "--config - --ftp-create-dirs --upload-file ./.data/db.sql" + # The upload takes a temporary name and replaces the latest dump only after + # it succeeds. + assert_string_contains "${args}" "--quote -*DELE db.sql --quote -RNFR db.sql.uploading --quote -RNTO db.sql" + assert_string_contains "${args}" "ftp://ftp.example.com:21/db.sql.uploading" + # The credentials reach curl on stdin, never as arguments. + assert_string_not_contains "${args}" "fetch-pass" + assert_file_contains "${BUILD_DIR}/curl-config" 'user = "fetch-user:fetch-pass"' + + rm -rf .data + + popd >/dev/null +} + +@test "push-db-ftp: Upload a timestamped copy when enabled" { + pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 + + mkdir -p .data + echo "database content" >.data/db.sql + + setup_ftp_fetch_values + export VORTEX_PUSH_DB_TIMESTAMPED_COPY=1 + + mock_curl=$(mock_command "curl") + + run .vortex/tooling/src/vortex-push-db-ftp + assert_success + assert_output_contains "[ OK ] Uploaded the database dump file." + assert_output_contains "[ OK ] Uploaded the timestamped copy db_" + assert_output_contains "[ OK ] Finished database dump push to FTP." + + assert_equal "2" "$(mock_get_call_num "${mock_curl}")" + + # The latest dump is uploaded first, then the copy under its own name. + assert_string_contains "$(mock_get_call_args "${mock_curl}" 1)" "ftp://ftp.example.com:21/db.sql.uploading" + copy_args="$(mock_get_call_args "${mock_curl}" 2)" + assert_string_contains "${copy_args}" "--upload-file ./.data/db.sql" + [[ ${copy_args} =~ --quote\ -RNTO\ db_[0-9]{8}-[0-9]{6}\.sql ]] + [[ ${copy_args} =~ ftp://ftp\.example\.com:21/db_[0-9]{8}-[0-9]{6}\.sql\.uploading ]] + + rm -rf .data + + popd >/dev/null +} + +@test "push-db-ftp: Push to a remote directory with a file name suffix" { + pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 + + mkdir -p .data + echo "database content" >.data/db.sql + + setup_ftp_fetch_values + export VORTEX_PUSH_DB_FTP_HOST="push.example.com" + export VORTEX_PUSH_DB_FTP_PORT="2121" + export VORTEX_PUSH_DB_FTP_FILE="/backups/site/db.sql" + export VORTEX_PUSH_DB_FILE_SUFFIX="_gha" + export VORTEX_PUSH_DB_TIMESTAMPED_COPY=1 + + mock_curl=$(mock_command "curl") + + run .vortex/tooling/src/vortex-push-db-ftp + assert_success + assert_output_contains "Remote file: backups/site/db_gha.sql" + assert_output_contains "FTP host: push.example.com:2121" + assert_output_contains "[ OK ] Uploaded the timestamped copy backups/site/db_gha_" + + # The rename commands run in the remote directory, so they use bare names. + args="$(mock_get_call_args "${mock_curl}" 1)" + assert_string_contains "${args}" "--quote -*DELE db_gha.sql --quote -RNFR db_gha.sql.uploading --quote -RNTO db_gha.sql" + assert_string_contains "${args}" "ftp://push.example.com:2121/backups/site/db_gha.sql.uploading" + [[ $(mock_get_call_args "${mock_curl}" 2) =~ ftp://push\.example\.com:2121/backups/site/db_gha_[0-9]{8}-[0-9]{6}\.sql\.uploading ]] + + rm -rf .data + + popd >/dev/null +} + +@test "push-db-ftp: Push an indexed database with its own variables and file" { + pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 + + mkdir -p .data + echo "migration content" >.data/db2.sql + + setup_ftp_fetch_values + export VORTEX_DB_INDEX=2 + export VORTEX_FETCH_DB2_FTP_USER="db2-user" + export VORTEX_FETCH_DB2_FTP_PASS="db2-pass" + export VORTEX_FETCH_DB2_FTP_HOST="ftp2.example.com" + export VORTEX_FETCH_DB2_FTP_PORT="21" + export VORTEX_FETCH_DB2_FTP_FILE="db2.sql" + export VORTEX_FETCH_DB2_FILE="db2.sql" + + mock_curl=$(mock_command "curl") + mock_set_side_effect "${mock_curl}" "cat >${BUILD_DIR}/curl-config" 1 + + run .vortex/tooling/src/vortex-push-db-ftp + assert_success + assert_output_contains "[INFO] Started database 2 dump push to FTP." + assert_output_contains "Local file: ./.data/db2.sql" + assert_output_contains "Remote file: db2.sql" + assert_output_contains "[ OK ] Finished database 2 dump push to FTP." + + assert_string_contains "$(mock_get_call_args "${mock_curl}" 1)" "ftp://ftp2.example.com:21/db2.sql.uploading" + assert_file_contains "${BUILD_DIR}/curl-config" 'user = "db2-user:db2-pass"' + + rm -rf .data + + popd >/dev/null +} + +@test "push-db-ftp: Escape quotes and backslashes in the credentials" { + pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 + + mkdir -p .data + echo "database content" >.data/db.sql + + setup_ftp_fetch_values + export VORTEX_FETCH_DB_FTP_PASS='pa"ss\word' + + mock_curl=$(mock_command "curl") + mock_set_side_effect "${mock_curl}" "cat >${BUILD_DIR}/curl-config" 1 + + run .vortex/tooling/src/vortex-push-db-ftp + assert_success + + assert_file_contains "${BUILD_DIR}/curl-config" 'user = "fetch-user:pa\"ss\\word"' + + rm -rf .data + + popd >/dev/null +} + +@test "push-db-ftp: Fail when a required value is missing" { + pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 + + mkdir -p .data + echo "database content" >.data/db.sql + + mock_curl=$(mock_command "curl") + + for name in USER PASS HOST FILE; do + setup_ftp_fetch_values + export "VORTEX_FETCH_DB_FTP_${name}=" + + run .vortex/tooling/src/vortex-push-db-ftp + assert_failure + assert_output_contains "[FAIL] Missing required value for VORTEX_PUSH_DB_FTP_${name} or VORTEX_FETCH_DB_FTP_${name}." + done + + assert_equal "0" "$(mock_get_call_num "${mock_curl}")" + + rm -rf .data + + popd >/dev/null +} + +@test "push-db-ftp: Fail when the database dump file does not exist" { + pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 + + rm -rf .data + + setup_ftp_fetch_values + + mock_curl=$(mock_command "curl") + + run .vortex/tooling/src/vortex-push-db-ftp + assert_failure + assert_output_contains "[FAIL] Database dump file ./.data/db.sql does not exist." + assert_output_not_contains "Started database dump push to FTP." + assert_equal "0" "$(mock_get_call_num "${mock_curl}")" + + popd >/dev/null +} + +@test "push-db-ftp: Fail when the upload fails" { + pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 + + mkdir -p .data + echo "database content" >.data/db.sql + + setup_ftp_fetch_values + export VORTEX_PUSH_DB_TIMESTAMPED_COPY=1 + + mock_curl=$(mock_command "curl") + mock_set_status "${mock_curl}" 21 1 + + run .vortex/tooling/src/vortex-push-db-ftp + assert_failure + assert_output_contains "[FAIL] Unable to push the database dump file to FTP." + assert_output_not_contains "Uploaded the timestamped copy" + assert_equal "1" "$(mock_get_call_num "${mock_curl}")" + + rm -rf .data + + popd >/dev/null +} + +@test "push-db-ftp: Fail when the timestamped copy upload fails" { + pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 + + mkdir -p .data + echo "database content" >.data/db.sql + + setup_ftp_fetch_values + export VORTEX_PUSH_DB_TIMESTAMPED_COPY=1 + + mock_curl=$(mock_command "curl") + mock_set_status "${mock_curl}" 0 1 + mock_set_status "${mock_curl}" 25 2 + + run .vortex/tooling/src/vortex-push-db-ftp + assert_failure + assert_output_contains "[ OK ] Uploaded the database dump file." + assert_output_contains "[FAIL] Unable to push the timestamped copy of the database dump file to FTP." + + rm -rf .data + + popd >/dev/null +} diff --git a/.vortex/tooling/tests/unit/push-db-s3.bats b/.vortex/tooling/tests/unit/push-db-s3.bats index 7f8f96d9c5..95a5ca50f8 100644 --- a/.vortex/tooling/tests/unit/push-db-s3.bats +++ b/.vortex/tooling/tests/unit/push-db-s3.bats @@ -28,6 +28,7 @@ load ../_helper.bash # Clear shortcut variables to prevent environment leakage. unset S3_ACCESS_KEY S3_SECRET_KEY S3_BUCKET S3_REGION S3_PREFIX + export VORTEX_FETCH_DB_S3_ACCESS_KEY="" VORTEX_FETCH_DB_S3_SECRET_KEY="" VORTEX_FETCH_DB_S3_BUCKET="" VORTEX_FETCH_DB_S3_REGION="" VORTEX_FETCH_DB_S3_PREFIX="" export VORTEX_PUSH_DB_S3_ACCESS_KEY="testaccesskey" export VORTEX_PUSH_DB_S3_SECRET_KEY="testsecretkey" @@ -67,6 +68,7 @@ load ../_helper.bash # Clear shortcut variables to prevent environment leakage. unset S3_ACCESS_KEY S3_SECRET_KEY S3_BUCKET S3_REGION S3_PREFIX + export VORTEX_FETCH_DB_S3_ACCESS_KEY="" VORTEX_FETCH_DB_S3_SECRET_KEY="" VORTEX_FETCH_DB_S3_BUCKET="" VORTEX_FETCH_DB_S3_REGION="" VORTEX_FETCH_DB_S3_PREFIX="" export VORTEX_PUSH_DB_S3_ACCESS_KEY="testaccesskey" export VORTEX_PUSH_DB_S3_SECRET_KEY="testsecretkey" @@ -115,6 +117,7 @@ load ../_helper.bash export S3_REGION="us-east-1" # Clear prefix shortcut to prevent environment leakage. unset S3_PREFIX + export VORTEX_FETCH_DB_S3_ACCESS_KEY="" VORTEX_FETCH_DB_S3_SECRET_KEY="" VORTEX_FETCH_DB_S3_BUCKET="" VORTEX_FETCH_DB_S3_REGION="" VORTEX_FETCH_DB_S3_PREFIX="" export VORTEX_PUSH_DB_S3_DB_DIR=".data" export VORTEX_PUSH_DB_S3_DB_FILE="db.sql" @@ -147,6 +150,7 @@ load ../_helper.bash # Clear shortcut variables to prevent environment leakage. unset S3_ACCESS_KEY S3_SECRET_KEY S3_BUCKET S3_REGION S3_PREFIX + export VORTEX_FETCH_DB_S3_ACCESS_KEY="" VORTEX_FETCH_DB_S3_SECRET_KEY="" VORTEX_FETCH_DB_S3_BUCKET="" VORTEX_FETCH_DB_S3_REGION="" VORTEX_FETCH_DB_S3_PREFIX="" export VORTEX_PUSH_DB_S3_ACCESS_KEY="testaccesskey" export VORTEX_PUSH_DB_S3_SECRET_KEY="testsecretkey" @@ -186,6 +190,7 @@ load ../_helper.bash # Clear shortcut variables to prevent environment leakage. unset S3_ACCESS_KEY S3_SECRET_KEY S3_BUCKET S3_REGION S3_PREFIX + export VORTEX_FETCH_DB_S3_ACCESS_KEY="" VORTEX_FETCH_DB_S3_SECRET_KEY="" VORTEX_FETCH_DB_S3_BUCKET="" VORTEX_FETCH_DB_S3_REGION="" VORTEX_FETCH_DB_S3_PREFIX="" export VORTEX_PUSH_DB_S3_ACCESS_KEY="testaccesskey" export VORTEX_PUSH_DB_S3_SECRET_KEY="testsecretkey" @@ -225,6 +230,7 @@ load ../_helper.bash # Clear shortcut variables to prevent environment leakage. unset S3_ACCESS_KEY S3_SECRET_KEY S3_BUCKET S3_REGION S3_PREFIX + export VORTEX_FETCH_DB_S3_ACCESS_KEY="" VORTEX_FETCH_DB_S3_SECRET_KEY="" VORTEX_FETCH_DB_S3_BUCKET="" VORTEX_FETCH_DB_S3_REGION="" VORTEX_FETCH_DB_S3_PREFIX="" export VORTEX_PUSH_DB_S3_ACCESS_KEY="testaccesskey" export VORTEX_PUSH_DB_S3_SECRET_KEY="testsecretkey" @@ -263,6 +269,7 @@ load ../_helper.bash # Clear shortcut variables to prevent environment leakage. unset S3_ACCESS_KEY S3_SECRET_KEY S3_BUCKET S3_REGION S3_PREFIX + export VORTEX_FETCH_DB_S3_ACCESS_KEY="" VORTEX_FETCH_DB_S3_SECRET_KEY="" VORTEX_FETCH_DB_S3_BUCKET="" VORTEX_FETCH_DB_S3_REGION="" VORTEX_FETCH_DB_S3_PREFIX="" export VORTEX_PUSH_DB_S3_ACCESS_KEY="testaccesskey" export VORTEX_PUSH_DB_S3_SECRET_KEY="testsecretkey" @@ -292,6 +299,7 @@ load ../_helper.bash # Clear shortcut variables to prevent environment leakage. unset S3_ACCESS_KEY S3_SECRET_KEY S3_BUCKET S3_REGION S3_PREFIX + export VORTEX_FETCH_DB_S3_ACCESS_KEY="" VORTEX_FETCH_DB_S3_SECRET_KEY="" VORTEX_FETCH_DB_S3_BUCKET="" VORTEX_FETCH_DB_S3_REGION="" VORTEX_FETCH_DB_S3_PREFIX="" export VORTEX_PUSH_DB_S3_ACCESS_KEY="" export VORTEX_PUSH_DB_S3_SECRET_KEY="testsecretkey" @@ -317,6 +325,7 @@ load ../_helper.bash # Clear shortcut variables to prevent environment leakage. unset S3_ACCESS_KEY S3_SECRET_KEY S3_BUCKET S3_REGION S3_PREFIX + export VORTEX_FETCH_DB_S3_ACCESS_KEY="" VORTEX_FETCH_DB_S3_SECRET_KEY="" VORTEX_FETCH_DB_S3_BUCKET="" VORTEX_FETCH_DB_S3_REGION="" VORTEX_FETCH_DB_S3_PREFIX="" export VORTEX_PUSH_DB_S3_ACCESS_KEY="testaccesskey" export VORTEX_PUSH_DB_S3_SECRET_KEY="" @@ -342,6 +351,7 @@ load ../_helper.bash # Clear shortcut variables to prevent environment leakage. unset S3_ACCESS_KEY S3_SECRET_KEY S3_BUCKET S3_REGION S3_PREFIX + export VORTEX_FETCH_DB_S3_ACCESS_KEY="" VORTEX_FETCH_DB_S3_SECRET_KEY="" VORTEX_FETCH_DB_S3_BUCKET="" VORTEX_FETCH_DB_S3_REGION="" VORTEX_FETCH_DB_S3_PREFIX="" export VORTEX_PUSH_DB_S3_ACCESS_KEY="testaccesskey" export VORTEX_PUSH_DB_S3_SECRET_KEY="testsecretkey" @@ -367,6 +377,7 @@ load ../_helper.bash # Clear shortcut variables to prevent environment leakage. unset S3_ACCESS_KEY S3_SECRET_KEY S3_BUCKET S3_REGION S3_PREFIX + export VORTEX_FETCH_DB_S3_ACCESS_KEY="" VORTEX_FETCH_DB_S3_SECRET_KEY="" VORTEX_FETCH_DB_S3_BUCKET="" VORTEX_FETCH_DB_S3_REGION="" VORTEX_FETCH_DB_S3_PREFIX="" export VORTEX_PUSH_DB_S3_ACCESS_KEY="testaccesskey" export VORTEX_PUSH_DB_S3_SECRET_KEY="testsecretkey" @@ -395,6 +406,7 @@ load ../_helper.bash # Clear shortcut variables to prevent environment leakage. unset S3_ACCESS_KEY S3_SECRET_KEY S3_BUCKET S3_REGION S3_PREFIX + export VORTEX_FETCH_DB_S3_ACCESS_KEY="" VORTEX_FETCH_DB_S3_SECRET_KEY="" VORTEX_FETCH_DB_S3_BUCKET="" VORTEX_FETCH_DB_S3_REGION="" VORTEX_FETCH_DB_S3_PREFIX="" export VORTEX_PUSH_DB_S3_ACCESS_KEY="testaccesskey" export VORTEX_PUSH_DB_S3_SECRET_KEY="testsecretkey" @@ -411,3 +423,238 @@ load ../_helper.bash popd >/dev/null } + +@test "push-db-s3: Use the database fetch values" { + pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 + + mkdir -p .data + echo "database content" >.data/db2.sql + + declare -a STEPS=( + "[INFO] Started database dump push to S3." + + "Local file: .data/db2.sql" + "Remote file: store/db2.sql" + "S3 bucket: fetch-bucket" + "S3 region: eu-west-1" + "S3 prefix: store/" + + "@curl * # 0 # 200" + + "[ OK ] Finished database dump push to S3." + ) + + unset S3_ACCESS_KEY S3_SECRET_KEY S3_BUCKET S3_REGION S3_PREFIX + + export VORTEX_PUSH_DB_S3_ACCESS_KEY="" + export VORTEX_PUSH_DB_S3_SECRET_KEY="" + export VORTEX_PUSH_DB_S3_BUCKET="" + export VORTEX_PUSH_DB_S3_REGION="" + export VORTEX_PUSH_DB_S3_PREFIX="" + export VORTEX_FETCH_DB_S3_ACCESS_KEY="fetch-access-key" + export VORTEX_FETCH_DB_S3_SECRET_KEY="fetch-secret-key" + export VORTEX_FETCH_DB_S3_BUCKET="fetch-bucket" + export VORTEX_FETCH_DB_S3_REGION="eu-west-1" + export VORTEX_FETCH_DB_S3_PREFIX="store" + export VORTEX_PUSH_DB_S3_DB_DIR=".data" + # The remote file name follows the dump file name. + export VORTEX_PUSH_DB_S3_DB_FILE="db2.sql" + unset VORTEX_PUSH_DB_S3_REMOTE_FILE + + mocks="$(steps_run "setup")" + run .vortex/tooling/src/vortex-push-db-s3 + steps_run "assert" "${mocks[@]}" + + assert_success + + rm -rf .data + + popd >/dev/null +} + +@test "push-db-s3: Push the database dump file the database is fetched with" { + pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 + + mkdir -p .data + echo "database content" >.data/db_fetched.sql + + declare -a STEPS=( + "[INFO] Started database dump push to S3." + + "Local file: ./.data/db_fetched.sql" + "Remote file: db_fetched.sql" + + "@curl * # 0 # 200" + + "[ OK ] Finished database dump push to S3." + ) + + unset S3_ACCESS_KEY S3_SECRET_KEY S3_BUCKET S3_REGION S3_PREFIX + export VORTEX_FETCH_DB_S3_ACCESS_KEY="" VORTEX_FETCH_DB_S3_SECRET_KEY="" VORTEX_FETCH_DB_S3_BUCKET="" VORTEX_FETCH_DB_S3_REGION="" VORTEX_FETCH_DB_S3_PREFIX="" + + export VORTEX_PUSH_DB_S3_ACCESS_KEY="testaccesskey" + export VORTEX_PUSH_DB_S3_SECRET_KEY="testsecretkey" + export VORTEX_PUSH_DB_S3_BUCKET="test-bucket" + export VORTEX_PUSH_DB_S3_REGION="ap-southeast-2" + export VORTEX_FETCH_DB_FILE="db_fetched.sql" + unset VORTEX_PUSH_DB_S3_DB_DIR VORTEX_PUSH_DB_S3_DB_FILE VORTEX_PUSH_DB_S3_REMOTE_FILE + + mocks="$(steps_run "setup")" + run .vortex/tooling/src/vortex-push-db-s3 + steps_run "assert" "${mocks[@]}" + + assert_success + + rm -rf .data + + popd >/dev/null +} + +# Exports the variables of a push to the 'test-bucket' bucket with the +# 'backups/' prefix. +setup_s3_push() { + mkdir -p .data + echo "database content" >.data/db.sql + + unset S3_ACCESS_KEY S3_SECRET_KEY S3_BUCKET S3_REGION S3_PREFIX + export VORTEX_FETCH_DB_S3_ACCESS_KEY="" VORTEX_FETCH_DB_S3_SECRET_KEY="" VORTEX_FETCH_DB_S3_BUCKET="" VORTEX_FETCH_DB_S3_REGION="" VORTEX_FETCH_DB_S3_PREFIX="" + + export VORTEX_PUSH_DB_S3_ACCESS_KEY="testaccesskey" + export VORTEX_PUSH_DB_S3_SECRET_KEY="testsecretkey" + export VORTEX_PUSH_DB_S3_BUCKET="test-bucket" + export VORTEX_PUSH_DB_S3_REGION="ap-southeast-2" + export VORTEX_PUSH_DB_S3_PREFIX="backups" + export VORTEX_PUSH_DB_S3_DB_DIR=".data" + export VORTEX_PUSH_DB_S3_DB_FILE="db.sql" + unset VORTEX_PUSH_DB_S3_REMOTE_FILE VORTEX_PUSH_DB_FILE_SUFFIX VORTEX_PUSH_DB_TIMESTAMPED_COPY +} + +@test "push-db-s3: Push only the latest dump by default" { + pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 + + setup_s3_push + + mock_curl=$(mock_command "curl") + mock_set_output "${mock_curl}" "200" 1 + + run .vortex/tooling/src/vortex-push-db-s3 + assert_success + assert_output_contains "Remote file: backups/db.sql" + assert_output_contains "[ OK ] Uploaded the database dump file." + assert_output_not_contains "Copying the database dump file" + assert_output_contains "[ OK ] Finished database dump push to S3." + + assert_equal "1" "$(mock_get_call_num "${mock_curl}")" + + upload_args="$(mock_get_call_args "${mock_curl}" 1)" + assert_string_contains "${upload_args}" "--upload-file .data/db.sql" + assert_string_contains "${upload_args}" "https://test-bucket.s3.ap-southeast-2.amazonaws.com/backups/db.sql" + + rm -rf .data + + popd >/dev/null +} + +@test "push-db-s3: Copy the upload to a timestamped copy when enabled" { + pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 + + setup_s3_push + export VORTEX_PUSH_DB_TIMESTAMPED_COPY=1 + + mock_curl=$(mock_command "curl") + mock_set_output "${mock_curl}" "200" 1 + mock_set_output "${mock_curl}" $'\n200' 2 + + run .vortex/tooling/src/vortex-push-db-s3 + assert_success + assert_output_contains "[ OK ] Uploaded the database dump file." + assert_output_contains "[ OK ] Copied the database dump file to backups/db_" + assert_output_contains "[ OK ] Finished database dump push to S3." + + assert_equal "2" "$(mock_get_call_num "${mock_curl}")" + + # The copy runs after the upload, names the latest dump as its source and + # sends no body. + assert_string_contains "$(mock_get_call_args "${mock_curl}" 1)" "https://test-bucket.s3.ap-southeast-2.amazonaws.com/backups/db.sql" + copy_args="$(mock_get_call_args "${mock_curl}" 2)" + assert_string_contains "${copy_args}" "--header x-amz-copy-source: /test-bucket/backups/db.sql" + assert_string_not_contains "${copy_args}" "--upload-file" + assert_string_contains "${copy_args}" "--header Content-Length: 0" + assert_string_not_contains "$(mock_get_call_args "${mock_curl}" 1)" "Content-Length: 0" + [[ ${copy_args} =~ https://test-bucket\.s3\.ap-southeast-2\.amazonaws\.com/backups/db_[0-9]{8}-[0-9]{6}\.sql ]] + + rm -rf .data + + popd >/dev/null +} + +@test "push-db-s3: Fail when the timestamped copy cannot be made" { + pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 + + setup_s3_push + export VORTEX_PUSH_DB_TIMESTAMPED_COPY=1 + + # A copy error can come with a 200 status. Each run makes 2 requests. + mock_curl=$(mock_command "curl") + mock_set_output "${mock_curl}" "200" 1 + mock_set_output "${mock_curl}" $'InternalError\n200' 2 + mock_set_output "${mock_curl}" "200" 3 + mock_set_output "${mock_curl}" $'AccessDenied\n403' 4 + + for code in 200 403; do + run .vortex/tooling/src/vortex-push-db-s3 + assert_failure + assert_output_contains "[ OK ] Uploaded the database dump file." + assert_output_contains "Response: " + assert_output_contains "[FAIL] Unable to copy the database dump file to the timestamped copy: HTTP status ${code}." + assert_output_not_contains "Finished database dump push to S3." + done + + assert_equal "4" "$(mock_get_call_num "${mock_curl}")" + + rm -rf .data + + popd >/dev/null +} + +@test "push-db-s3: Push an indexed database with a file name suffix" { + pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 + + mkdir -p .data + echo "migration content" >.data/db2.sql + + unset S3_ACCESS_KEY S3_SECRET_KEY S3_BUCKET S3_REGION S3_PREFIX + export VORTEX_DB_INDEX=2 + export VORTEX_FETCH_DB2_S3_ACCESS_KEY="db2-access-key" + export VORTEX_FETCH_DB2_S3_SECRET_KEY="db2-secret-key" + export VORTEX_FETCH_DB2_S3_BUCKET="db2-bucket" + export VORTEX_FETCH_DB2_S3_REGION="eu-west-1" + export VORTEX_FETCH_DB2_S3_PREFIX="" + export VORTEX_FETCH_DB2_FILE="db2.sql" + # The dump directory follows the directory the database is fetched to. + mkdir -p .data/db2 + mv .data/db2.sql .data/db2/db2.sql + export VORTEX_FETCH_DB2_DIR=".data/db2" + export VORTEX_PUSH_DB_FILE_SUFFIX="_circleci" + export VORTEX_PUSH_DB_TIMESTAMPED_COPY=1 + + mock_curl=$(mock_command "curl") + mock_set_output "${mock_curl}" "200" 1 + mock_set_output "${mock_curl}" $'\n200' 2 + + run .vortex/tooling/src/vortex-push-db-s3 + assert_success + assert_output_contains "[INFO] Started database 2 dump push to S3." + assert_output_contains "Local file: .data/db2/db2.sql" + assert_output_contains "Remote file: db2_circleci.sql" + assert_output_contains "S3 bucket: db2-bucket" + assert_output_contains "[ OK ] Copied the database dump file to db2_circleci_" + assert_output_contains "[ OK ] Finished database 2 dump push to S3." + + assert_string_contains "$(mock_get_call_args "${mock_curl}" 1)" "https://db2-bucket.s3.eu-west-1.amazonaws.com/db2_circleci.sql" + assert_string_contains "$(mock_get_call_args "${mock_curl}" 2)" "--header x-amz-copy-source: /db2-bucket/db2_circleci.sql" + + rm -rf .data + + popd >/dev/null +} diff --git a/.vortex/tooling/tests/unit/push-db.bats b/.vortex/tooling/tests/unit/push-db.bats new file mode 100644 index 0000000000..dbc3d256f4 --- /dev/null +++ b/.vortex/tooling/tests/unit/push-db.bats @@ -0,0 +1,173 @@ +#!/usr/bin/env bats +## +# Unit tests for the 'vortex-push-db' script. +# +# The stub markers use single quotes on purpose: the `${...}` must reach the +# generated stub literally and expand when the stub runs - so SC2016 is +# suppressed here. +# +# shellcheck disable=SC2016,SC2030,SC2031 + +load ../_helper.bash + +@test "push-db: Skip when the push is not requested" { + pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 + + stub_sibling "vortex-push-db-ftp" "Pushed to ftp." + + export VORTEX_FETCH_DB_SOURCE="ftp" + export VORTEX_REFRESH_DB_SOURCE="lagoon" + + for proceed in "" 0; do + export VORTEX_PUSH_DB_PROCEED="${proceed}" + + run .vortex/tooling/src/vortex-push-db + assert_success + assert_output_contains "[INFO] Started database push." + assert_output_contains "[ OK ] Skipped database push as VORTEX_PUSH_DB_PROCEED is not set to 1." + assert_output_not_contains "Pushed to ftp." + assert_output_not_contains "Finished database push." + done + + popd >/dev/null +} + +@test "push-db: Route each dump store source to its push script" { + pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 + + stub_sibling "vortex-push-db-ftp" 'Pushed to ftp with index [${VORTEX_DB_INDEX}].' + stub_sibling "vortex-push-db-s3" 'Pushed to s3 with index [${VORTEX_DB_INDEX}].' + + export VORTEX_PUSH_DB_PROCEED=1 + export VORTEX_REFRESH_DB_SOURCE="lagoon" + + for db_source in ftp s3; do + export VORTEX_FETCH_DB_SOURCE="${db_source}" + + run .vortex/tooling/src/vortex-push-db + assert_success + assert_output_contains "Pushed to ${db_source} with index []." + assert_output_contains "[ OK ] Finished database push." + + for other_source in ftp s3; do + [ "${other_source}" = "${db_source}" ] && continue + assert_output_not_contains "Pushed to ${other_source}" + done + done + + popd >/dev/null +} + +@test "push-db: Push an indexed database with its own variables" { + pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 + + stub_sibling "vortex-push-db-ftp" 'Pushed to ftp with index [${VORTEX_DB_INDEX}].' + + export VORTEX_DB_INDEX=2 + export VORTEX_PUSH_DB_PROCEED=1 + export VORTEX_FETCH_DB2_SOURCE="ftp" + export VORTEX_REFRESH_DB2_SOURCE="url" + # The indexed database is pushed as fetched, so the sanitization setting + # does not apply to it. + export VORTEX_PROVISION_SANITIZE_DB_SKIP=1 + + run .vortex/tooling/src/vortex-push-db + assert_success + assert_output_contains "[INFO] Started database 2 push." + assert_output_contains "Pushed to ftp with index [2]." + assert_output_contains "[ OK ] Finished database 2 push." + + popd >/dev/null +} + +@test "push-db: Skip for a source that does not accept a push" { + pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 + + stub_sibling "vortex-push-db-ftp" "Pushed to ftp." + stub_sibling "vortex-push-db-s3" "Pushed to s3." + + export VORTEX_PUSH_DB_PROCEED=1 + export VORTEX_REFRESH_DB_SOURCE="lagoon" + + for db_source in url acquia lagoon container_registry none; do + export VORTEX_FETCH_DB_SOURCE="${db_source}" + + run .vortex/tooling/src/vortex-push-db + assert_success + assert_output_contains "[ OK ] Skipped database push as the \"${db_source}\" database source in VORTEX_FETCH_DB_SOURCE does not accept a push. Use one of: ftp, s3." + assert_output_not_contains "Pushed to" + assert_output_not_contains "Finished database push." + done + + popd >/dev/null +} + +@test "push-db: Skip for a dump store source without an upstream source" { + pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 + + stub_sibling "vortex-push-db-ftp" "Pushed to ftp." + + export VORTEX_PUSH_DB_PROCEED=1 + export VORTEX_FETCH_DB_SOURCE="ftp" + export VORTEX_REFRESH_DB_SOURCE="" + + run .vortex/tooling/src/vortex-push-db + assert_success + assert_output_contains "[ OK ] Skipped database push as VORTEX_REFRESH_DB_SOURCE is not set, so the ftp source is not a dump store." + assert_output_not_contains "Pushed to ftp." + + popd >/dev/null +} + +@test "push-db: Use the source from .env by default" { + pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 + + stub_sibling "vortex-push-db-ftp" "Pushed to ftp." + + unset VORTEX_FETCH_DB_SOURCE + export VORTEX_PUSH_DB_PROCEED=1 + + run .vortex/tooling/src/vortex-push-db + assert_success + assert_output_contains '[ OK ] Skipped database push as the "url" database source in VORTEX_FETCH_DB_SOURCE does not accept a push. Use one of: ftp, s3.' + assert_output_not_contains "Pushed to ftp." + + popd >/dev/null +} + +@test "push-db: Refuse to push a database that is not sanitized" { + pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 + + stub_sibling "vortex-push-db-ftp" "Pushed to ftp." + + export VORTEX_FETCH_DB_SOURCE="ftp" + export VORTEX_REFRESH_DB_SOURCE="lagoon" + export VORTEX_PUSH_DB_PROCEED=1 + export VORTEX_PROVISION_SANITIZE_DB_SKIP=1 + + run .vortex/tooling/src/vortex-push-db + assert_failure + assert_output_contains "[FAIL] Refused to push the database as VORTEX_PROVISION_SANITIZE_DB_SKIP is set to 1 and the database dump is not sanitized." + assert_output_not_contains "Pushed to ftp." + assert_output_not_contains "Finished database push." + + popd >/dev/null +} + +@test "push-db: Fail when the push script fails" { + pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 + + printf '#!/usr/bin/env bash\necho "Failed to push to s3."\nexit 1\n' >.vortex/tooling/src/vortex-push-db-s3 + chmod +x .vortex/tooling/src/vortex-push-db-s3 + + export VORTEX_FETCH_DB_SOURCE="s3" + export VORTEX_REFRESH_DB_SOURCE="lagoon" + export VORTEX_PUSH_DB_PROCEED=1 + + run .vortex/tooling/src/vortex-push-db + assert_failure + assert_output_contains "Failed to push to s3." + assert_output_not_contains "Finished database push." + + popd >/dev/null +} diff --git a/.vortex/tooling/tests/unit/refresh-db.bats b/.vortex/tooling/tests/unit/refresh-db.bats index 091c08d083..9160756ad6 100644 --- a/.vortex/tooling/tests/unit/refresh-db.bats +++ b/.vortex/tooling/tests/unit/refresh-db.bats @@ -246,6 +246,156 @@ load ../_helper.bash popd >/dev/null } +@test "refresh-db: Sanitizes the database for a dump store source with a push requested" { + pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 + + mkdir -p .data + echo "upstream" >.data/db.sql + + unset VORTEX_DB_IMAGE VORTEX_DB_IMAGE_BASE + export VORTEX_PUSH_DB_PROCEED=1 + + stub_sibling "vortex-login-container-registry" "Logged in to the registry." + stub_sibling "vortex-fetch-db" "Fetched the dump file." + stub_sibling "vortex-export-db" 'Exported the database to ${1}.' + + export VORTEX_FETCH_DB_SOURCE="ftp" + export VORTEX_REFRESH_DB_SOURCE="lagoon" + + mock_command "sleep" >/dev/null + mock_docker=$(mock_command "docker") + mock_set_output "${mock_docker}" "Started the stack." 1 + mock_set_output "${mock_docker}" "Created the directory." 2 + mock_set_output "${mock_docker}" "Copied the dump file." 3 + mock_set_output "${mock_docker}" "Provisioned the site." 4 + mock_set_output "${mock_docker}" "Sanitized the database." 5 + mock_set_output "${mock_docker}" "Copied the dump file back." 6 + + run .vortex/tooling/src/vortex-refresh-db + assert_success + # The fetch step already fetched the dump from the upstream source. + assert_output_not_contains "Fetched the dump file." + assert_output_contains "[ OK ] Copied database dump file ./.data/db.sql into the cli container." + assert_output_contains "Sanitized the database." + assert_output_contains "Exported the database to db.sql." + assert_output_contains "[ OK ] Finished database refresh." + + # The store holds no database image, so the provisioning imports the dump. + assert_string_contains "$(mock_get_call_args "${mock_docker}" 4)" "-e VORTEX_DB_IMAGE= -e VORTEX_PROVISION_POST_OPERATIONS_SKIP=1 -T cli ./vendor/bin/vortex-provision" + # The provisioning skips the sanitization, so it runs before the export. + assert_string_contains "$(mock_get_call_args "${mock_docker}" 5)" "-T cli ./vendor/drevops/vortex-tooling/src/vortex-provision-sanitize-db" + assert_string_contains "$(mock_get_call_args "${mock_docker}" 6)" "compose --progress quiet cp -L cli:/app/.data/db.sql ./.data/db.sql" + assert_equal "6" "$(mock_get_call_num "${mock_docker}")" + + popd >/dev/null +} + +@test "refresh-db: Exports the dump store database unsanitized when a push is not requested" { + pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 + + mkdir -p .data + echo "store" >.data/db.sql + + unset VORTEX_DB_IMAGE VORTEX_DB_IMAGE_BASE VORTEX_PUSH_DB_PROCEED + export VORTEX_FETCH_DB_SOURCE="ftp" + export VORTEX_REFRESH_DB_SOURCE="lagoon" + + stub_sibling "vortex-login-container-registry" "Logged in to the registry." + stub_sibling "vortex-export-db" 'Exported the database to ${1}.' + + mock_command "sleep" >/dev/null + mock_docker=$(mock_command "docker") + + run .vortex/tooling/src/vortex-refresh-db + assert_success + assert_output_contains "Exported the database to db.sql." + assert_output_contains "[ OK ] Finished database refresh." + + # The dump from the store is already sanitized. + assert_string_contains "$(mock_get_call_args "${mock_docker}" 5)" "compose --progress quiet cp -L cli:/app/.data/db.sql ./.data/db.sql" + assert_equal "5" "$(mock_get_call_num "${mock_docker}")" + + popd >/dev/null +} + +@test "refresh-db: Exports an ftp source database unsanitized without a refresh source" { + pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 + + mkdir -p .data + echo "upstream" >.data/db.sql + + unset VORTEX_DB_IMAGE VORTEX_DB_IMAGE_BASE VORTEX_REFRESH_DB_SOURCE + export VORTEX_FETCH_DB_SOURCE="ftp" + export VORTEX_PUSH_DB_PROCEED=1 + + stub_sibling "vortex-login-container-registry" "Logged in to the registry." + stub_sibling "vortex-export-db" 'Exported the database to ${1}.' + + mock_command "sleep" >/dev/null + mock_docker=$(mock_command "docker") + + run .vortex/tooling/src/vortex-refresh-db + assert_success + assert_output_contains "[ OK ] Finished database refresh." + + # Without a refresh source, the ftp source is not a dump store. + assert_equal "5" "$(mock_get_call_num "${mock_docker}")" + + popd >/dev/null +} + +@test "refresh-db: Fails when the sanitization is skipped for a dump store source with a push requested" { + pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 + + unset VORTEX_DB_IMAGE VORTEX_DB_IMAGE_BASE + export VORTEX_FETCH_DB_SOURCE="ftp" + export VORTEX_REFRESH_DB_SOURCE="lagoon" + export VORTEX_PUSH_DB_PROCEED=1 + export VORTEX_PROVISION_SANITIZE_DB_SKIP=1 + + stub_sibling "vortex-login-container-registry" "Logged in to the registry." + + mock_docker=$(mock_command "docker") + + run .vortex/tooling/src/vortex-refresh-db + assert_failure + assert_output_contains "[FAIL] The database dump file pushed to ftp must be sanitized, but VORTEX_PROVISION_SANITIZE_DB_SKIP is set to 1." + assert_output_not_contains "Logged in to the registry." + assert_equal "0" "$(mock_get_call_num "${mock_docker}")" + + popd >/dev/null +} + +@test "refresh-db: Fails when the sanitization fails for a dump store source" { + pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 + + mkdir -p .data + echo "upstream" >.data/db.sql + + unset VORTEX_DB_IMAGE VORTEX_DB_IMAGE_BASE + export VORTEX_FETCH_DB_SOURCE="s3" + export VORTEX_REFRESH_DB_SOURCE="lagoon" + export VORTEX_PUSH_DB_PROCEED=1 + + stub_sibling "vortex-login-container-registry" "Logged in to the registry." + stub_sibling "vortex-export-db" 'Exported the database to ${1}.' + + mock_command "sleep" >/dev/null + mock_docker=$(mock_command "docker") + mock_set_output "${mock_docker}" "Started the stack." 1 + mock_set_output "${mock_docker}" "Created the directory." 2 + mock_set_output "${mock_docker}" "Copied the dump file." 3 + mock_set_output "${mock_docker}" "Provisioned the site." 4 + mock_set_status "${mock_docker}" 1 5 + + run .vortex/tooling/src/vortex-refresh-db + assert_failure + assert_output_contains "[FAIL] Unable to sanitize the database." + assert_output_not_contains "Exported the database to db.sql." + + popd >/dev/null +} + @test "refresh-db: Fails when the exported dump file cannot be copied from the cli container" { pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1