From 558cad6d20e5540b1d48843b3dee0c5b759b9efb Mon Sep 17 00:00:00 2001 From: Alex Skrypnyk Date: Wed, 11 Nov 2026 10:21:45 +1100 Subject: [PATCH 01/21] [#154] Added Google Drive database fetch and push scripts and the 'vortex-push-db' router. --- .vortex/tooling/composer.json | 1 + .vortex/tooling/src/vortex-fetch-db | 3 +- .vortex/tooling/src/vortex-fetch-db-gdrive | 135 ++++++++ .vortex/tooling/src/vortex-push-db | 54 ++++ .vortex/tooling/src/vortex-push-db-gdrive | 160 ++++++++++ .vortex/tooling/src/vortex-push-db-s3 | 18 +- .vortex/tooling/src/vortex-refresh-db | 49 ++- .vortex/tooling/tests/_helper.bash | 1 + .../tooling/tests/unit/fetch-db-gdrive.bats | 293 +++++++++++++++++ .vortex/tooling/tests/unit/fetch-db.bats | 9 +- .../tooling/tests/unit/push-db-gdrive.bats | 297 ++++++++++++++++++ .vortex/tooling/tests/unit/push-db-s3.bats | 60 ++++ .vortex/tooling/tests/unit/push-db.bats | 125 ++++++++ .vortex/tooling/tests/unit/refresh-db.bats | 180 ++++++++++- 14 files changed, 1369 insertions(+), 16 deletions(-) create mode 100755 .vortex/tooling/src/vortex-fetch-db-gdrive create mode 100755 .vortex/tooling/src/vortex-push-db create mode 100755 .vortex/tooling/src/vortex-push-db-gdrive create mode 100644 .vortex/tooling/tests/unit/fetch-db-gdrive.bats create mode 100644 .vortex/tooling/tests/unit/push-db-gdrive.bats create mode 100644 .vortex/tooling/tests/unit/push-db.bats diff --git a/.vortex/tooling/composer.json b/.vortex/tooling/composer.json index 32d784ea24..7ed3f155a3 100644 --- a/.vortex/tooling/composer.json +++ b/.vortex/tooling/composer.json @@ -34,6 +34,7 @@ "src/vortex-logout", "src/vortex-notify", "src/vortex-provision", + "src/vortex-push-db", "src/vortex-push-db-image", "src/vortex-refresh-db", "src/vortex-reset", diff --git a/.vortex/tooling/src/vortex-fetch-db b/.vortex/tooling/src/vortex-fetch-db index 1cc7d7acf7..faf32b4ac9 100755 --- a/.vortex/tooling/src/vortex-fetch-db +++ b/.vortex/tooling/src/vortex-fetch-db @@ -64,8 +64,9 @@ case "${VORTEX_FETCH_DB_SOURCE}" in lagoon) fetch_script="vortex-fetch-db-lagoon" ;; container_registry) fetch_script="vortex-fetch-db-container-registry" ;; s3) fetch_script="vortex-fetch-db-s3" ;; + gdrive) fetch_script="vortex-fetch-db-gdrive" ;; none) fetch_script="" ;; - *) fail "Unsupported database source \"${VORTEX_FETCH_DB_SOURCE}\" in VORTEX_FETCH_DB${_db_index}_SOURCE. Use one of: url, ftp, acquia, lagoon, container_registry, s3, none." ;; + *) fail "Unsupported database source \"${VORTEX_FETCH_DB_SOURCE}\" in VORTEX_FETCH_DB${_db_index}_SOURCE. Use one of: url, ftp, acquia, lagoon, container_registry, s3, gdrive, none." ;; esac # Skip file existence check for container_registry source as the database is diff --git a/.vortex/tooling/src/vortex-fetch-db-gdrive b/.vortex/tooling/src/vortex-fetch-db-gdrive new file mode 100755 index 0000000000..546b40dc53 --- /dev/null +++ b/.vortex/tooling/src/vortex-fetch-db-gdrive @@ -0,0 +1,135 @@ +#!/usr/bin/env bash +## +# Fetch DB dump from Google Drive. +# +# Authenticates with a Google Cloud service account key and downloads the file +# with the database dump file name from a Google Drive folder. The folder may +# be in a shared drive. +# +# IMPORTANT! This script runs outside the container on the host system. +# +# shellcheck disable=SC1090,SC1091 + +t=$(mktemp) && export -p >"${t}" && set -a && . ./.env && if [ -f ./.env.local ]; then . ./.env.local; fi && set +a && . "${t}" && rm "${t}" && unset t + +set -eu + +# Database index suffix. When set (e.g., "2"), all DB-related variable lookups +# use the indexed variant. +_db_index="${VORTEX_DB_INDEX:-}" + +# Google Cloud service account key: the JSON content or a path to the JSON +# key file. +_v="VORTEX_FETCH_DB${_db_index}_GDRIVE_SERVICE_ACCOUNT_KEY" +VORTEX_FETCH_DB_GDRIVE_SERVICE_ACCOUNT_KEY="${!_v:-${VORTEX_GDRIVE_SERVICE_ACCOUNT_KEY:-}}" + +# Google Drive folder ID with the database dump file. +# +# The ID is the last segment of the folder URL. The folder must be shared with +# the service account. +_v="VORTEX_FETCH_DB${_db_index}_GDRIVE_FOLDER_ID" +VORTEX_FETCH_DB_GDRIVE_FOLDER_ID="${!_v:-}" + +# Directory with database dump file. +_v="VORTEX_FETCH_DB${_db_index}_GDRIVE_DB_DIR" +_vs="VORTEX_FETCH_DB${_db_index}_DIR" +_vss="VORTEX_DB${_db_index}_DIR" +VORTEX_FETCH_DB_GDRIVE_DB_DIR="${!_v:-${!_vs:-${!_vss:-${VORTEX_DB_DIR:-./.data}}}}" + +# Database dump file name. +# +# The same name is used for the file in the Google Drive folder. +_v="VORTEX_FETCH_DB${_db_index}_GDRIVE_DB_FILE" +_vs="VORTEX_FETCH_DB${_db_index}_FILE" +_vss="VORTEX_DB${_db_index}_FILE" +VORTEX_FETCH_DB_GDRIVE_DB_FILE="${!_v:-${!_vs:-${!_vss:-${VORTEX_DB_FILE:-db.sql}}}}" + +# ------------------------------------------------------------------------------ + +# @formatter:off +info() { [ "${TERM:-}" != "dumb" ] && tput colors >/dev/null 2>&1 && printf "\033[36m[INFO] %s\033[0m\n" "${1}" || printf "[INFO] %s\n" "${1}"; } +note() { printf " %s\n" "${1}"; } +task() { _TASK_START=$(date +%s); [ "${TERM:-}" != "dumb" ] && tput colors >/dev/null 2>&1 && printf "\033[34m[TASK] %s\033[0m\n" "${1}" || printf "[TASK] %s\n" "${1}"; } +pass() { _d=""; [ -n "${_TASK_START:-}" ] && _d=" ($(($(date +%s) - _TASK_START))s)" && unset _TASK_START; [ "${TERM:-}" != "dumb" ] && tput colors >/dev/null 2>&1 && printf "\033[32m[ OK ] %s%s\033[0m\n" "${1}" "${_d}" || printf "[ OK ] %s%s\n" "${1}" "${_d}"; } +fail() { [ "${TERM:-}" != "dumb" ] && tput colors >/dev/null 2>&1 && printf "\033[31m[FAIL] %s\033[0m\n" "${1}" || printf "[FAIL] %s\n" "${1}"; exit "${2:-1}"; } +# @formatter:on + +extract_json_last_value() { + local key="${1}" + php -r "\$data=json_decode(file_get_contents('php://stdin'), TRUE); \$last=array_pop(\$data); isset(\$last[\"${key}\"]) ? print trim(json_encode(\$last[\"${key}\"], JSON_UNESCAPED_SLASHES), '\"') : exit(1);" +} + +extract_json_value() { + local key="${1}" + php -r "\$data=json_decode(file_get_contents('php://stdin'), TRUE); isset(\$data[\"${key}\"]) ? print trim(json_encode(\$data[\"${key}\"], JSON_UNESCAPED_SLASHES), '\"') : exit(1);" +} + +# Print a Google OAuth access token for a service account key and a scope. +# +# The key is passed to PHP in the environment, so it never appears in the +# process list. +gdrive_access_token() { + local key="${1}" + local scope="${2}" + + [ -f "${key}" ] && key="$(cat "${key}")" + + local assertion + # shellcheck disable=SC2016 + assertion=$(GDRIVE_KEY="${key}" GDRIVE_SCOPE="${scope}" php -r '$k=json_decode((string) getenv("GDRIVE_KEY"), TRUE); if (!is_array($k) || empty($k["client_email"]) || empty($k["private_key"])) { exit(1); } $b=fn($s) => rtrim(strtr(base64_encode($s), "+/", "-_"), "="); $t=time(); $u=$b(json_encode(["alg" => "RS256", "typ" => "JWT"])) . "." . $b(json_encode(["iss" => $k["client_email"], "scope" => getenv("GDRIVE_SCOPE"), "aud" => "https://oauth2.googleapis.com/token", "iat" => $t, "exp" => $t + 3600])); if (!@openssl_sign($u, $s, $k["private_key"], OPENSSL_ALGO_SHA256)) { exit(1); } print $u . "." . $b($s);') || return 1 + + local token_json + token_json=$(curl -sS -L https://oauth2.googleapis.com/token --data-urlencode "grant_type=urn:ietf:params:oauth:grant-type:jwt-bearer" --data-urlencode "assertion=${assertion}") || return 1 + + echo "${token_json}" | extract_json_value "access_token" +} + +# Print the ID of the most recently modified file with a name in a folder. +# +# Prints nothing when the folder has no such file, and fails when the folder +# cannot be listed. +gdrive_find_file() { + local token="${1}" + local folder_id="${2}" + local name="${3}" + + local files_json + files_json=$(curl -sS -G https://www.googleapis.com/drive/v3/files -H "Authorization: Bearer ${token}" --data-urlencode "q=name = '${name//\'/\\\'}' and '${folder_id}' in parents and trashed = false" --data-urlencode "fields=files(id)" --data-urlencode "orderBy=modifiedTime desc" --data-urlencode "pageSize=1" --data-urlencode "supportsAllDrives=true" --data-urlencode "includeItemsFromAllDrives=true") || return 1 + + local files + files=$(echo "${files_json}" | extract_json_value "files") || return 1 + + echo "${files}" | extract_json_last_value "id" || true +} + +for cmd in php curl; do command -v "${cmd}" >/dev/null || fail "Command ${cmd} is not available."; done + +[ -z "${VORTEX_FETCH_DB_GDRIVE_SERVICE_ACCOUNT_KEY}" ] && fail "Missing required value for VORTEX_FETCH_DB${_db_index}_GDRIVE_SERVICE_ACCOUNT_KEY or VORTEX_GDRIVE_SERVICE_ACCOUNT_KEY." +[ -z "${VORTEX_FETCH_DB_GDRIVE_FOLDER_ID}" ] && fail "Missing required value for VORTEX_FETCH_DB${_db_index}_GDRIVE_FOLDER_ID." + +info "Started database dump fetch from Google Drive." + +note "Remote file: ${VORTEX_FETCH_DB_GDRIVE_DB_FILE}" +note "Local path: ${VORTEX_FETCH_DB_GDRIVE_DB_DIR}/${VORTEX_FETCH_DB_GDRIVE_DB_FILE}" +note "Folder ID: ${VORTEX_FETCH_DB_GDRIVE_FOLDER_ID}" + +task "Authenticating with the service account." +token=$(gdrive_access_token "${VORTEX_FETCH_DB_GDRIVE_SERVICE_ACCOUNT_KEY}" "https://www.googleapis.com/auth/drive.readonly") || fail "Unable to authenticate with the service account. Check that VORTEX_FETCH_DB${_db_index}_GDRIVE_SERVICE_ACCOUNT_KEY or VORTEX_GDRIVE_SERVICE_ACCOUNT_KEY is a valid service account JSON key." +pass "Authenticated with the service account." + +task "Finding the database dump file in the folder." +file_id=$(gdrive_find_file "${token}" "${VORTEX_FETCH_DB_GDRIVE_FOLDER_ID}" "${VORTEX_FETCH_DB_GDRIVE_DB_FILE}") || fail "Unable to list the ${VORTEX_FETCH_DB_GDRIVE_FOLDER_ID} folder. Check that the folder ID is correct and that the folder is shared with the service account." +[ -z "${file_id}" ] && fail "Unable to find the ${VORTEX_FETCH_DB_GDRIVE_DB_FILE} file in the ${VORTEX_FETCH_DB_GDRIVE_FOLDER_ID} folder." +pass "Found the database dump file with ID ${file_id}." + +mkdir -p "${VORTEX_FETCH_DB_GDRIVE_DB_DIR}" + +task "Downloading the database dump file." +curl -sS -f -L -H "Authorization: Bearer ${token}" -o "${VORTEX_FETCH_DB_GDRIVE_DB_DIR}/${VORTEX_FETCH_DB_GDRIVE_DB_FILE}" "https://www.googleapis.com/drive/v3/files/${file_id}?alt=media&supportsAllDrives=true" || fail "Unable to download the database dump file." +pass "Downloaded the database dump file." + +# Enable xtrace only after the authenticated requests so the access token is +# never written to the trace output. +[ "${VORTEX_DEBUG-}" = "1" ] && set -x + +pass "Finished database dump fetch from Google Drive." diff --git a/.vortex/tooling/src/vortex-push-db b/.vortex/tooling/src/vortex-push-db new file mode 100755 index 0000000000..4ba20e8c4d --- /dev/null +++ b/.vortex/tooling/src/vortex-push-db @@ -0,0 +1,54 @@ +#!/usr/bin/env bash +## +# Push database dump. +# +# This is a router script to call relevant scripts based on type. +# +# The database dump file is pushed to the storage the database is fetched +# from, set in VORTEX_FETCH_DB_SOURCE. Only the 'gdrive' and 's3' sources +# accept a push; the push is skipped for other sources. +# +# IMPORTANT! This script runs outside the container on the host system. +# +# shellcheck disable=SC1090,SC1091 + +t=$(mktemp) && export -p >"${t}" && set -a && . ./.env && if [ -f ./.env.local ]; then . ./.env.local; fi && set +a && . "${t}" && rm "${t}" && unset t + +set -eu +[ "${VORTEX_DEBUG-}" = "1" ] && set -x + +# Proceed with push. +# +# Set to 1 in the nightly CI job only, so the storage is refreshed once a day +# and never from a feature branch build. +VORTEX_PUSH_DB_PROCEED="${VORTEX_PUSH_DB_PROCEED:-}" + +# Database fetch source, which is also the push destination. +VORTEX_FETCH_DB_SOURCE="${VORTEX_FETCH_DB_SOURCE:-url}" + +# ------------------------------------------------------------------------------ + +# @formatter:off +info() { [ "${TERM:-}" != "dumb" ] && tput colors >/dev/null 2>&1 && printf "\033[36m[INFO] %s\033[0m\n" "${1}" || printf "[INFO] %s\n" "${1}"; } +note() { printf " %s\n" "${1}"; } +task() { _TASK_START=$(date +%s); [ "${TERM:-}" != "dumb" ] && tput colors >/dev/null 2>&1 && printf "\033[34m[TASK] %s\033[0m\n" "${1}" || printf "[TASK] %s\n" "${1}"; } +pass() { _d=""; [ -n "${_TASK_START:-}" ] && _d=" ($(($(date +%s) - _TASK_START))s)" && unset _TASK_START; [ "${TERM:-}" != "dumb" ] && tput colors >/dev/null 2>&1 && printf "\033[32m[ OK ] %s%s\033[0m\n" "${1}" "${_d}" || printf "[ OK ] %s%s\n" "${1}" "${_d}"; } +fail() { [ "${TERM:-}" != "dumb" ] && tput colors >/dev/null 2>&1 && printf "\033[31m[FAIL] %s\033[0m\n" "${1}" || printf "[FAIL] %s\n" "${1}"; exit "${2:-1}"; } +# @formatter:on + +info "Started database push." + +[ "${VORTEX_PUSH_DB_PROCEED}" != "1" ] && pass "Skipped database push as VORTEX_PUSH_DB_PROCEED is not set to 1." && exit 0 + +case "${VORTEX_FETCH_DB_SOURCE}" in + gdrive) push_script="vortex-push-db-gdrive" ;; + s3) push_script="vortex-push-db-s3" ;; + *) pass "Skipped database push as the \"${VORTEX_FETCH_DB_SOURCE}\" database source in VORTEX_FETCH_DB_SOURCE does not accept a push. Use one of: gdrive, s3." && exit 0 ;; +esac + +# The pushed dump leaves the hosting and CI, so it must be sanitized. +[ "${VORTEX_PROVISION_SANITIZE_DB_SKIP:-}" = "1" ] && fail "Refused to push the database as VORTEX_PROVISION_SANITIZE_DB_SKIP is set to 1 and the database dump is not sanitized." + +"$(dirname "${BASH_SOURCE[0]}")/${push_script}" + +pass "Finished database push." diff --git a/.vortex/tooling/src/vortex-push-db-gdrive b/.vortex/tooling/src/vortex-push-db-gdrive new file mode 100755 index 0000000000..0f153bc016 --- /dev/null +++ b/.vortex/tooling/src/vortex-push-db-gdrive @@ -0,0 +1,160 @@ +#!/usr/bin/env bash +## +# Push DB dump to Google Drive. +# +# Authenticates with a Google Cloud service account key and uploads the +# database dump file to a Google Drive folder. A file with the same name in +# the folder is replaced in place, so its ID and sharing settings are kept. +# +# Service accounts have no storage quota of their own, so the folder must be +# in a shared drive. +# +# IMPORTANT! This script runs outside the container on the host system. +# +# shellcheck disable=SC1090,SC1091 + +t=$(mktemp) && export -p >"${t}" && set -a && . ./.env && if [ -f ./.env.local ]; then . ./.env.local; fi && set +a && . "${t}" && rm "${t}" && unset t + +set -eu +set -o pipefail + +# Google Cloud service account key: the JSON content or a path to the JSON +# key file. +VORTEX_PUSH_DB_GDRIVE_SERVICE_ACCOUNT_KEY="${VORTEX_PUSH_DB_GDRIVE_SERVICE_ACCOUNT_KEY:-${VORTEX_GDRIVE_SERVICE_ACCOUNT_KEY:-}}" + +# Google Drive folder ID to push the database dump file to. +# +# Defaults to the folder the database is fetched from. +VORTEX_PUSH_DB_GDRIVE_FOLDER_ID="${VORTEX_PUSH_DB_GDRIVE_FOLDER_ID:-${VORTEX_FETCH_DB_GDRIVE_FOLDER_ID:-}}" + +# Directory with database dump file. +VORTEX_PUSH_DB_GDRIVE_DB_DIR="${VORTEX_PUSH_DB_GDRIVE_DB_DIR:-${VORTEX_DB_DIR:-./.data}}" + +# Database dump file name. +# +# The same name is used for the file in the Google Drive folder. +VORTEX_PUSH_DB_GDRIVE_DB_FILE="${VORTEX_PUSH_DB_GDRIVE_DB_FILE:-${VORTEX_DB_FILE:-db.sql}}" + +# ------------------------------------------------------------------------------ + +# @formatter:off +info() { [ "${TERM:-}" != "dumb" ] && tput colors >/dev/null 2>&1 && printf "\033[36m[INFO] %s\033[0m\n" "${1}" || printf "[INFO] %s\n" "${1}"; } +note() { printf " %s\n" "${1}"; } +task() { _TASK_START=$(date +%s); [ "${TERM:-}" != "dumb" ] && tput colors >/dev/null 2>&1 && printf "\033[34m[TASK] %s\033[0m\n" "${1}" || printf "[TASK] %s\n" "${1}"; } +pass() { _d=""; [ -n "${_TASK_START:-}" ] && _d=" ($(($(date +%s) - _TASK_START))s)" && unset _TASK_START; [ "${TERM:-}" != "dumb" ] && tput colors >/dev/null 2>&1 && printf "\033[32m[ OK ] %s%s\033[0m\n" "${1}" "${_d}" || printf "[ OK ] %s%s\n" "${1}" "${_d}"; } +fail() { [ "${TERM:-}" != "dumb" ] && tput colors >/dev/null 2>&1 && printf "\033[31m[FAIL] %s\033[0m\n" "${1}" || printf "[FAIL] %s\n" "${1}"; exit "${2:-1}"; } +# @formatter:on + +extract_json_last_value() { + local key="${1}" + php -r "\$data=json_decode(file_get_contents('php://stdin'), TRUE); \$last=array_pop(\$data); isset(\$last[\"${key}\"]) ? print trim(json_encode(\$last[\"${key}\"], JSON_UNESCAPED_SLASHES), '\"') : exit(1);" +} + +extract_json_value() { + local key="${1}" + php -r "\$data=json_decode(file_get_contents('php://stdin'), TRUE); isset(\$data[\"${key}\"]) ? print trim(json_encode(\$data[\"${key}\"], JSON_UNESCAPED_SLASHES), '\"') : exit(1);" +} + +# Print a Google OAuth access token for a service account key and a scope. +# +# The key is passed to PHP in the environment, so it never appears in the +# process list. +gdrive_access_token() { + local key="${1}" + local scope="${2}" + + [ -f "${key}" ] && key="$(cat "${key}")" + + local assertion + # shellcheck disable=SC2016 + assertion=$(GDRIVE_KEY="${key}" GDRIVE_SCOPE="${scope}" php -r '$k=json_decode((string) getenv("GDRIVE_KEY"), TRUE); if (!is_array($k) || empty($k["client_email"]) || empty($k["private_key"])) { exit(1); } $b=fn($s) => rtrim(strtr(base64_encode($s), "+/", "-_"), "="); $t=time(); $u=$b(json_encode(["alg" => "RS256", "typ" => "JWT"])) . "." . $b(json_encode(["iss" => $k["client_email"], "scope" => getenv("GDRIVE_SCOPE"), "aud" => "https://oauth2.googleapis.com/token", "iat" => $t, "exp" => $t + 3600])); if (!@openssl_sign($u, $s, $k["private_key"], OPENSSL_ALGO_SHA256)) { exit(1); } print $u . "." . $b($s);') || return 1 + + local token_json + token_json=$(curl -sS -L https://oauth2.googleapis.com/token --data-urlencode "grant_type=urn:ietf:params:oauth:grant-type:jwt-bearer" --data-urlencode "assertion=${assertion}") || return 1 + + echo "${token_json}" | extract_json_value "access_token" +} + +# Print the ID of the most recently modified file with a name in a folder. +# +# Prints nothing when the folder has no such file, and fails when the folder +# cannot be listed. +gdrive_find_file() { + local token="${1}" + local folder_id="${2}" + local name="${3}" + + local files_json + files_json=$(curl -sS -G https://www.googleapis.com/drive/v3/files -H "Authorization: Bearer ${token}" --data-urlencode "q=name = '${name//\'/\\\'}' and '${folder_id}' in parents and trashed = false" --data-urlencode "fields=files(id)" --data-urlencode "orderBy=modifiedTime desc" --data-urlencode "pageSize=1" --data-urlencode "supportsAllDrives=true" --data-urlencode "includeItemsFromAllDrives=true") || return 1 + + local files + files=$(echo "${files_json}" | extract_json_value "files") || return 1 + + echo "${files}" | extract_json_last_value "id" || true +} + +for cmd in php curl; do command -v "${cmd}" >/dev/null || fail "Command ${cmd} is not available."; done + +[ -z "${VORTEX_PUSH_DB_GDRIVE_SERVICE_ACCOUNT_KEY}" ] && fail "Missing required value for VORTEX_PUSH_DB_GDRIVE_SERVICE_ACCOUNT_KEY or VORTEX_GDRIVE_SERVICE_ACCOUNT_KEY." +[ -z "${VORTEX_PUSH_DB_GDRIVE_FOLDER_ID}" ] && fail "Missing required value for VORTEX_PUSH_DB_GDRIVE_FOLDER_ID or VORTEX_FETCH_DB_GDRIVE_FOLDER_ID." + +local_file="${VORTEX_PUSH_DB_GDRIVE_DB_DIR}/${VORTEX_PUSH_DB_GDRIVE_DB_FILE}" +[ ! -f "${local_file}" ] && fail "Database dump file ${local_file} does not exist." + +info "Started database dump push to Google Drive." + +note "Local file: ${local_file}" +note "Remote file: ${VORTEX_PUSH_DB_GDRIVE_DB_FILE}" +note "Folder ID: ${VORTEX_PUSH_DB_GDRIVE_FOLDER_ID}" + +task "Authenticating with the service account." +token=$(gdrive_access_token "${VORTEX_PUSH_DB_GDRIVE_SERVICE_ACCOUNT_KEY}" "https://www.googleapis.com/auth/drive") || fail "Unable to authenticate with the service account. Check that VORTEX_PUSH_DB_GDRIVE_SERVICE_ACCOUNT_KEY or VORTEX_GDRIVE_SERVICE_ACCOUNT_KEY is a valid service account JSON key." +pass "Authenticated with the service account." + +file_id=$(gdrive_find_file "${token}" "${VORTEX_PUSH_DB_GDRIVE_FOLDER_ID}" "${VORTEX_PUSH_DB_GDRIVE_DB_FILE}") || fail "Unable to list the ${VORTEX_PUSH_DB_GDRIVE_FOLDER_ID} folder. Check that the folder ID is correct and that the folder is shared with the service account." + +upload_url="https://www.googleapis.com/upload/drive/v3/files" +if [ -n "${file_id}" ]; then + note "Replacing the existing file with ID ${file_id}." + request_type="PATCH" + upload_url="${upload_url}/${file_id}" + metadata="{}" +else + note "Creating a new file in the folder." + request_type="POST" + metadata=$(GDRIVE_NAME="${VORTEX_PUSH_DB_GDRIVE_DB_FILE}" GDRIVE_FOLDER_ID="${VORTEX_PUSH_DB_GDRIVE_FOLDER_ID}" php -r 'print json_encode(["name" => getenv("GDRIVE_NAME"), "parents" => [getenv("GDRIVE_FOLDER_ID")]]);') +fi + +# A resumable upload session accepts the whole file in a single request, so +# the dump is not limited by the 5 MB multipart upload size. +task "Starting the upload session." +session_headers=$(curl -sS -X "${request_type}" -D - -o /dev/null -H "Authorization: Bearer ${token}" -H "Content-Type: application/json; charset=UTF-8" -H "X-Upload-Content-Type: application/octet-stream" --data "${metadata}" "${upload_url}?uploadType=resumable&supportsAllDrives=true") || fail "Unable to start the upload session." +session_url=$(echo "${session_headers}" | sed -n 's/^[Ll]ocation: *//p' | tr -d '\r') +[ -z "${session_url}" ] && fail "Unable to start the upload session: the response has no upload location. Check that the folder is in a shared drive and that the service account can edit it." +pass "Started the upload session." + +task "Uploading the database dump file." +set +e +response=$(curl -sS -X PUT --upload-file "${local_file}" -H "Content-Type: application/octet-stream" --write-out "\n%{http_code}" "${session_url}") +curl_status=$? +set -e + +# Enable xtrace only after the authenticated requests so the access token and +# the upload session URL are never written to the trace output. +[ "${VORTEX_DEBUG-}" = "1" ] && set -x + +[ "${curl_status}" -ne 0 ] && fail "Google Drive upload request failed with curl exit ${curl_status}." + +http_code=$(echo "${response}" | tail -1) +response_body=$(echo "${response}" | sed '$d') + +case "${http_code}" in + 2*) ;; + *) + [ -n "${response_body}" ] && echo "Response: ${response_body}" + fail "Unable to push to Google Drive: HTTP status ${http_code}." + ;; +esac +pass "Uploaded the database dump file." + +pass "Finished database dump push to Google Drive." diff --git a/.vortex/tooling/src/vortex-push-db-s3 b/.vortex/tooling/src/vortex-push-db-s3 index 93470ad6ed..569555f344 100755 --- a/.vortex/tooling/src/vortex-push-db-s3 +++ b/.vortex/tooling/src/vortex-push-db-s3 @@ -13,20 +13,23 @@ t=$(mktemp) && export -p >"${t}" && set -a && . ./.env && if [ -f ./.env.local ] set -eu set -o pipefail +# The S3 values fall back to the values the database is fetched with, so a +# bucket used as the fetch source is configured once. + # AWS access key. -VORTEX_PUSH_DB_S3_ACCESS_KEY="${VORTEX_PUSH_DB_S3_ACCESS_KEY:-${S3_ACCESS_KEY:-}}" +VORTEX_PUSH_DB_S3_ACCESS_KEY="${VORTEX_PUSH_DB_S3_ACCESS_KEY:-${VORTEX_FETCH_DB_S3_ACCESS_KEY:-${S3_ACCESS_KEY:-}}}" # AWS secret key. -VORTEX_PUSH_DB_S3_SECRET_KEY="${VORTEX_PUSH_DB_S3_SECRET_KEY:-${S3_SECRET_KEY:-}}" +VORTEX_PUSH_DB_S3_SECRET_KEY="${VORTEX_PUSH_DB_S3_SECRET_KEY:-${VORTEX_FETCH_DB_S3_SECRET_KEY:-${S3_SECRET_KEY:-}}}" # S3 bucket name. -VORTEX_PUSH_DB_S3_BUCKET="${VORTEX_PUSH_DB_S3_BUCKET:-${S3_BUCKET:-}}" +VORTEX_PUSH_DB_S3_BUCKET="${VORTEX_PUSH_DB_S3_BUCKET:-${VORTEX_FETCH_DB_S3_BUCKET:-${S3_BUCKET:-}}}" # S3 region. -VORTEX_PUSH_DB_S3_REGION="${VORTEX_PUSH_DB_S3_REGION:-${S3_REGION:-}}" +VORTEX_PUSH_DB_S3_REGION="${VORTEX_PUSH_DB_S3_REGION:-${VORTEX_FETCH_DB_S3_REGION:-${S3_REGION:-}}}" # S3 prefix (path within the bucket). -VORTEX_PUSH_DB_S3_PREFIX="${VORTEX_PUSH_DB_S3_PREFIX:-${S3_PREFIX:-}}" +VORTEX_PUSH_DB_S3_PREFIX="${VORTEX_PUSH_DB_S3_PREFIX:-${VORTEX_FETCH_DB_S3_PREFIX:-${S3_PREFIX:-}}}" # Directory with database dump file. VORTEX_PUSH_DB_S3_DB_DIR="${VORTEX_PUSH_DB_S3_DB_DIR:-${VORTEX_DB_DIR:-./.data}}" @@ -35,7 +38,10 @@ VORTEX_PUSH_DB_S3_DB_DIR="${VORTEX_PUSH_DB_S3_DB_DIR:-${VORTEX_DB_DIR:-./.data}} VORTEX_PUSH_DB_S3_DB_FILE="${VORTEX_PUSH_DB_S3_DB_FILE:-${VORTEX_DB_FILE:-db.sql}}" # Remote database dump file name. -VORTEX_PUSH_DB_S3_REMOTE_FILE="${VORTEX_PUSH_DB_S3_REMOTE_FILE:-db.sql}" +# +# Defaults to the database dump file name, which is the name the database is +# fetched with. +VORTEX_PUSH_DB_S3_REMOTE_FILE="${VORTEX_PUSH_DB_S3_REMOTE_FILE:-${VORTEX_PUSH_DB_S3_DB_FILE}}" # S3 storage class. VORTEX_PUSH_DB_S3_STORAGE_CLASS="${VORTEX_PUSH_DB_S3_STORAGE_CLASS:-STANDARD}" diff --git a/.vortex/tooling/src/vortex-refresh-db b/.vortex/tooling/src/vortex-refresh-db index bc8ee64a8c..9b2f01e2e5 100755 --- a/.vortex/tooling/src/vortex-refresh-db +++ b/.vortex/tooling/src/vortex-refresh-db @@ -15,6 +15,11 @@ # the service is exported and pushed as VORTEX_DB2_IMAGE, without # provisioning. This runs only when VORTEX_REFRESH_DB2_PROCEED is set to 1. # +# With a dump store fetch source ('gdrive' or 's3') and VORTEX_PUSH_DB_PROCEED +# set to 1, the dump file is fetched from VORTEX_REFRESH_DB_SOURCE before +# provisioning and the database is sanitized after it, so the exported dump +# can be pushed to the store with 'vortex-push-db'. +# # Every variable of the calling environment is passed to the provisioning in # the 'cli' container, so the script is intended to run in CI. # @@ -39,12 +44,16 @@ _db_index="${VORTEX_DB_INDEX:-}" # VORTEX_DB2_IMAGE_BASE and falls back to this value. VORTEX_DB_IMAGE_BASE="${VORTEX_DB_IMAGE_BASE:-}" -# Fetch source of the database dump file to import into the base image. +# Upstream fetch source of the database dump file. # # Required when the fetch source is 'container_registry' and a base image is # set: the database image is built from a dump file, so it needs a dump file -# source such as 'url', 'ftp', 'acquia', 'lagoon' or 's3'. An indexed -# database uses VORTEX_REFRESH_DB2_SOURCE. +# source such as 'url', 'ftp', 'acquia', 'lagoon', 's3' or 'gdrive'. An +# indexed database uses VORTEX_REFRESH_DB2_SOURCE. +# +# Also required when the fetch source is a dump store ('gdrive' or 's3') and +# VORTEX_PUSH_DB_PROCEED is set to 1: the dump pushed to the store comes from +# an upstream source such as 'url', 'ftp', 'acquia' or 'lagoon'. _v="VORTEX_REFRESH_DB${_db_index}_SOURCE" VORTEX_REFRESH_DB_SOURCE="${!_v:-}" @@ -98,14 +107,40 @@ fi rebuild_image=0 [ -n "${db_image}" ] && [ -n "${db_image_base}" ] && rebuild_image=1 +# A dump store ('gdrive' or 's3') receives the dump this refresh exports, so +# with a push requested the dump is fetched again from its upstream source. +refresh_store=0 +if [ -z "${_db_index}" ] && [ -z "${db_image}" ] && [ "${VORTEX_PUSH_DB_PROCEED:-}" = "1" ]; then + case "${fetch_source}" in + gdrive | s3) refresh_store=1 ;; + esac +fi + +if [ "${refresh_store}" -eq 1 ]; then + case "${VORTEX_REFRESH_DB_SOURCE}" in + "" | gdrive | s3 | container_registry | none) fail "The database dump file pushed to ${fetch_source} is fetched from an upstream source, but VORTEX_REFRESH_DB_SOURCE is \"${VORTEX_REFRESH_DB_SOURCE}\". Set it to the source of the dump file: url, ftp, acquia or lagoon." ;; + esac + [ "${VORTEX_PROVISION_SANITIZE_DB_SKIP:-}" = "1" ] && fail "The database dump file pushed to ${fetch_source} must be sanitized, but VORTEX_PROVISION_SANITIZE_DB_SKIP is set to 1." +fi + if [ "${rebuild_image}" -eq 1 ] && [ "${fetch_source}" = "container_registry" ]; then case "${VORTEX_REFRESH_DB_SOURCE}" in - "" | container_registry | none) fail "The ${db_image} image is built from a database dump file, but VORTEX_REFRESH_DB${_db_index}_SOURCE is \"${VORTEX_REFRESH_DB_SOURCE}\". Set it to the source of the dump file: url, ftp, acquia, lagoon or s3." ;; + "" | container_registry | none) fail "The ${db_image} image is built from a database dump file, but VORTEX_REFRESH_DB${_db_index}_SOURCE is \"${VORTEX_REFRESH_DB_SOURCE}\". Set it to the source of the dump file: url, ftp, acquia, lagoon, s3 or gdrive." ;; esac fi "$(dirname "${BASH_SOURCE[0]}")/vortex-login-container-registry" +if [ "${refresh_store}" -eq 1 ]; then + # The dump fetched from the store is removed, so the check below proves the + # upstream source wrote a new one. + rm -f "${db_file}" + # CircleCI re-sources the job variables from BASH_ENV in every bash + # script, which would replace the prefixed values. + env BASH_ENV='' "VORTEX_FETCH_DB_SOURCE=${VORTEX_REFRESH_DB_SOURCE}" "VORTEX_FETCH_DB_FORCE=1" "$(dirname "${BASH_SOURCE[0]}")/vortex-fetch-db" + [ -f "${db_file}" ] || fail "The database dump file was not fetched to ${db_file}. Remove the output directory and file name overrides of the database fetch source." +fi + # The value of VORTEX_DB_IMAGE passed to the provisioning. An empty value # makes the provisioning import the dump file. provision_db_image="${db_image}" @@ -160,6 +195,12 @@ else docker compose exec "${env_args[@]}" -e "VORTEX_DB_IMAGE=${provision_db_image}" -e VORTEX_PROVISION_POST_OPERATIONS_SKIP=1 -T cli ./vendor/bin/vortex-provision + # The provisioning skips the post-provision operations, which include the + # sanitization, so the dump leaves the hosting and CI only after this step. + if [ "${refresh_store}" -eq 1 ]; then + docker compose exec "${env_args[@]}" -T cli ./vendor/drevops/vortex-tooling/src/vortex-provision-sanitize-db || fail "Unable to sanitize the database." + fi + # Database-in-image storage exports the image archive instead of the dump. if [ -n "${db_image}" ]; then rm -f "${db_file}" diff --git a/.vortex/tooling/tests/_helper.bash b/.vortex/tooling/tests/_helper.bash index 4cbded3356..61bb8a2d46 100644 --- a/.vortex/tooling/tests/_helper.bash +++ b/.vortex/tooling/tests/_helper.bash @@ -121,6 +121,7 @@ setup() { unset VORTEX_DB_IMAGE unset VORTEX_FETCH_DB_FORCE unset VORTEX_EXPORT_DB_CONTAINER_REGISTRY_PUSH_PROCEED + unset VORTEX_PUSH_DB_PROCEED # Isolate ambient credentials from the scripts under test. Scripts fall back # to these unprefixed values whenever their own variable is unset or empty, diff --git a/.vortex/tooling/tests/unit/fetch-db-gdrive.bats b/.vortex/tooling/tests/unit/fetch-db-gdrive.bats new file mode 100644 index 0000000000..863221bf25 --- /dev/null +++ b/.vortex/tooling/tests/unit/fetch-db-gdrive.bats @@ -0,0 +1,293 @@ +#!/usr/bin/env bats +## +# Unit tests for the 'vortex-fetch-db-gdrive' script. +# +# shellcheck disable=SC2016,SC2030,SC2031,SC2034 + +load ../_helper.bash + +# Writes a service account key file with a newly generated private key. +fixture_gdrive_key() { + openssl genpkey -algorithm RSA -pkeyopt rsa_keygen_bits:2048 -out "${BUILD_DIR}/gdrive.pem" 2>/dev/null + php -r 'print json_encode(["type" => "service_account", "client_email" => "vortex@example.iam.gserviceaccount.com", "private_key" => file_get_contents($argv[1])]);' "${BUILD_DIR}/gdrive.pem" >"${BUILD_DIR}/gdrive.json" +} + +@test "fetch-db-gdrive: Fetch database file successfully with a key file" { + pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 + + rm -rf .data + fixture_gdrive_key + + unset VORTEX_GDRIVE_SERVICE_ACCOUNT_KEY + export VORTEX_FETCH_DB_GDRIVE_SERVICE_ACCOUNT_KEY="${BUILD_DIR}/gdrive.json" + export VORTEX_FETCH_DB_GDRIVE_FOLDER_ID="folder-123" + export VORTEX_FETCH_DB_GDRIVE_DB_DIR=".data" + export VORTEX_FETCH_DB_GDRIVE_DB_FILE="db.sql" + + mock_curl=$(mock_command "curl") + mock_set_output "${mock_curl}" '{"access_token":"test-token","expires_in":3599}' 1 + mock_set_output "${mock_curl}" '{"files":[{"id":"file-456"}]}' 2 + mock_set_side_effect "${mock_curl}" "echo 'CREATE TABLE test (id INT);' > .data/db.sql" 3 + + run .vortex/tooling/src/vortex-fetch-db-gdrive + assert_success + assert_output_contains "[INFO] Started database dump fetch from Google Drive." + assert_output_contains "Remote file: db.sql" + assert_output_contains "Local path: .data/db.sql" + assert_output_contains "Folder ID: folder-123" + assert_output_contains "[ OK ] Authenticated with the service account." + assert_output_contains "[ OK ] Found the database dump file with ID file-456." + assert_output_contains "[ OK ] Downloaded the database dump file." + assert_output_contains "[ OK ] Finished database dump fetch from Google Drive." + assert_output_not_contains "test-token" + + assert_equal "3" "$(mock_get_call_num "${mock_curl}")" + + # The token request carries a signed JWT assertion for the read-only scope. + token_args="$(mock_get_call_args "${mock_curl}" 1)" + assert_string_contains "${token_args}" "https://oauth2.googleapis.com/token" + assert_string_contains "${token_args}" "grant_type=urn:ietf:params:oauth:grant-type:jwt-bearer" + assertion="${token_args##*assertion=}" + assertion="${assertion%% *}" + IFS='.' read -r jwt_header jwt_claims jwt_signature <<<"${assertion}" + assert_not_empty "${jwt_signature}" + claims="$(php -r 'print base64_decode(strtr($argv[1], "-_", "+/"));' "${jwt_claims}")" + assert_string_contains "${claims}" '"iss":"vortex@example.iam.gserviceaccount.com"' + assert_string_contains "${claims}" '"scope":"https:\/\/www.googleapis.com\/auth\/drive.readonly"' + + list_args="$(mock_get_call_args "${mock_curl}" 2)" + assert_string_contains "${list_args}" "https://www.googleapis.com/drive/v3/files" + assert_string_contains "${list_args}" "Authorization: Bearer test-token" + assert_string_contains "${list_args}" "q=name = 'db.sql' and 'folder-123' in parents and trashed = false" + assert_string_contains "${list_args}" "supportsAllDrives=true" + + download_args="$(mock_get_call_args "${mock_curl}" 3)" + assert_string_contains "${download_args}" "Authorization: Bearer test-token" + assert_string_contains "${download_args}" "-o .data/db.sql" + assert_string_contains "${download_args}" "https://www.googleapis.com/drive/v3/files/file-456?alt=media&supportsAllDrives=true" + + assert_file_contains ".data/db.sql" "CREATE TABLE test" + + rm -rf .data + + popd >/dev/null +} + +@test "fetch-db-gdrive: Fetch database file with the key content in the shared variable" { + pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 + + rm -rf .data + fixture_gdrive_key + + export VORTEX_FETCH_DB_GDRIVE_SERVICE_ACCOUNT_KEY="" + VORTEX_GDRIVE_SERVICE_ACCOUNT_KEY="$(cat "${BUILD_DIR}/gdrive.json")" + export VORTEX_GDRIVE_SERVICE_ACCOUNT_KEY + export VORTEX_FETCH_DB_GDRIVE_FOLDER_ID="folder-123" + unset VORTEX_FETCH_DB_GDRIVE_DB_DIR VORTEX_FETCH_DB_GDRIVE_DB_FILE + + mock_curl=$(mock_command "curl") + mock_set_output "${mock_curl}" '{"access_token":"test-token"}' 1 + mock_set_output "${mock_curl}" '{"files":[{"id":"file-456"}]}' 2 + + run .vortex/tooling/src/vortex-fetch-db-gdrive + assert_success + # The default directory and file name come from .env. + assert_output_contains "Local path: ./.data/db.sql" + assert_output_contains "[ OK ] Finished database dump fetch from Google Drive." + + assert_string_contains "$(mock_get_call_args "${mock_curl}" 3)" "-o ./.data/db.sql" + + rm -rf .data + + popd >/dev/null +} + +@test "fetch-db-gdrive: Resolve indexed variables with VORTEX_DB_INDEX" { + pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 + + rm -rf .data + fixture_gdrive_key + + unset VORTEX_GDRIVE_SERVICE_ACCOUNT_KEY + export VORTEX_DB_INDEX=3 + export VORTEX_FETCH_DB3_GDRIVE_SERVICE_ACCOUNT_KEY="${BUILD_DIR}/gdrive.json" + export VORTEX_FETCH_DB3_GDRIVE_FOLDER_ID="folder-789" + export VORTEX_FETCH_DB3_FILE="db3.sql" + + mock_curl=$(mock_command "curl") + mock_set_output "${mock_curl}" '{"access_token":"test-token"}' 1 + mock_set_output "${mock_curl}" '{"files":[{"id":"file-3"}]}' 2 + + run .vortex/tooling/src/vortex-fetch-db-gdrive + assert_success + assert_output_contains "Remote file: db3.sql" + assert_output_contains "Folder ID: folder-789" + + assert_string_contains "$(mock_get_call_args "${mock_curl}" 2)" "q=name = 'db3.sql' and 'folder-789' in parents and trashed = false" + + rm -rf .data + + popd >/dev/null +} + +@test "fetch-db-gdrive: Escape a quote in the file name of the folder query" { + pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 + + rm -rf .data + fixture_gdrive_key + + export VORTEX_FETCH_DB_GDRIVE_SERVICE_ACCOUNT_KEY="${BUILD_DIR}/gdrive.json" + export VORTEX_FETCH_DB_GDRIVE_FOLDER_ID="folder-123" + export VORTEX_FETCH_DB_GDRIVE_DB_FILE="site's.sql" + + mock_curl=$(mock_command "curl") + mock_set_output "${mock_curl}" '{"access_token":"test-token"}' 1 + mock_set_output "${mock_curl}" '{"files":[{"id":"file-456"}]}' 2 + + run .vortex/tooling/src/vortex-fetch-db-gdrive + assert_success + + assert_string_contains "$(mock_get_call_args "${mock_curl}" 2)" "q=name = 'site\\'s.sql' and" + + rm -rf .data + + popd >/dev/null +} + +@test "fetch-db-gdrive: Fail when the service account key is missing" { + pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 + + export VORTEX_FETCH_DB_GDRIVE_SERVICE_ACCOUNT_KEY="" + export VORTEX_GDRIVE_SERVICE_ACCOUNT_KEY="" + export VORTEX_FETCH_DB_GDRIVE_FOLDER_ID="folder-123" + + mock_curl=$(mock_command "curl") + + run .vortex/tooling/src/vortex-fetch-db-gdrive + assert_failure + assert_output_contains "[FAIL] Missing required value for VORTEX_FETCH_DB_GDRIVE_SERVICE_ACCOUNT_KEY or VORTEX_GDRIVE_SERVICE_ACCOUNT_KEY." + assert_equal "0" "$(mock_get_call_num "${mock_curl}")" + + popd >/dev/null +} + +@test "fetch-db-gdrive: Fail when the folder ID is missing" { + pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 + + export VORTEX_FETCH_DB_GDRIVE_SERVICE_ACCOUNT_KEY="{}" + export VORTEX_FETCH_DB_GDRIVE_FOLDER_ID="" + + mock_curl=$(mock_command "curl") + + run .vortex/tooling/src/vortex-fetch-db-gdrive + assert_failure + assert_output_contains "[FAIL] Missing required value for VORTEX_FETCH_DB_GDRIVE_FOLDER_ID." + assert_equal "0" "$(mock_get_call_num "${mock_curl}")" + + popd >/dev/null +} + +@test "fetch-db-gdrive: Fail when the service account key is invalid" { + pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 + + export VORTEX_FETCH_DB_GDRIVE_FOLDER_ID="folder-123" + + mock_curl=$(mock_command "curl") + + for key in "not json" '{"client_email":"vortex@example.iam.gserviceaccount.com"}' '{"client_email":"vortex@example.iam.gserviceaccount.com","private_key":"not a key"}'; do + export VORTEX_FETCH_DB_GDRIVE_SERVICE_ACCOUNT_KEY="${key}" + + run .vortex/tooling/src/vortex-fetch-db-gdrive + assert_failure + assert_output_contains "[FAIL] Unable to authenticate with the service account. Check that VORTEX_FETCH_DB_GDRIVE_SERVICE_ACCOUNT_KEY or VORTEX_GDRIVE_SERVICE_ACCOUNT_KEY is a valid service account JSON key." + done + + assert_equal "0" "$(mock_get_call_num "${mock_curl}")" + + popd >/dev/null +} + +@test "fetch-db-gdrive: Fail when the token request is rejected" { + pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 + + fixture_gdrive_key + + export VORTEX_FETCH_DB_GDRIVE_SERVICE_ACCOUNT_KEY="${BUILD_DIR}/gdrive.json" + export VORTEX_FETCH_DB_GDRIVE_FOLDER_ID="folder-123" + + mock_curl=$(mock_command "curl") + mock_set_output "${mock_curl}" '{"error":"invalid_grant","error_description":"Invalid JWT Signature."}' 1 + + run .vortex/tooling/src/vortex-fetch-db-gdrive + assert_failure + assert_output_contains "[FAIL] Unable to authenticate with the service account." + assert_output_not_contains "Found the database dump file" + assert_equal "1" "$(mock_get_call_num "${mock_curl}")" + + popd >/dev/null +} + +@test "fetch-db-gdrive: Fail when the folder cannot be listed" { + pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 + + fixture_gdrive_key + + export VORTEX_FETCH_DB_GDRIVE_SERVICE_ACCOUNT_KEY="${BUILD_DIR}/gdrive.json" + export VORTEX_FETCH_DB_GDRIVE_FOLDER_ID="folder-123" + + mock_curl=$(mock_command "curl") + mock_set_output "${mock_curl}" '{"access_token":"test-token"}' 1 + mock_set_output "${mock_curl}" '{"error":{"code":404,"message":"File not found: folder-123."}}' 2 + + run .vortex/tooling/src/vortex-fetch-db-gdrive + assert_failure + assert_output_contains "[FAIL] Unable to list the folder-123 folder. Check that the folder ID is correct and that the folder is shared with the service account." + assert_equal "2" "$(mock_get_call_num "${mock_curl}")" + + popd >/dev/null +} + +@test "fetch-db-gdrive: Fail when the folder has no database dump file" { + pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 + + fixture_gdrive_key + + export VORTEX_FETCH_DB_GDRIVE_SERVICE_ACCOUNT_KEY="${BUILD_DIR}/gdrive.json" + export VORTEX_FETCH_DB_GDRIVE_FOLDER_ID="folder-123" + export VORTEX_FETCH_DB_GDRIVE_DB_FILE="db.sql" + + mock_curl=$(mock_command "curl") + mock_set_output "${mock_curl}" '{"access_token":"test-token"}' 1 + mock_set_output "${mock_curl}" '{"files":[]}' 2 + + run .vortex/tooling/src/vortex-fetch-db-gdrive + assert_failure + assert_output_contains "[FAIL] Unable to find the db.sql file in the folder-123 folder." + assert_equal "2" "$(mock_get_call_num "${mock_curl}")" + + popd >/dev/null +} + +@test "fetch-db-gdrive: Fail when the download fails" { + pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 + + rm -rf .data + fixture_gdrive_key + + export VORTEX_FETCH_DB_GDRIVE_SERVICE_ACCOUNT_KEY="${BUILD_DIR}/gdrive.json" + export VORTEX_FETCH_DB_GDRIVE_FOLDER_ID="folder-123" + + mock_curl=$(mock_command "curl") + mock_set_output "${mock_curl}" '{"access_token":"test-token"}' 1 + mock_set_output "${mock_curl}" '{"files":[{"id":"file-456"}]}' 2 + mock_set_status "${mock_curl}" 22 3 + + run .vortex/tooling/src/vortex-fetch-db-gdrive + assert_failure + assert_output_contains "[FAIL] Unable to download the database dump file." + assert_output_not_contains "Finished database dump fetch from Google Drive." + + rm -rf .data + + popd >/dev/null +} diff --git a/.vortex/tooling/tests/unit/fetch-db.bats b/.vortex/tooling/tests/unit/fetch-db.bats index fec06732f3..c274716c30 100644 --- a/.vortex/tooling/tests/unit/fetch-db.bats +++ b/.vortex/tooling/tests/unit/fetch-db.bats @@ -14,6 +14,7 @@ stub_fetch_scripts() { stub_sibling "vortex-fetch-db-lagoon" "Fetched from lagoon." stub_sibling "vortex-fetch-db-container-registry" "Fetched from container_registry." stub_sibling "vortex-fetch-db-s3" "Fetched from s3." + stub_sibling "vortex-fetch-db-gdrive" "Fetched from gdrive." } @test "fetch-db: Fetch with URL source" { @@ -307,7 +308,7 @@ EOF export VORTEX_FETCH_DB_PROCEED="1" - for db_source in url ftp acquia lagoon container_registry s3; do + for db_source in url ftp acquia lagoon container_registry s3 gdrive; do export VORTEX_FETCH_DB_SOURCE="${db_source}" run .vortex/tooling/src/vortex-fetch-db @@ -315,7 +316,7 @@ EOF assert_output_contains "Fetched from ${db_source}." assert_output_contains "Finished database fetch." - for other_source in url ftp acquia lagoon container_registry s3; do + for other_source in url ftp acquia lagoon container_registry s3 gdrive; do if [ "${other_source}" != "${db_source}" ]; then assert_output_not_contains "Fetched from ${other_source}." fi @@ -357,7 +358,7 @@ EOF run .vortex/tooling/src/vortex-fetch-db assert_failure_status 1 assert_output_contains "Started database fetch." - assert_output_contains 'Unsupported database source "VORTEX_CONTAINER_REGISTRY" in VORTEX_FETCH_DB_SOURCE. Use one of: url, ftp, acquia, lagoon, container_registry, s3, none.' + assert_output_contains 'Unsupported database source "VORTEX_CONTAINER_REGISTRY" in VORTEX_FETCH_DB_SOURCE. Use one of: url, ftp, acquia, lagoon, container_registry, s3, gdrive, none.' assert_output_not_contains "Fetched from" assert_output_not_contains "Finished database fetch." assert_file_not_exists "${BUILD_DIR}/fetch-db-fresh" @@ -397,7 +398,7 @@ EOF run .vortex/tooling/src/vortex-fetch-db assert_failure_status 1 assert_output_contains "Started database 2 fetch." - assert_output_contains 'Unsupported database source "sftp" in VORTEX_FETCH_DB2_SOURCE. Use one of: url, ftp, acquia, lagoon, container_registry, s3, none.' + assert_output_contains 'Unsupported database source "sftp" in VORTEX_FETCH_DB2_SOURCE. Use one of: url, ftp, acquia, lagoon, container_registry, s3, gdrive, none.' assert_output_not_contains "Finished database 2 fetch." popd >/dev/null diff --git a/.vortex/tooling/tests/unit/push-db-gdrive.bats b/.vortex/tooling/tests/unit/push-db-gdrive.bats new file mode 100644 index 0000000000..0d7e6b4cc6 --- /dev/null +++ b/.vortex/tooling/tests/unit/push-db-gdrive.bats @@ -0,0 +1,297 @@ +#!/usr/bin/env bats +## +# Unit tests for the 'vortex-push-db-gdrive' script. +# +# shellcheck disable=SC2016,SC2030,SC2031,SC2034 + +load ../_helper.bash + +# Writes a service account key file with a newly generated private key. +fixture_gdrive_key() { + openssl genpkey -algorithm RSA -pkeyopt rsa_keygen_bits:2048 -out "${BUILD_DIR}/gdrive.pem" 2>/dev/null + php -r 'print json_encode(["type" => "service_account", "client_email" => "vortex@example.iam.gserviceaccount.com", "private_key" => file_get_contents($argv[1])]);' "${BUILD_DIR}/gdrive.pem" >"${BUILD_DIR}/gdrive.json" +} + +@test "push-db-gdrive: Create the database dump file in the folder" { + pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 + + mkdir -p .data + echo "CREATE TABLE test (id INT);" >.data/db.sql + fixture_gdrive_key + + unset VORTEX_GDRIVE_SERVICE_ACCOUNT_KEY + export VORTEX_PUSH_DB_GDRIVE_SERVICE_ACCOUNT_KEY="${BUILD_DIR}/gdrive.json" + export VORTEX_PUSH_DB_GDRIVE_FOLDER_ID="folder-123" + export VORTEX_PUSH_DB_GDRIVE_DB_DIR=".data" + export VORTEX_PUSH_DB_GDRIVE_DB_FILE="db.sql" + + mock_curl=$(mock_command "curl") + mock_set_output "${mock_curl}" '{"access_token":"test-token"}' 1 + mock_set_output "${mock_curl}" '{"files":[]}' 2 + mock_set_output "${mock_curl}" $'HTTP/2 200\r\nlocation: https://www.googleapis.com/upload/drive/v3/files?uploadType=resumable&upload_id=session-1\r\ncontent-length: 0\r\n' 3 + mock_set_output "${mock_curl}" $'{"id":"file-new"}\n200' 4 + + run .vortex/tooling/src/vortex-push-db-gdrive + assert_success + assert_output_contains "[INFO] Started database dump push to Google Drive." + assert_output_contains "Local file: .data/db.sql" + assert_output_contains "Remote file: db.sql" + assert_output_contains "Folder ID: folder-123" + assert_output_contains "[ OK ] Authenticated with the service account." + assert_output_contains "Creating a new file in the folder." + assert_output_contains "[ OK ] Started the upload session." + assert_output_contains "[ OK ] Uploaded the database dump file." + assert_output_contains "[ OK ] Finished database dump push to Google Drive." + assert_output_not_contains "test-token" + assert_output_not_contains "session-1" + + assert_equal "4" "$(mock_get_call_num "${mock_curl}")" + + # The token request carries a signed JWT assertion for the full Drive scope. + token_args="$(mock_get_call_args "${mock_curl}" 1)" + assertion="${token_args##*assertion=}" + assertion="${assertion%% *}" + IFS='.' read -r jwt_header jwt_claims jwt_signature <<<"${assertion}" + assert_not_empty "${jwt_signature}" + claims="$(php -r 'print base64_decode(strtr($argv[1], "-_", "+/"));' "${jwt_claims}")" + assert_string_contains "${claims}" '"scope":"https:\/\/www.googleapis.com\/auth\/drive"' + + assert_string_contains "$(mock_get_call_args "${mock_curl}" 2)" "q=name = 'db.sql' and 'folder-123' in parents and trashed = false" + + session_args="$(mock_get_call_args "${mock_curl}" 3)" + assert_string_contains "${session_args}" "-X POST" + assert_string_contains "${session_args}" "Authorization: Bearer test-token" + assert_string_contains "${session_args}" '--data {"name":"db.sql","parents":["folder-123"]}' + assert_string_contains "${session_args}" "https://www.googleapis.com/upload/drive/v3/files?uploadType=resumable&supportsAllDrives=true" + + upload_args="$(mock_get_call_args "${mock_curl}" 4)" + assert_string_contains "${upload_args}" "-X PUT --upload-file .data/db.sql" + assert_string_contains "${upload_args}" "https://www.googleapis.com/upload/drive/v3/files?uploadType=resumable&upload_id=session-1" + + rm -rf .data + + popd >/dev/null +} + +@test "push-db-gdrive: Replace the existing database dump file in place" { + pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 + + mkdir -p .data + echo "database content" >.data/db.sql + fixture_gdrive_key + + export VORTEX_PUSH_DB_GDRIVE_SERVICE_ACCOUNT_KEY="" + VORTEX_GDRIVE_SERVICE_ACCOUNT_KEY="$(cat "${BUILD_DIR}/gdrive.json")" + export VORTEX_GDRIVE_SERVICE_ACCOUNT_KEY + # The folder falls back to the folder the database is fetched from. + export VORTEX_PUSH_DB_GDRIVE_FOLDER_ID="" + export VORTEX_FETCH_DB_GDRIVE_FOLDER_ID="fetch-folder" + unset VORTEX_PUSH_DB_GDRIVE_DB_DIR VORTEX_PUSH_DB_GDRIVE_DB_FILE + + mock_curl=$(mock_command "curl") + mock_set_output "${mock_curl}" '{"access_token":"test-token"}' 1 + mock_set_output "${mock_curl}" '{"files":[{"id":"file-456"}]}' 2 + mock_set_output "${mock_curl}" $'HTTP/2 200\r\nLocation: https://www.googleapis.com/upload/drive/v3/files/file-456?uploadType=resumable&upload_id=session-2\r\n' 3 + mock_set_output "${mock_curl}" $'{"id":"file-456"}\n200' 4 + + run .vortex/tooling/src/vortex-push-db-gdrive + assert_success + assert_output_contains "Local file: ./.data/db.sql" + assert_output_contains "Folder ID: fetch-folder" + assert_output_contains "Replacing the existing file with ID file-456." + assert_output_contains "[ OK ] Finished database dump push to Google Drive." + + session_args="$(mock_get_call_args "${mock_curl}" 3)" + assert_string_contains "${session_args}" "-X PATCH" + assert_string_contains "${session_args}" "--data {}" + assert_string_contains "${session_args}" "https://www.googleapis.com/upload/drive/v3/files/file-456?uploadType=resumable&supportsAllDrives=true" + + assert_string_contains "$(mock_get_call_args "${mock_curl}" 4)" "upload_id=session-2" + + rm -rf .data + + popd >/dev/null +} + +@test "push-db-gdrive: Fail when the service account key is missing" { + pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 + + export VORTEX_PUSH_DB_GDRIVE_SERVICE_ACCOUNT_KEY="" + export VORTEX_GDRIVE_SERVICE_ACCOUNT_KEY="" + export VORTEX_PUSH_DB_GDRIVE_FOLDER_ID="folder-123" + + mock_curl=$(mock_command "curl") + + run .vortex/tooling/src/vortex-push-db-gdrive + assert_failure + assert_output_contains "[FAIL] Missing required value for VORTEX_PUSH_DB_GDRIVE_SERVICE_ACCOUNT_KEY or VORTEX_GDRIVE_SERVICE_ACCOUNT_KEY." + assert_equal "0" "$(mock_get_call_num "${mock_curl}")" + + popd >/dev/null +} + +@test "push-db-gdrive: Fail when the folder ID is missing" { + pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 + + export VORTEX_PUSH_DB_GDRIVE_SERVICE_ACCOUNT_KEY="{}" + export VORTEX_PUSH_DB_GDRIVE_FOLDER_ID="" + export VORTEX_FETCH_DB_GDRIVE_FOLDER_ID="" + + mock_curl=$(mock_command "curl") + + run .vortex/tooling/src/vortex-push-db-gdrive + assert_failure + assert_output_contains "[FAIL] Missing required value for VORTEX_PUSH_DB_GDRIVE_FOLDER_ID or VORTEX_FETCH_DB_GDRIVE_FOLDER_ID." + assert_equal "0" "$(mock_get_call_num "${mock_curl}")" + + popd >/dev/null +} + +@test "push-db-gdrive: Fail when the database dump file does not exist" { + pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 + + rm -rf .data + + export VORTEX_PUSH_DB_GDRIVE_SERVICE_ACCOUNT_KEY="{}" + export VORTEX_PUSH_DB_GDRIVE_FOLDER_ID="folder-123" + export VORTEX_PUSH_DB_GDRIVE_DB_DIR=".data" + export VORTEX_PUSH_DB_GDRIVE_DB_FILE="db.sql" + + mock_curl=$(mock_command "curl") + + run .vortex/tooling/src/vortex-push-db-gdrive + assert_failure + assert_output_contains "[FAIL] Database dump file .data/db.sql does not exist." + assert_output_not_contains "Started database dump push to Google Drive." + assert_equal "0" "$(mock_get_call_num "${mock_curl}")" + + popd >/dev/null +} + +@test "push-db-gdrive: Fail when the service account key is invalid" { + pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 + + mkdir -p .data + echo "database content" >.data/db.sql + + export VORTEX_PUSH_DB_GDRIVE_SERVICE_ACCOUNT_KEY="not json" + export VORTEX_PUSH_DB_GDRIVE_FOLDER_ID="folder-123" + export VORTEX_PUSH_DB_GDRIVE_DB_DIR=".data" + + mock_curl=$(mock_command "curl") + + run .vortex/tooling/src/vortex-push-db-gdrive + assert_failure + assert_output_contains "[FAIL] Unable to authenticate with the service account. Check that VORTEX_PUSH_DB_GDRIVE_SERVICE_ACCOUNT_KEY or VORTEX_GDRIVE_SERVICE_ACCOUNT_KEY is a valid service account JSON key." + assert_equal "0" "$(mock_get_call_num "${mock_curl}")" + + rm -rf .data + + popd >/dev/null +} + +@test "push-db-gdrive: Fail without creating a file when the folder cannot be listed" { + pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 + + mkdir -p .data + echo "database content" >.data/db.sql + fixture_gdrive_key + + export VORTEX_PUSH_DB_GDRIVE_SERVICE_ACCOUNT_KEY="${BUILD_DIR}/gdrive.json" + export VORTEX_PUSH_DB_GDRIVE_FOLDER_ID="folder-123" + export VORTEX_PUSH_DB_GDRIVE_DB_DIR=".data" + + mock_curl=$(mock_command "curl") + mock_set_output "${mock_curl}" '{"access_token":"test-token"}' 1 + mock_set_output "${mock_curl}" '{"error":{"code":403,"message":"Insufficient permissions."}}' 2 + + run .vortex/tooling/src/vortex-push-db-gdrive + assert_failure + assert_output_contains "[FAIL] Unable to list the folder-123 folder. Check that the folder ID is correct and that the folder is shared with the service account." + assert_output_not_contains "Creating a new file in the folder." + assert_equal "2" "$(mock_get_call_num "${mock_curl}")" + + rm -rf .data + + popd >/dev/null +} + +@test "push-db-gdrive: Fail when the upload session has no location" { + pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 + + mkdir -p .data + echo "database content" >.data/db.sql + fixture_gdrive_key + + export VORTEX_PUSH_DB_GDRIVE_SERVICE_ACCOUNT_KEY="${BUILD_DIR}/gdrive.json" + export VORTEX_PUSH_DB_GDRIVE_FOLDER_ID="folder-123" + export VORTEX_PUSH_DB_GDRIVE_DB_DIR=".data" + + mock_curl=$(mock_command "curl") + mock_set_output "${mock_curl}" '{"access_token":"test-token"}' 1 + mock_set_output "${mock_curl}" '{"files":[]}' 2 + mock_set_output "${mock_curl}" $'HTTP/2 403\r\ncontent-type: application/json\r\n' 3 + + run .vortex/tooling/src/vortex-push-db-gdrive + assert_failure + assert_output_contains "[FAIL] Unable to start the upload session: the response has no upload location. Check that the folder is in a shared drive and that the service account can edit it." + assert_equal "3" "$(mock_get_call_num "${mock_curl}")" + + rm -rf .data + + popd >/dev/null +} + +@test "push-db-gdrive: Fail when the upload returns a non-2xx status" { + pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 + + mkdir -p .data + echo "database content" >.data/db.sql + fixture_gdrive_key + + export VORTEX_PUSH_DB_GDRIVE_SERVICE_ACCOUNT_KEY="${BUILD_DIR}/gdrive.json" + export VORTEX_PUSH_DB_GDRIVE_FOLDER_ID="folder-123" + export VORTEX_PUSH_DB_GDRIVE_DB_DIR=".data" + + mock_curl=$(mock_command "curl") + mock_set_output "${mock_curl}" '{"access_token":"test-token"}' 1 + mock_set_output "${mock_curl}" '{"files":[]}' 2 + mock_set_output "${mock_curl}" $'HTTP/2 200\r\nlocation: https://www.googleapis.com/upload/drive/v3/files?upload_id=session-3\r\n' 3 + mock_set_output "${mock_curl}" $'{"error":{"code":403,"message":"The user does not have sufficient permissions."}}\n403' 4 + + run .vortex/tooling/src/vortex-push-db-gdrive + assert_failure + assert_output_contains "The user does not have sufficient permissions." + assert_output_contains "[FAIL] Unable to push to Google Drive: HTTP status 403." + assert_output_not_contains "Finished database dump push to Google Drive." + + rm -rf .data + + popd >/dev/null +} + +@test "push-db-gdrive: Fail when the upload request fails" { + pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 + + mkdir -p .data + echo "database content" >.data/db.sql + fixture_gdrive_key + + export VORTEX_PUSH_DB_GDRIVE_SERVICE_ACCOUNT_KEY="${BUILD_DIR}/gdrive.json" + export VORTEX_PUSH_DB_GDRIVE_FOLDER_ID="folder-123" + export VORTEX_PUSH_DB_GDRIVE_DB_DIR=".data" + + mock_curl=$(mock_command "curl") + mock_set_output "${mock_curl}" '{"access_token":"test-token"}' 1 + mock_set_output "${mock_curl}" '{"files":[]}' 2 + mock_set_output "${mock_curl}" $'HTTP/2 200\r\nlocation: https://www.googleapis.com/upload/drive/v3/files?upload_id=session-4\r\n' 3 + mock_set_status "${mock_curl}" 28 4 + + run .vortex/tooling/src/vortex-push-db-gdrive + assert_failure + assert_output_contains "[FAIL] Google Drive upload request failed with curl exit 28." + + rm -rf .data + + popd >/dev/null +} diff --git a/.vortex/tooling/tests/unit/push-db-s3.bats b/.vortex/tooling/tests/unit/push-db-s3.bats index 7f8f96d9c5..6c7ccbe3d3 100644 --- a/.vortex/tooling/tests/unit/push-db-s3.bats +++ b/.vortex/tooling/tests/unit/push-db-s3.bats @@ -28,6 +28,7 @@ load ../_helper.bash # Clear shortcut variables to prevent environment leakage. unset S3_ACCESS_KEY S3_SECRET_KEY S3_BUCKET S3_REGION S3_PREFIX + export VORTEX_FETCH_DB_S3_ACCESS_KEY="" VORTEX_FETCH_DB_S3_SECRET_KEY="" VORTEX_FETCH_DB_S3_BUCKET="" VORTEX_FETCH_DB_S3_REGION="" VORTEX_FETCH_DB_S3_PREFIX="" export VORTEX_PUSH_DB_S3_ACCESS_KEY="testaccesskey" export VORTEX_PUSH_DB_S3_SECRET_KEY="testsecretkey" @@ -67,6 +68,7 @@ load ../_helper.bash # Clear shortcut variables to prevent environment leakage. unset S3_ACCESS_KEY S3_SECRET_KEY S3_BUCKET S3_REGION S3_PREFIX + export VORTEX_FETCH_DB_S3_ACCESS_KEY="" VORTEX_FETCH_DB_S3_SECRET_KEY="" VORTEX_FETCH_DB_S3_BUCKET="" VORTEX_FETCH_DB_S3_REGION="" VORTEX_FETCH_DB_S3_PREFIX="" export VORTEX_PUSH_DB_S3_ACCESS_KEY="testaccesskey" export VORTEX_PUSH_DB_S3_SECRET_KEY="testsecretkey" @@ -115,6 +117,7 @@ load ../_helper.bash export S3_REGION="us-east-1" # Clear prefix shortcut to prevent environment leakage. unset S3_PREFIX + export VORTEX_FETCH_DB_S3_ACCESS_KEY="" VORTEX_FETCH_DB_S3_SECRET_KEY="" VORTEX_FETCH_DB_S3_BUCKET="" VORTEX_FETCH_DB_S3_REGION="" VORTEX_FETCH_DB_S3_PREFIX="" export VORTEX_PUSH_DB_S3_DB_DIR=".data" export VORTEX_PUSH_DB_S3_DB_FILE="db.sql" @@ -147,6 +150,7 @@ load ../_helper.bash # Clear shortcut variables to prevent environment leakage. unset S3_ACCESS_KEY S3_SECRET_KEY S3_BUCKET S3_REGION S3_PREFIX + export VORTEX_FETCH_DB_S3_ACCESS_KEY="" VORTEX_FETCH_DB_S3_SECRET_KEY="" VORTEX_FETCH_DB_S3_BUCKET="" VORTEX_FETCH_DB_S3_REGION="" VORTEX_FETCH_DB_S3_PREFIX="" export VORTEX_PUSH_DB_S3_ACCESS_KEY="testaccesskey" export VORTEX_PUSH_DB_S3_SECRET_KEY="testsecretkey" @@ -186,6 +190,7 @@ load ../_helper.bash # Clear shortcut variables to prevent environment leakage. unset S3_ACCESS_KEY S3_SECRET_KEY S3_BUCKET S3_REGION S3_PREFIX + export VORTEX_FETCH_DB_S3_ACCESS_KEY="" VORTEX_FETCH_DB_S3_SECRET_KEY="" VORTEX_FETCH_DB_S3_BUCKET="" VORTEX_FETCH_DB_S3_REGION="" VORTEX_FETCH_DB_S3_PREFIX="" export VORTEX_PUSH_DB_S3_ACCESS_KEY="testaccesskey" export VORTEX_PUSH_DB_S3_SECRET_KEY="testsecretkey" @@ -225,6 +230,7 @@ load ../_helper.bash # Clear shortcut variables to prevent environment leakage. unset S3_ACCESS_KEY S3_SECRET_KEY S3_BUCKET S3_REGION S3_PREFIX + export VORTEX_FETCH_DB_S3_ACCESS_KEY="" VORTEX_FETCH_DB_S3_SECRET_KEY="" VORTEX_FETCH_DB_S3_BUCKET="" VORTEX_FETCH_DB_S3_REGION="" VORTEX_FETCH_DB_S3_PREFIX="" export VORTEX_PUSH_DB_S3_ACCESS_KEY="testaccesskey" export VORTEX_PUSH_DB_S3_SECRET_KEY="testsecretkey" @@ -263,6 +269,7 @@ load ../_helper.bash # Clear shortcut variables to prevent environment leakage. unset S3_ACCESS_KEY S3_SECRET_KEY S3_BUCKET S3_REGION S3_PREFIX + export VORTEX_FETCH_DB_S3_ACCESS_KEY="" VORTEX_FETCH_DB_S3_SECRET_KEY="" VORTEX_FETCH_DB_S3_BUCKET="" VORTEX_FETCH_DB_S3_REGION="" VORTEX_FETCH_DB_S3_PREFIX="" export VORTEX_PUSH_DB_S3_ACCESS_KEY="testaccesskey" export VORTEX_PUSH_DB_S3_SECRET_KEY="testsecretkey" @@ -292,6 +299,7 @@ load ../_helper.bash # Clear shortcut variables to prevent environment leakage. unset S3_ACCESS_KEY S3_SECRET_KEY S3_BUCKET S3_REGION S3_PREFIX + export VORTEX_FETCH_DB_S3_ACCESS_KEY="" VORTEX_FETCH_DB_S3_SECRET_KEY="" VORTEX_FETCH_DB_S3_BUCKET="" VORTEX_FETCH_DB_S3_REGION="" VORTEX_FETCH_DB_S3_PREFIX="" export VORTEX_PUSH_DB_S3_ACCESS_KEY="" export VORTEX_PUSH_DB_S3_SECRET_KEY="testsecretkey" @@ -317,6 +325,7 @@ load ../_helper.bash # Clear shortcut variables to prevent environment leakage. unset S3_ACCESS_KEY S3_SECRET_KEY S3_BUCKET S3_REGION S3_PREFIX + export VORTEX_FETCH_DB_S3_ACCESS_KEY="" VORTEX_FETCH_DB_S3_SECRET_KEY="" VORTEX_FETCH_DB_S3_BUCKET="" VORTEX_FETCH_DB_S3_REGION="" VORTEX_FETCH_DB_S3_PREFIX="" export VORTEX_PUSH_DB_S3_ACCESS_KEY="testaccesskey" export VORTEX_PUSH_DB_S3_SECRET_KEY="" @@ -342,6 +351,7 @@ load ../_helper.bash # Clear shortcut variables to prevent environment leakage. unset S3_ACCESS_KEY S3_SECRET_KEY S3_BUCKET S3_REGION S3_PREFIX + export VORTEX_FETCH_DB_S3_ACCESS_KEY="" VORTEX_FETCH_DB_S3_SECRET_KEY="" VORTEX_FETCH_DB_S3_BUCKET="" VORTEX_FETCH_DB_S3_REGION="" VORTEX_FETCH_DB_S3_PREFIX="" export VORTEX_PUSH_DB_S3_ACCESS_KEY="testaccesskey" export VORTEX_PUSH_DB_S3_SECRET_KEY="testsecretkey" @@ -367,6 +377,7 @@ load ../_helper.bash # Clear shortcut variables to prevent environment leakage. unset S3_ACCESS_KEY S3_SECRET_KEY S3_BUCKET S3_REGION S3_PREFIX + export VORTEX_FETCH_DB_S3_ACCESS_KEY="" VORTEX_FETCH_DB_S3_SECRET_KEY="" VORTEX_FETCH_DB_S3_BUCKET="" VORTEX_FETCH_DB_S3_REGION="" VORTEX_FETCH_DB_S3_PREFIX="" export VORTEX_PUSH_DB_S3_ACCESS_KEY="testaccesskey" export VORTEX_PUSH_DB_S3_SECRET_KEY="testsecretkey" @@ -395,6 +406,7 @@ load ../_helper.bash # Clear shortcut variables to prevent environment leakage. unset S3_ACCESS_KEY S3_SECRET_KEY S3_BUCKET S3_REGION S3_PREFIX + export VORTEX_FETCH_DB_S3_ACCESS_KEY="" VORTEX_FETCH_DB_S3_SECRET_KEY="" VORTEX_FETCH_DB_S3_BUCKET="" VORTEX_FETCH_DB_S3_REGION="" VORTEX_FETCH_DB_S3_PREFIX="" export VORTEX_PUSH_DB_S3_ACCESS_KEY="testaccesskey" export VORTEX_PUSH_DB_S3_SECRET_KEY="testsecretkey" @@ -411,3 +423,51 @@ load ../_helper.bash popd >/dev/null } + +@test "push-db-s3: Use the database fetch values" { + pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 + + mkdir -p .data + echo "database content" >.data/db2.sql + + declare -a STEPS=( + "[INFO] Started database dump push to S3." + + "Local file: .data/db2.sql" + "Remote file: store/db2.sql" + "S3 bucket: fetch-bucket" + "S3 region: eu-west-1" + "S3 prefix: store/" + + "@curl * # 0 # 200" + + "[ OK ] Finished database dump push to S3." + ) + + unset S3_ACCESS_KEY S3_SECRET_KEY S3_BUCKET S3_REGION S3_PREFIX + + export VORTEX_PUSH_DB_S3_ACCESS_KEY="" + export VORTEX_PUSH_DB_S3_SECRET_KEY="" + export VORTEX_PUSH_DB_S3_BUCKET="" + export VORTEX_PUSH_DB_S3_REGION="" + export VORTEX_PUSH_DB_S3_PREFIX="" + export VORTEX_FETCH_DB_S3_ACCESS_KEY="fetch-access-key" + export VORTEX_FETCH_DB_S3_SECRET_KEY="fetch-secret-key" + export VORTEX_FETCH_DB_S3_BUCKET="fetch-bucket" + export VORTEX_FETCH_DB_S3_REGION="eu-west-1" + export VORTEX_FETCH_DB_S3_PREFIX="store" + export VORTEX_PUSH_DB_S3_DB_DIR=".data" + # The remote file name follows the dump file name. + export VORTEX_PUSH_DB_S3_DB_FILE="db2.sql" + unset VORTEX_PUSH_DB_S3_REMOTE_FILE + + mocks="$(steps_run "setup")" + run .vortex/tooling/src/vortex-push-db-s3 + steps_run "assert" "${mocks[@]}" + + assert_success + + rm -rf .data + + popd >/dev/null +} diff --git a/.vortex/tooling/tests/unit/push-db.bats b/.vortex/tooling/tests/unit/push-db.bats new file mode 100644 index 0000000000..d2dee534f0 --- /dev/null +++ b/.vortex/tooling/tests/unit/push-db.bats @@ -0,0 +1,125 @@ +#!/usr/bin/env bats +## +# Unit tests for the 'vortex-push-db' script. +# +# shellcheck disable=SC2030,SC2031 + +load ../_helper.bash + +@test "push-db: Skip when the push is not requested" { + pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 + + stub_sibling "vortex-push-db-gdrive" "Pushed to gdrive." + + export VORTEX_FETCH_DB_SOURCE="gdrive" + + for proceed in "" 0; do + export VORTEX_PUSH_DB_PROCEED="${proceed}" + + run .vortex/tooling/src/vortex-push-db + assert_success + assert_output_contains "[INFO] Started database push." + assert_output_contains "[ OK ] Skipped database push as VORTEX_PUSH_DB_PROCEED is not set to 1." + assert_output_not_contains "Pushed to gdrive." + assert_output_not_contains "Finished database push." + done + + popd >/dev/null +} + +@test "push-db: Route each dump store source to its push script" { + pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 + + stub_sibling "vortex-push-db-gdrive" "Pushed to gdrive." + stub_sibling "vortex-push-db-s3" "Pushed to s3." + + export VORTEX_PUSH_DB_PROCEED=1 + + for db_source in gdrive s3; do + export VORTEX_FETCH_DB_SOURCE="${db_source}" + + run .vortex/tooling/src/vortex-push-db + assert_success + assert_output_contains "Pushed to ${db_source}." + assert_output_contains "[ OK ] Finished database push." + + for other_source in gdrive s3; do + [ "${other_source}" = "${db_source}" ] && continue + assert_output_not_contains "Pushed to ${other_source}." + done + done + + popd >/dev/null +} + +@test "push-db: Skip for a source that does not accept a push" { + pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 + + stub_sibling "vortex-push-db-gdrive" "Pushed to gdrive." + stub_sibling "vortex-push-db-s3" "Pushed to s3." + + export VORTEX_PUSH_DB_PROCEED=1 + + for db_source in url ftp acquia lagoon container_registry none; do + export VORTEX_FETCH_DB_SOURCE="${db_source}" + + run .vortex/tooling/src/vortex-push-db + assert_success + assert_output_contains "[ OK ] Skipped database push as the \"${db_source}\" database source in VORTEX_FETCH_DB_SOURCE does not accept a push. Use one of: gdrive, s3." + assert_output_not_contains "Pushed to" + assert_output_not_contains "Finished database push." + done + + popd >/dev/null +} + +@test "push-db: Use the source from .env by default" { + pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 + + stub_sibling "vortex-push-db-gdrive" "Pushed to gdrive." + + unset VORTEX_FETCH_DB_SOURCE + export VORTEX_PUSH_DB_PROCEED=1 + + run .vortex/tooling/src/vortex-push-db + assert_success + assert_output_contains '[ OK ] Skipped database push as the "url" database source in VORTEX_FETCH_DB_SOURCE does not accept a push. Use one of: gdrive, s3.' + assert_output_not_contains "Pushed to gdrive." + + popd >/dev/null +} + +@test "push-db: Refuse to push a database that is not sanitized" { + pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 + + stub_sibling "vortex-push-db-gdrive" "Pushed to gdrive." + + export VORTEX_FETCH_DB_SOURCE="gdrive" + export VORTEX_PUSH_DB_PROCEED=1 + export VORTEX_PROVISION_SANITIZE_DB_SKIP=1 + + run .vortex/tooling/src/vortex-push-db + assert_failure + assert_output_contains "[FAIL] Refused to push the database as VORTEX_PROVISION_SANITIZE_DB_SKIP is set to 1 and the database dump is not sanitized." + assert_output_not_contains "Pushed to gdrive." + assert_output_not_contains "Finished database push." + + popd >/dev/null +} + +@test "push-db: Fail when the push script fails" { + pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 + + printf '#!/usr/bin/env bash\necho "Failed to push to s3."\nexit 1\n' >.vortex/tooling/src/vortex-push-db-s3 + chmod +x .vortex/tooling/src/vortex-push-db-s3 + + export VORTEX_FETCH_DB_SOURCE="s3" + export VORTEX_PUSH_DB_PROCEED=1 + + run .vortex/tooling/src/vortex-push-db + assert_failure + assert_output_contains "Failed to push to s3." + assert_output_not_contains "Finished database push." + + popd >/dev/null +} diff --git a/.vortex/tooling/tests/unit/refresh-db.bats b/.vortex/tooling/tests/unit/refresh-db.bats index 091c08d083..f81a93af38 100644 --- a/.vortex/tooling/tests/unit/refresh-db.bats +++ b/.vortex/tooling/tests/unit/refresh-db.bats @@ -236,7 +236,7 @@ load ../_helper.bash run .vortex/tooling/src/vortex-refresh-db assert_failure - assert_output_contains "[FAIL] The myorg/db image is built from a database dump file, but VORTEX_REFRESH_DB_SOURCE is \"${refresh_source}\". Set it to the source of the dump file: url, ftp, acquia, lagoon or s3." + assert_output_contains "[FAIL] The myorg/db image is built from a database dump file, but VORTEX_REFRESH_DB_SOURCE is \"${refresh_source}\". Set it to the source of the dump file: url, ftp, acquia, lagoon, s3 or gdrive." assert_output_not_contains "Logged in to the registry." assert_output_not_contains "Fetched the dump file." done @@ -246,6 +246,184 @@ load ../_helper.bash popd >/dev/null } +@test "refresh-db: Fetches the dump file from the refresh source for a dump store source with a push requested" { + pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 + + unset VORTEX_DB_IMAGE VORTEX_DB_IMAGE_BASE + export VORTEX_PUSH_DB_PROCEED=1 + export VORTEX_REFRESH_DB_SOURCE="lagoon" + touch "${BUILD_DIR}/bash_env" + export BASH_ENV="${BUILD_DIR}/bash_env" + + stub_sibling "vortex-login-container-registry" "Logged in to the registry." + stub_sibling "vortex-export-db" 'Exported the database to ${1}.' + printf '#!/usr/bin/env bash\nmkdir -p .data\necho upstream >.data/db.sql\necho "Fetched from ${VORTEX_FETCH_DB_SOURCE} with force ${VORTEX_FETCH_DB_FORCE} and BASH_ENV [${BASH_ENV}]."\n' >.vortex/tooling/src/vortex-fetch-db + chmod +x .vortex/tooling/src/vortex-fetch-db + + export VORTEX_FETCH_DB_SOURCE="gdrive" + mkdir -p .data + echo "store" >.data/db.sql + + mock_command "sleep" >/dev/null + mock_docker=$(mock_command "docker") + mock_set_output "${mock_docker}" "Started the stack." 1 + mock_set_output "${mock_docker}" "Created the directory." 2 + mock_set_output "${mock_docker}" "Copied the dump file." 3 + mock_set_output "${mock_docker}" "Provisioned the site." 4 + mock_set_output "${mock_docker}" "Sanitized the database." 5 + mock_set_output "${mock_docker}" "Copied the dump file back." 6 + + run .vortex/tooling/src/vortex-refresh-db + assert_success + assert_output_contains "Fetched from lagoon with force 1 and BASH_ENV []." + assert_output_contains "[ OK ] Copied database dump file ./.data/db.sql into the cli container." + assert_output_contains "Sanitized the database." + assert_output_contains "Exported the database to db.sql." + assert_output_contains "[ OK ] Finished database refresh." + + # The store holds no database image, so the provisioning imports the dump. + assert_string_contains "$(mock_get_call_args "${mock_docker}" 4)" "-e VORTEX_DB_IMAGE= -e VORTEX_PROVISION_POST_OPERATIONS_SKIP=1 -T cli ./vendor/bin/vortex-provision" + # The provisioning skips the sanitization, so it runs before the export. + assert_string_contains "$(mock_get_call_args "${mock_docker}" 5)" "-T cli ./vendor/drevops/vortex-tooling/src/vortex-provision-sanitize-db" + assert_string_contains "$(mock_get_call_args "${mock_docker}" 6)" "compose --progress quiet cp -L cli:/app/.data/db.sql ./.data/db.sql" + assert_equal "6" "$(mock_get_call_num "${mock_docker}")" + + assert_file_contains .data/db.sql "upstream" + + popd >/dev/null +} + +@test "refresh-db: Provisions from the dump store without the refresh source when a push is not requested" { + pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 + + mkdir -p .data + echo "store" >.data/db.sql + + unset VORTEX_DB_IMAGE VORTEX_DB_IMAGE_BASE VORTEX_PUSH_DB_PROCEED VORTEX_REFRESH_DB_SOURCE + export VORTEX_FETCH_DB_SOURCE="gdrive" + + stub_sibling "vortex-login-container-registry" "Logged in to the registry." + stub_sibling "vortex-fetch-db" "Fetched the dump file." + stub_sibling "vortex-export-db" 'Exported the database to ${1}.' + + mock_command "sleep" >/dev/null + mock_docker=$(mock_command "docker") + + run .vortex/tooling/src/vortex-refresh-db + assert_success + assert_output_not_contains "Fetched the dump file." + assert_output_contains "Exported the database to db.sql." + assert_output_contains "[ OK ] Finished database refresh." + + assert_equal "5" "$(mock_get_call_num "${mock_docker}")" + assert_file_contains .data/db.sql "store" + + popd >/dev/null +} + +@test "refresh-db: Fails without an upstream source for a dump store source with a push requested" { + pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 + + unset VORTEX_DB_IMAGE VORTEX_DB_IMAGE_BASE + export VORTEX_FETCH_DB_SOURCE="gdrive" + export VORTEX_PUSH_DB_PROCEED=1 + + stub_sibling "vortex-login-container-registry" "Logged in to the registry." + stub_sibling "vortex-fetch-db" "Fetched the dump file." + + mock_docker=$(mock_command "docker") + + for refresh_source in "" gdrive s3 container_registry none; do + export VORTEX_REFRESH_DB_SOURCE="${refresh_source}" + + run .vortex/tooling/src/vortex-refresh-db + assert_failure + assert_output_contains "[FAIL] The database dump file pushed to gdrive is fetched from an upstream source, but VORTEX_REFRESH_DB_SOURCE is \"${refresh_source}\". Set it to the source of the dump file: url, ftp, acquia or lagoon." + assert_output_not_contains "Logged in to the registry." + assert_output_not_contains "Fetched the dump file." + done + + assert_equal "0" "$(mock_get_call_num "${mock_docker}")" + + popd >/dev/null +} + +@test "refresh-db: Fails when the sanitization is skipped for a dump store source with a push requested" { + pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 + + unset VORTEX_DB_IMAGE VORTEX_DB_IMAGE_BASE + export VORTEX_FETCH_DB_SOURCE="gdrive" + export VORTEX_PUSH_DB_PROCEED=1 + export VORTEX_REFRESH_DB_SOURCE="lagoon" + export VORTEX_PROVISION_SANITIZE_DB_SKIP=1 + + stub_sibling "vortex-login-container-registry" "Logged in to the registry." + stub_sibling "vortex-fetch-db" "Fetched the dump file." + + mock_docker=$(mock_command "docker") + + run .vortex/tooling/src/vortex-refresh-db + assert_failure + assert_output_contains "[FAIL] The database dump file pushed to gdrive must be sanitized, but VORTEX_PROVISION_SANITIZE_DB_SKIP is set to 1." + assert_output_not_contains "Fetched the dump file." + assert_equal "0" "$(mock_get_call_num "${mock_docker}")" + + popd >/dev/null +} + +@test "refresh-db: Fails when the sanitization fails for a dump store source" { + pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 + + unset VORTEX_DB_IMAGE VORTEX_DB_IMAGE_BASE + export VORTEX_FETCH_DB_SOURCE="gdrive" + export VORTEX_PUSH_DB_PROCEED=1 + export VORTEX_REFRESH_DB_SOURCE="url" + + stub_sibling "vortex-login-container-registry" "Logged in to the registry." + stub_sibling "vortex-export-db" 'Exported the database to ${1}.' + printf '#!/usr/bin/env bash\nmkdir -p .data\necho upstream >.data/db.sql\n' >.vortex/tooling/src/vortex-fetch-db + chmod +x .vortex/tooling/src/vortex-fetch-db + + mock_command "sleep" >/dev/null + mock_docker=$(mock_command "docker") + mock_set_status "${mock_docker}" 1 5 + + run .vortex/tooling/src/vortex-refresh-db + assert_failure + assert_output_contains "[FAIL] Unable to sanitize the database." + assert_output_not_contains "Exported the database to db.sql." + + popd >/dev/null +} + +@test "refresh-db: Fails when the refresh source does not fetch the dump file for a dump store source" { + pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 + + mkdir -p .data + echo "store" >.data/db.sql + + unset VORTEX_DB_IMAGE VORTEX_DB_IMAGE_BASE + export VORTEX_FETCH_DB_SOURCE="s3" + export VORTEX_PUSH_DB_PROCEED=1 + export VORTEX_REFRESH_DB_SOURCE="url" + + stub_sibling "vortex-login-container-registry" "Logged in to the registry." + stub_sibling "vortex-fetch-db" "Fetched the dump file elsewhere." + + mock_docker=$(mock_command "docker") + + run .vortex/tooling/src/vortex-refresh-db + assert_failure + assert_output_contains "Fetched the dump file elsewhere." + assert_output_contains "[FAIL] The database dump file was not fetched to ./.data/db.sql. Remove the output directory and file name overrides of the database fetch source." + + # The stale dump from the store is never provisioned or pushed. + assert_file_not_exists .data/db.sql + assert_equal "0" "$(mock_get_call_num "${mock_docker}")" + + popd >/dev/null +} + @test "refresh-db: Fails when the exported dump file cannot be copied from the cli container" { pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 From 105eecedb40b40bbb21d899068e7cef7af5ff3b9 Mon Sep 17 00:00:00 2001 From: Alex Skrypnyk Date: Wed, 11 Nov 2026 10:25:21 +1100 Subject: [PATCH 02/21] [#154] Fixed mock expectations in the Google Drive and refresh tests. --- .circleci/config.yml | 16 ++++++++ .env | 37 ++++++++++++++++--- .env.local.example | 8 ++++ .github/workflows/build-test-deploy.yml | 22 +++++++++++ .../Prompts/Handlers/DatabaseFetchSource.php | 10 +++++ .../Prompts/Handlers/MigrationFetchSource.php | 4 ++ .../DatabaseFetchSourceHandlerProcessTest.php | 3 ++ ...MigrationFetchSourceHandlerProcessTest.php | 14 +++++++ .../tooling/tests/unit/fetch-db-gdrive.bats | 3 ++ .vortex/tooling/tests/unit/refresh-db.bats | 4 ++ 10 files changed, 115 insertions(+), 6 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 9a39ac1d10..025c05723d 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -363,6 +363,18 @@ jobs: no_output_timeout: 30m #;> DB_FETCH_SOURCE_CONTAINER_REGISTRY + #;< DB_PUSH + # Push the freshly-exported sanitized database dump to the database fetch + # source. No-op unless a fresh DB was fetched (the fetch semaphore) and the + # push was requested (VORTEX_PUSH_DB_PROCEED, in the script). + - run: + name: Push DB + command: | + [ ! -f /tmp/fetch-db-fresh ] && echo "==> No freshly fetched database. DB push will not proceed." && exit 0 + ./vendor/bin/vortex-push-db + no_output_timeout: 30m + #;> DB_PUSH + - save_cache: # Save cache per default branch and the timestamp. # The cache will not be saved if it already exists. @@ -387,6 +399,10 @@ jobs: # Deploy container image. VORTEX_EXPORT_DB_CONTAINER_REGISTRY_PUSH_PROCEED: 1 #;> DB_IN_IMAGE + #;< DB_PUSH + # Refresh the database dump in the database fetch source. + VORTEX_PUSH_DB_PROCEED: 1 + #;> DB_PUSH # Do not build the Drupal front-end. VORTEX_FRONTEND_BUILD_SKIP: 1 #;> !PROVISION_TYPE_PROFILE diff --git a/.env b/.env index 94f358f542..eaa76a2e98 100644 --- a/.env +++ b/.env @@ -222,9 +222,9 @@ VORTEX_FETCH_DB_SOURCE=url # VORTEX_DB_IMAGE=your_org/your_site:latest # Source of the database dump file the nightly CI job imports into a clean -# base image to build the database container image: url, ftp, acquia, lagoon -# or s3. The fetch script of that source reads its own variables, such as -# VORTEX_FETCH_DB_URL for url. Add them, and keep secrets in CI. +# base image to build the database container image: url, ftp, acquia, lagoon, +# s3 or gdrive. The fetch script of that source reads its own variables, such +# as VORTEX_FETCH_DB_URL for url. Add them, and keep secrets in CI. # VORTEX_REFRESH_DB_SOURCE=url #;> DB_FETCH_SOURCE_CONTAINER_REGISTRY @@ -258,6 +258,24 @@ VORTEX_FETCH_DB_S3_BUCKET= VORTEX_FETCH_DB_S3_REGION=ap-southeast-2 #;> DB_FETCH_SOURCE_S3 +#;< DB_FETCH_SOURCE_GDRIVE +# Database dump file sourced from Google Drive. + +# Google Drive folder ID with the database dump file. +# +# The ID is the last segment of the folder URL. The folder must be in a shared +# drive and shared with the service account. +VORTEX_FETCH_DB_GDRIVE_FOLDER_ID= +#;> DB_FETCH_SOURCE_GDRIVE + +#;< DB_PUSH +# Source of the database dump file the nightly CI job sanitizes and pushes to +# the database fetch source: url, ftp, acquia or lagoon. The fetch script of +# that source reads its own variables, such as VORTEX_FETCH_DB_URL for url. +# Add them, and keep secrets in CI. +# VORTEX_REFRESH_DB_SOURCE=url +#;> DB_PUSH + # Environment to fetch the database from. # # Applies to hosting environments. @@ -297,9 +315,9 @@ VORTEX_FETCH_DB2_SOURCE=url # Source of the second database dump file the nightly CI job imports into a # clean base image to rebuild the migration database container image: url, -# ftp, acquia, lagoon or s3. The fetch script of that source reads its own -# variables, such as VORTEX_FETCH_DB2_URL for url. Add them, and keep secrets -# in CI. +# ftp, acquia, lagoon, s3 or gdrive. The fetch script of that source reads its +# own variables, such as VORTEX_FETCH_DB2_URL for url. Add them, and keep +# secrets in CI. # VORTEX_REFRESH_DB2_SOURCE=url #;> MIGRATION_DB_FETCH_SOURCE_CONTAINER_REGISTRY @@ -333,6 +351,13 @@ VORTEX_FETCH_DB2_S3_BUCKET= VORTEX_FETCH_DB2_S3_REGION=ap-southeast-2 #;> MIGRATION_DB_FETCH_SOURCE_S3 +#;< MIGRATION_DB_FETCH_SOURCE_GDRIVE +# Second database dump file sourced from Google Drive. + +# Google Drive folder ID with the second database dump file. +VORTEX_FETCH_DB2_GDRIVE_FOLDER_ID= +#;> MIGRATION_DB_FETCH_SOURCE_GDRIVE + # Environment to fetch the second database from. # # Applies to hosting environments. diff --git a/.env.local.example b/.env.local.example index b7a2eb4d0c..e876e08dd8 100644 --- a/.env.local.example +++ b/.env.local.example @@ -69,6 +69,14 @@ VORTEX_FETCH_DB_S3_ACCESS_KEY= VORTEX_FETCH_DB_S3_SECRET_KEY= #;> DB_FETCH_SOURCE_S3 +#;< DB_FETCH_SOURCE_GDRIVE +# Database dump file sourced from Google Drive. +# Service account key: Google Cloud console -> IAM & Admin -> Service accounts -> Keys -> Add key + +# Google Cloud service account key: the JSON content or a path to the JSON key file. +VORTEX_GDRIVE_SERVICE_ACCOUNT_KEY= +#;> DB_FETCH_SOURCE_GDRIVE + #;< DB_FETCH_SOURCE_CONTAINER_REGISTRY # Database image sourced from a container registry. diff --git a/.github/workflows/build-test-deploy.yml b/.github/workflows/build-test-deploy.yml index b9daa0d69d..f876f1867b 100644 --- a/.github/workflows/build-test-deploy.yml +++ b/.github/workflows/build-test-deploy.yml @@ -225,6 +225,13 @@ jobs: VORTEX_CI_DB_CACHE_FALLBACK: "yes" # Which branch to use as a source of DB caches. VORTEX_CI_DB_CACHE_BRANCH: "develop" + #;< DB_FETCH_SOURCE_S3 + VORTEX_FETCH_DB_S3_ACCESS_KEY: ${{ secrets.VORTEX_FETCH_DB_S3_ACCESS_KEY }} + VORTEX_FETCH_DB_S3_SECRET_KEY: ${{ secrets.VORTEX_FETCH_DB_S3_SECRET_KEY }} + #;> DB_FETCH_SOURCE_S3 + #;< DB_FETCH_SOURCE_GDRIVE + VORTEX_GDRIVE_SERVICE_ACCOUNT_KEY: ${{ secrets.VORTEX_GDRIVE_SERVICE_ACCOUNT_KEY }} + #;> DB_FETCH_SOURCE_GDRIVE steps: # Frees disk space by removing preinstalled toolchains unused by this project. Steps run in a container, so the host is reached via the Docker socket. @@ -281,6 +288,10 @@ jobs: # Deploy container image. echo "VORTEX_EXPORT_DB_CONTAINER_REGISTRY_PUSH_PROCEED=1" #;> DB_IN_IMAGE + #;< DB_PUSH + # Refresh the database dump in the database fetch source. + echo "VORTEX_PUSH_DB_PROCEED=1" + #;> DB_PUSH } >> "$GITHUB_ENV" - name: Create cache keys files for database caching @@ -343,6 +354,17 @@ jobs: timeout-minutes: 30 #;> DB_FETCH_SOURCE_CONTAINER_REGISTRY + #;< DB_PUSH + # Push the freshly-exported sanitized database dump to the database fetch + # source. No-op unless a fresh DB was fetched (the fetch semaphore) and the + # push was requested (VORTEX_PUSH_DB_PROCEED, in the script). + - name: Push DB + run: | + if [ ! -f /tmp/fetch-db-fresh ]; then echo "==> No freshly fetched database. DB push will not proceed."; exit 0; fi + ./vendor/bin/vortex-push-db + timeout-minutes: 30 + #;> DB_PUSH + # Save cache per default branch and the timestamp. # The cache will not be saved if it already exists. # Note that the cache fallback flag is enabled for this case in order diff --git a/.vortex/installer/src/Prompts/Handlers/DatabaseFetchSource.php b/.vortex/installer/src/Prompts/Handlers/DatabaseFetchSource.php index 57c5f1841a..f8e78d3d35 100644 --- a/.vortex/installer/src/Prompts/Handlers/DatabaseFetchSource.php +++ b/.vortex/installer/src/Prompts/Handlers/DatabaseFetchSource.php @@ -21,6 +21,8 @@ class DatabaseFetchSource extends AbstractHandler { const S3 = 's3'; + const GDRIVE = 'gdrive'; + const NONE = 'none'; /** @@ -55,6 +57,7 @@ public function options(array $responses): ?array { self::LAGOON => 'Lagoon environment', self::CONTAINER_REGISTRY => 'Container registry', self::S3 => 'S3 bucket', + self::GDRIVE => 'Google Drive', self::NONE => 'None', ]; @@ -129,6 +132,7 @@ public function process(): void { self::LAGOON, self::CONTAINER_REGISTRY, self::S3, + self::GDRIVE, ]; foreach ($types as $type) { @@ -147,6 +151,12 @@ public function process(): void { if ($v !== self::CONTAINER_REGISTRY && !$is_migration_in_image) { File::removeTokenAsync('DB_IN_IMAGE'); } + + // DB_PUSH wraps the nightly push of the sanitized dump, which only a + // dump store source accepts. + if (!in_array($v, [self::S3, self::GDRIVE], TRUE)) { + File::removeTokenAsync('DB_PUSH'); + } } } diff --git a/.vortex/installer/src/Prompts/Handlers/MigrationFetchSource.php b/.vortex/installer/src/Prompts/Handlers/MigrationFetchSource.php index 8c87b6a1ed..636cdc11ce 100644 --- a/.vortex/installer/src/Prompts/Handlers/MigrationFetchSource.php +++ b/.vortex/installer/src/Prompts/Handlers/MigrationFetchSource.php @@ -21,6 +21,8 @@ class MigrationFetchSource extends AbstractHandler { const S3 = 's3'; + const GDRIVE = 'gdrive'; + /** * {@inheritdoc} */ @@ -53,6 +55,7 @@ public function options(array $responses): ?array { self::LAGOON => 'Lagoon environment', self::CONTAINER_REGISTRY => 'Container registry', self::S3 => 'S3 bucket', + self::GDRIVE => 'Google Drive', ]; if (isset($responses[HostingProvider::id()])) { @@ -130,6 +133,7 @@ public function process(): void { self::LAGOON, self::CONTAINER_REGISTRY, self::S3, + self::GDRIVE, ]; foreach ($types as $type) { diff --git a/.vortex/installer/tests/Functional/Prompts/Handlers/DatabaseFetchSourceHandlerProcessTest.php b/.vortex/installer/tests/Functional/Prompts/Handlers/DatabaseFetchSourceHandlerProcessTest.php index 9ca89d1241..0b9cd9d62b 100644 --- a/.vortex/installer/tests/Functional/Prompts/Handlers/DatabaseFetchSourceHandlerProcessTest.php +++ b/.vortex/installer/tests/Functional/Prompts/Handlers/DatabaseFetchSourceHandlerProcessTest.php @@ -36,6 +36,9 @@ public static function dataProviderHandlerProcess(): \Iterator { yield 'db_fetch_source_s3' => [ static::cw(fn(AbstractHandlerProcessTestCase $test): string => $test->prompts[DatabaseFetchSource::id()] = DatabaseFetchSource::S3), ]; + yield 'db_fetch_source_gdrive' => [ + static::cw(fn(AbstractHandlerProcessTestCase $test): string => $test->prompts[DatabaseFetchSource::id()] = DatabaseFetchSource::GDRIVE), + ]; } } diff --git a/.vortex/installer/tests/Functional/Prompts/Handlers/MigrationFetchSourceHandlerProcessTest.php b/.vortex/installer/tests/Functional/Prompts/Handlers/MigrationFetchSourceHandlerProcessTest.php index 0c63da7a4c..e410c67712 100644 --- a/.vortex/installer/tests/Functional/Prompts/Handlers/MigrationFetchSourceHandlerProcessTest.php +++ b/.vortex/installer/tests/Functional/Prompts/Handlers/MigrationFetchSourceHandlerProcessTest.php @@ -77,6 +77,20 @@ public static function dataProviderHandlerProcess(): \Iterator { $test->assertFileNotContainsString(static::$sut . '/.env', 'VORTEX_FETCH_DB2_FTP_HOST'); }), ]; + yield 'migration_fetch_source_gdrive' => [ + static::cw(function (AbstractHandlerProcessTestCase $test): void { + $test->prompts[Migration::id()] = TRUE; + $test->prompts[MigrationFetchSource::id()] = MigrationFetchSource::GDRIVE; + }), + static::cw(function (AbstractHandlerProcessTestCase $test): void { + $test->assertFileContainsString(static::$sut . '/.env', 'VORTEX_FETCH_DB2_SOURCE=gdrive'); + $test->assertFileContainsString(static::$sut . '/.env', 'VORTEX_FETCH_DB2_GDRIVE_FOLDER_ID='); + $test->assertFileNotContainsString(static::$sut . '/.env', 'VORTEX_FETCH_DB2_URL='); + $test->assertFileNotContainsString(static::$sut . '/.env', 'VORTEX_FETCH_DB2_S3_BUCKET'); + // The migration database is fetched from the store but never pushed. + $test->assertFileNotContainsString(static::$sut . '/.github/workflows/build-test-deploy.yml', 'vortex-push-db'); + }), + ]; yield 'migration_fetch_source_container_registry' => [ static::cw(function (AbstractHandlerProcessTestCase $test): void { $test->prompts[Migration::id()] = TRUE; diff --git a/.vortex/tooling/tests/unit/fetch-db-gdrive.bats b/.vortex/tooling/tests/unit/fetch-db-gdrive.bats index 863221bf25..987005923e 100644 --- a/.vortex/tooling/tests/unit/fetch-db-gdrive.bats +++ b/.vortex/tooling/tests/unit/fetch-db-gdrive.bats @@ -88,6 +88,7 @@ fixture_gdrive_key() { mock_curl=$(mock_command "curl") mock_set_output "${mock_curl}" '{"access_token":"test-token"}' 1 mock_set_output "${mock_curl}" '{"files":[{"id":"file-456"}]}' 2 + mock_set_status "${mock_curl}" 0 3 run .vortex/tooling/src/vortex-fetch-db-gdrive assert_success @@ -117,6 +118,7 @@ fixture_gdrive_key() { mock_curl=$(mock_command "curl") mock_set_output "${mock_curl}" '{"access_token":"test-token"}' 1 mock_set_output "${mock_curl}" '{"files":[{"id":"file-3"}]}' 2 + mock_set_status "${mock_curl}" 0 3 run .vortex/tooling/src/vortex-fetch-db-gdrive assert_success @@ -143,6 +145,7 @@ fixture_gdrive_key() { mock_curl=$(mock_command "curl") mock_set_output "${mock_curl}" '{"access_token":"test-token"}' 1 mock_set_output "${mock_curl}" '{"files":[{"id":"file-456"}]}' 2 + mock_set_status "${mock_curl}" 0 3 run .vortex/tooling/src/vortex-fetch-db-gdrive assert_success diff --git a/.vortex/tooling/tests/unit/refresh-db.bats b/.vortex/tooling/tests/unit/refresh-db.bats index f81a93af38..338b9a7e8a 100644 --- a/.vortex/tooling/tests/unit/refresh-db.bats +++ b/.vortex/tooling/tests/unit/refresh-db.bats @@ -386,6 +386,10 @@ load ../_helper.bash mock_command "sleep" >/dev/null mock_docker=$(mock_command "docker") + mock_set_output "${mock_docker}" "Started the stack." 1 + mock_set_output "${mock_docker}" "Created the directory." 2 + mock_set_output "${mock_docker}" "Copied the dump file." 3 + mock_set_output "${mock_docker}" "Provisioned the site." 4 mock_set_status "${mock_docker}" 1 5 run .vortex/tooling/src/vortex-refresh-db From 1f45d43bbfd4a233beee56e714dad2eda16d73f1 Mon Sep 17 00:00:00 2001 From: Alex Skrypnyk Date: Wed, 11 Nov 2026 10:31:49 +1100 Subject: [PATCH 03/21] [#154] Documented pushing the sanitized database to a Google Drive or S3 dump store. --- .circleci/vortex-test-common.yml | 12 ++++ .github/workflows/vortex-test-didi.yml | 44 +++++++++++++ .../content/continuous-integration/README.mdx | 53 ++++++++++++++- .../docs/content/development/provision.mdx | 5 ++ .../docs/content/development/variables.mdx | 17 ++++- .vortex/docs/content/features.mdx | 9 ++- .vortex/docs/cspell.json | 2 + .../tooling/playground/try-fetch-db-gdrive.sh | 57 ++++++++++++++++ .../tooling/playground/try-push-db-gdrive.sh | 65 +++++++++++++++++++ 9 files changed, 257 insertions(+), 7 deletions(-) create mode 100755 .vortex/tooling/playground/try-fetch-db-gdrive.sh create mode 100755 .vortex/tooling/playground/try-push-db-gdrive.sh diff --git a/.circleci/vortex-test-common.yml b/.circleci/vortex-test-common.yml index 371f8c610a..7433dfe36e 100644 --- a/.circleci/vortex-test-common.yml +++ b/.circleci/vortex-test-common.yml @@ -183,6 +183,18 @@ jobs: no_output_timeout: 30m #;> DB_FETCH_SOURCE_CONTAINER_REGISTRY + #;< DB_PUSH + # Push the freshly-exported sanitized database dump to the database fetch + # source. No-op unless a fresh DB was fetched (the fetch semaphore) and the + # push was requested (VORTEX_PUSH_DB_PROCEED, in the script). + - run: + name: Push DB + command: | + [ ! -f /tmp/fetch-db-fresh ] && echo "==> No freshly fetched database. DB push will not proceed." && exit 0 + ./vendor/bin/vortex-push-db + no_output_timeout: 30m + #;> DB_PUSH + - save_cache: # Save cache per default branch and the timestamp. # The cache will not be saved if it already exists. diff --git a/.github/workflows/vortex-test-didi.yml b/.github/workflows/vortex-test-didi.yml index 4e8807d301..4a8857b8ab 100644 --- a/.github/workflows/vortex-test-didi.yml +++ b/.github/workflows/vortex-test-didi.yml @@ -64,6 +64,13 @@ jobs: VORTEX_CI_DB_CACHE_FALLBACK: "yes" # Which branch to use as a source of DB caches. VORTEX_CI_DB_CACHE_BRANCH: vortex-dev-didi-fi + #;< DB_FETCH_SOURCE_S3 + VORTEX_FETCH_DB_S3_ACCESS_KEY: ${{ secrets.VORTEX_FETCH_DB_S3_ACCESS_KEY }} + VORTEX_FETCH_DB_S3_SECRET_KEY: ${{ secrets.VORTEX_FETCH_DB_S3_SECRET_KEY }} + #;> DB_FETCH_SOURCE_S3 + #;< DB_FETCH_SOURCE_GDRIVE + VORTEX_GDRIVE_SERVICE_ACCOUNT_KEY: ${{ secrets.VORTEX_GDRIVE_SERVICE_ACCOUNT_KEY }} + #;> DB_FETCH_SOURCE_GDRIVE VORTEX_FETCH_DB_SOURCE: url VORTEX_FETCH_DB_FORCE: 1 VORTEX_DB_IMAGE: drevops/vortex-dev-mariadb-drupal-data-demo-destination-11.x @@ -129,6 +136,10 @@ jobs: # Deploy container image. echo "VORTEX_EXPORT_DB_CONTAINER_REGISTRY_PUSH_PROCEED=1" #;> DB_IN_IMAGE + #;< DB_PUSH + # Refresh the database dump in the database fetch source. + echo "VORTEX_PUSH_DB_PROCEED=1" + #;> DB_PUSH } >> "$GITHUB_ENV" - name: Create cache keys files for database caching @@ -191,6 +202,17 @@ jobs: timeout-minutes: 30 #;> DB_FETCH_SOURCE_CONTAINER_REGISTRY + #;< DB_PUSH + # Push the freshly-exported sanitized database dump to the database fetch + # source. No-op unless a fresh DB was fetched (the fetch semaphore) and the + # push was requested (VORTEX_PUSH_DB_PROCEED, in the script). + - name: Push DB + run: | + if [ ! -f /tmp/fetch-db-fresh ]; then echo "==> No freshly fetched database. DB push will not proceed."; exit 0; fi + ./vendor/bin/vortex-push-db + timeout-minutes: 30 + #;> DB_PUSH + # Save cache per default branch and the timestamp. # The cache will not be saved if it already exists. # Note that the cache fallback flag is enabled for this case in order @@ -570,6 +592,13 @@ jobs: VORTEX_CI_DB_CACHE_FALLBACK: "yes" # Which branch to use as a source of DB caches. VORTEX_CI_DB_CACHE_BRANCH: vortex-dev-didi-ii + #;< DB_FETCH_SOURCE_S3 + VORTEX_FETCH_DB_S3_ACCESS_KEY: ${{ secrets.VORTEX_FETCH_DB_S3_ACCESS_KEY }} + VORTEX_FETCH_DB_S3_SECRET_KEY: ${{ secrets.VORTEX_FETCH_DB_S3_SECRET_KEY }} + #;> DB_FETCH_SOURCE_S3 + #;< DB_FETCH_SOURCE_GDRIVE + VORTEX_GDRIVE_SERVICE_ACCOUNT_KEY: ${{ secrets.VORTEX_GDRIVE_SERVICE_ACCOUNT_KEY }} + #;> DB_FETCH_SOURCE_GDRIVE VORTEX_FETCH_DB_SOURCE: container_registry VORTEX_FETCH_DB_FORCE: 1 VORTEX_DB_IMAGE: drevops/vortex-dev-mariadb-drupal-data-demo-destination-11.x @@ -634,6 +663,10 @@ jobs: # Deploy container image. echo "VORTEX_EXPORT_DB_CONTAINER_REGISTRY_PUSH_PROCEED=1" #;> DB_IN_IMAGE + #;< DB_PUSH + # Refresh the database dump in the database fetch source. + echo "VORTEX_PUSH_DB_PROCEED=1" + #;> DB_PUSH } >> "$GITHUB_ENV" - name: Create cache keys files for database caching @@ -715,6 +748,17 @@ jobs: timeout-minutes: 30 #;> DB_FETCH_SOURCE_CONTAINER_REGISTRY + #;< DB_PUSH + # Push the freshly-exported sanitized database dump to the database fetch + # source. No-op unless a fresh DB was fetched (the fetch semaphore) and the + # push was requested (VORTEX_PUSH_DB_PROCEED, in the script). + - name: Push DB + run: | + if [ ! -f /tmp/fetch-db-fresh ]; then echo "==> No freshly fetched database. DB push will not proceed."; exit 0; fi + ./vendor/bin/vortex-push-db + timeout-minutes: 30 + #;> DB_PUSH + # Save cache per default branch and the timestamp. # The cache will not be saved if it already exists. # Note that the cache fallback flag is enabled for this case in order diff --git a/.vortex/docs/content/continuous-integration/README.mdx b/.vortex/docs/content/continuous-integration/README.mdx index 75f6286b9f..a7b34f8b52 100644 --- a/.vortex/docs/content/continuous-integration/README.mdx +++ b/.vortex/docs/content/continuous-integration/README.mdx @@ -79,6 +79,9 @@ site from: - Caches the dump, so the `build` job and later runs skip the download - On the nightly run of a project that stores its database in a container image, pushes the refreshed image to the registry +- On the nightly run of a project with the `s3` or `gdrive` fetch source, + pushes a sanitized dump to that source (see [Push the sanitized database + to a dump store](#push-the-sanitized-database-to-a-dump-store)) The job exists only when the site provisions from a database dump. With the `profile` provision type, the pipeline has no `database` job and no nightly @@ -386,7 +389,7 @@ future **Vortex** update ships. The scheduled run builds a database container image from a dump file. With the `container_registry` fetch source, set `VORTEX_REFRESH_DB_SOURCE` to the -source of that file: `url`, `ftp`, `acquia`, `lagoon` or `s3`. +source of that file: `url`, `ftp`, `acquia`, `lagoon`, `s3` or `gdrive`. `VORTEX_REFRESH_DB_SOURCE` selects a fetch script, and each fetch script reads its own variables. The installer adds to `.env` the variables of the @@ -409,6 +412,54 @@ For the migration database, set `VORTEX_REFRESH_DB2_SOURCE` and the variables apply to both databases. See [Refresh the migration database image](../development/migrations.mdx#refresh-the-migration-database-image). +### Push the sanitized database to a dump store + +A dump store is an `s3` or `gdrive` fetch source that holds a sanitized +copy of your hosting database. Developers and branch builds fetch that +copy and need no hosting access. On the nightly run, the `database` job +fetches a dump from `VORTEX_REFRESH_DB_SOURCE` and sanitizes it. Its +**Push DB** step then uploads the dump to the fetch source. + +To refresh a dump store every night: + +1. In `.env`, set `VORTEX_FETCH_DB_SOURCE` to `s3` or `gdrive`. The + installer sets it and adds the variables of that source. +2. In `.env`, set `VORTEX_REFRESH_DB_SOURCE` to the hosting source: `url`, + `ftp`, `acquia` or `lagoon`. Add the variables of that source from the + table in [Dump file source for a database + image](#dump-file-source-for-a-database-image). +3. Add the credentials of the dump store as CI secrets: + + | Source | Secrets | + |---|---| + | `s3` | `VORTEX_FETCH_DB_S3_ACCESS_KEY` and `VORTEX_FETCH_DB_S3_SECRET_KEY` | + | `gdrive` | `VORTEX_GDRIVE_SERVICE_ACCOUNT_KEY`, with the content of the JSON key file | + + GitHub Actions passes these secrets to the `database` job. Add the + secrets of the hosting source to the `env` block of that job. + +4. After the next nightly run, confirm that its **Push DB** step ends with + `Finished database push.` + +The `s3` push writes to the bucket, region and prefix that the fetch +reads. The `gdrive` push replaces a file with the same name in place, so +the file keeps its ID and sharing. Both pushes stop with an error when +`VORTEX_PROVISION_SANITIZE_DB_SKIP` is `1`. + +The `gdrive` source signs in with a Google Cloud service account key. +Service accounts have no storage of their own, so the folder must be in a +shared drive. To set up the folder: + +1. In the Google Cloud console, enable the Google Drive API and create a + service account with a JSON key. +2. In a shared drive, create a folder and add the service account email + as a **Content manager**. +3. Copy the folder ID, the last segment of the folder URL, into + `VORTEX_FETCH_DB_GDRIVE_FOLDER_ID` in `.env`. +4. Give developers the key of a second service account with the + **Viewer** role. They set `VORTEX_GDRIVE_SERVICE_ACCOUNT_KEY` in + `.env.local` to the path of the key file or to its content. + ## Trigger conditions Both providers build on branch pushes, pull requests and tag pushes, plus diff --git a/.vortex/docs/content/development/provision.mdx b/.vortex/docs/content/development/provision.mdx index ec35e263e2..eecfbc5c06 100644 --- a/.vortex/docs/content/development/provision.mdx +++ b/.vortex/docs/content/development/provision.mdx @@ -375,10 +375,15 @@ following values: - `container_registry`: a database container image, pulled instead of a file - `s3`: a file in an S3 bucket +- `gdrive`: a file in a Google Drive folder - `none`: no source; you put the dump into `.data` yourself Any other value stops the command with an error that lists these values. +The nightly CI run can refresh an `s3` or `gdrive` source with a sanitized +dump from your hosting. See [Push the sanitized database to a dump +store](../continuous-integration/README.mdx#push-the-sanitized-database-to-a-dump-store). + Each source reads its own settings, which the [Variables reference](variables.mdx) lists. The hosting sources copy the environment named in `VORTEX_FETCH_DB_ENVIRONMENT`. `.env` ships with diff --git a/.vortex/docs/content/development/variables.mdx b/.vortex/docs/content/development/variables.mdx index 680f9ab2ac..ac37b24c5c 100644 --- a/.vortex/docs/content/development/variables.mdx +++ b/.vortex/docs/content/development/variables.mdx @@ -321,6 +321,7 @@ The list below is automatically generated with [Shellvar](https://github.com/ale | `VORTEX_FETCH_DB2_FTP_FILE` | Second database dump FTP file name. | `db2.sql` | `.env` | | `VORTEX_FETCH_DB2_FTP_HOST` | Second database dump FTP host. | `UNDEFINED` | `.env` | | `VORTEX_FETCH_DB2_FTP_PORT` | Second database dump FTP port. | `21` | `.env` | +| `VORTEX_FETCH_DB2_GDRIVE_FOLDER_ID` | Google Drive folder ID with the second database dump file. | `UNDEFINED` | `.env` | | `VORTEX_FETCH_DB2_S3_BUCKET` | AWS S3 bucket name for second database fetch. | `UNDEFINED` | `.env` | | `VORTEX_FETCH_DB2_S3_REGION` | AWS S3 region. | `ap-southeast-2` | `.env` | | `VORTEX_FETCH_DB2_SOURCE` | Second database fetch source. | `url` | `.env` | @@ -350,6 +351,10 @@ The list below is automatically generated with [Shellvar](https://github.com/ale | `VORTEX_FETCH_DB_FTP_PASS` | Database dump FTP password. | `UNDEFINED` | `.env.local.example`, `.vortex/tooling/src/vortex-fetch-db-ftp` | | `VORTEX_FETCH_DB_FTP_PORT` | Database dump FTP port. | `21` | `.env`, `.vortex/tooling/src/vortex-fetch-db-ftp` | | `VORTEX_FETCH_DB_FTP_USER` | Database dump FTP user. | `UNDEFINED` | `.env.local.example`, `.vortex/tooling/src/vortex-fetch-db-ftp` | +| `VORTEX_FETCH_DB_GDRIVE_DB_DIR` | Directory with database dump file. | `./.data` | `.vortex/tooling/src/vortex-fetch-db-gdrive` | +| `VORTEX_FETCH_DB_GDRIVE_DB_FILE` | Database dump file name.

The same name is used for the file in the Google Drive folder. | `db.sql` | `.vortex/tooling/src/vortex-fetch-db-gdrive` | +| `VORTEX_FETCH_DB_GDRIVE_FOLDER_ID` | Google Drive folder ID with the database dump file.

The ID is the last segment of the folder URL. The folder must be in a shared drive and shared with the service account. | `UNDEFINED` | `.env`, `.vortex/tooling/src/vortex-fetch-db-gdrive` | +| `VORTEX_FETCH_DB_GDRIVE_SERVICE_ACCOUNT_KEY` | Google Cloud service account key: the JSON content or a path to the JSON key file. | `${VORTEX_GDRIVE_SERVICE_ACCOUNT_KEY}` | `.vortex/tooling/src/vortex-fetch-db-gdrive` | | `VORTEX_FETCH_DB_LAGOON_DB_DIR` | Directory where DB dumps are stored on the host. | `./.data` | `.vortex/tooling/src/vortex-fetch-db-lagoon` | | `VORTEX_FETCH_DB_LAGOON_DB_FILE` | Database dump file name on the host. | `db.sql` | `.vortex/tooling/src/vortex-fetch-db-lagoon` | | `VORTEX_FETCH_DB_LAGOON_PROJECT` | Lagoon project name. | `${LAGOON_PROJECT}` | `.vortex/tooling/src/vortex-fetch-db-lagoon` | @@ -367,7 +372,7 @@ The list below is automatically generated with [Shellvar](https://github.com/ale | `VORTEX_FETCH_DB_S3_PREFIX` | S3 prefix (path within the bucket). | `${S3_PREFIX}` | `.vortex/tooling/src/vortex-fetch-db-s3` | | `VORTEX_FETCH_DB_S3_REGION` | AWS S3 region. | `ap-southeast-2` | `.env`, `.vortex/tooling/src/vortex-fetch-db-s3` | | `VORTEX_FETCH_DB_S3_SECRET_KEY` | AWS secret key for S3 database fetch. | `UNDEFINED` | `.env.local.example`, `.vortex/tooling/src/vortex-fetch-db-s3` | -| `VORTEX_FETCH_DB_SOURCE` | Database fetch source. | `url` | `.env`, `.vortex/tooling/src/vortex-fetch-db` | +| `VORTEX_FETCH_DB_SOURCE` | Database fetch source. | `url` | `.env`, `.vortex/tooling/src/vortex-fetch-db`, `.vortex/tooling/src/vortex-push-db` | | `VORTEX_FETCH_DB_SSH_FILE` | Default SSH file used if custom fingerprint is not provided. | `${HOME}/.ssh/id_rsa` | `.vortex/tooling/src/vortex-fetch-db-lagoon` | | `VORTEX_FETCH_DB_SSH_FINGERPRINT` | SSH key fingerprint used to connect to a remote. | `UNDEFINED` | `.vortex/tooling/src/vortex-fetch-db-lagoon` | | `VORTEX_FETCH_DB_UNZIP_PASSWORD` | Password for unzipping password-protected zip files. | `UNDEFINED` | `.vortex/tooling/src/vortex-fetch-db-url` | @@ -392,6 +397,7 @@ The list below is automatically generated with [Shellvar](https://github.com/ale | `VORTEX_FETCH_FILES_SSH_FINGERPRINT` | SSH key fingerprint used to connect to a remote. | `UNDEFINED` | `.vortex/tooling/src/vortex-fetch-files-acquia`, `.vortex/tooling/src/vortex-fetch-files-lagoon` | | `VORTEX_FETCH_FILES_URL` | Public files archive sourced from a URL.

A tar archive, optionally compressed, with the content of the public files directory at its root. HTTP Basic Authentication credentials should be embedded into the value. | `UNDEFINED` | `.env`, `.vortex/tooling/src/vortex-fetch-files-url` | | `VORTEX_FRONTEND_BUILD_SKIP` | Skip building of the front-end assets inside the container image.

Set to `1` to build theme assets on the host or as part of deployment instead of during the container image build. | `UNDEFINED` | `.env` | +| `VORTEX_GDRIVE_SERVICE_ACCOUNT_KEY` | Google Cloud service account key: the JSON content or a path to the JSON key file. | `UNDEFINED` | `.env.local.example` | | `VORTEX_IMPORT_DB_FILE` | Database dump file name. | `db.sql` | `.vortex/tooling/src/vortex-import-db-file` | | `VORTEX_IMPORT_DB_FILE_DIR` | Directory with database dump file. | `./.data` | `.vortex/tooling/src/vortex-import-db-file` | | `VORTEX_INSTALLER_INTERACTIVE` | Run installer in interactive mode. | `0` | `.vortex/tooling/src/vortex-update` | @@ -536,17 +542,22 @@ The list below is automatically generated with [Shellvar](https://github.com/ale | `VORTEX_PUSH_CONTAINER_REGISTRY_MAP` | Comma-separated map of container services and images to push in format "service1=org/image1,service2=org/image2". | `UNDEFINED` | `.vortex/tooling/src/vortex-push-container-registry`, `.vortex/tooling/src/vortex-push-db-image` | | `VORTEX_PUSH_CONTAINER_REGISTRY_PASS` | The password to login into the container registry. | `${VORTEX_CONTAINER_REGISTRY_PASS}` | `.vortex/tooling/src/vortex-push-container-registry` | | `VORTEX_PUSH_CONTAINER_REGISTRY_USER` | The username to login into the container registry. | `${VORTEX_CONTAINER_REGISTRY_USER}` | `.vortex/tooling/src/vortex-push-container-registry` | +| `VORTEX_PUSH_DB_GDRIVE_DB_DIR` | Directory with database dump file. | `./.data` | `.vortex/tooling/src/vortex-push-db-gdrive` | +| `VORTEX_PUSH_DB_GDRIVE_DB_FILE` | Database dump file name.

The same name is used for the file in the Google Drive folder. | `db.sql` | `.vortex/tooling/src/vortex-push-db-gdrive` | +| `VORTEX_PUSH_DB_GDRIVE_FOLDER_ID` | Google Drive folder ID to push the database dump file to.

Defaults to the folder the database is fetched from. | `${VORTEX_FETCH_DB_GDRIVE_FOLDER_ID}` | `.vortex/tooling/src/vortex-push-db-gdrive` | +| `VORTEX_PUSH_DB_GDRIVE_SERVICE_ACCOUNT_KEY` | Google Cloud service account key: the JSON content or a path to the JSON key file. | `${VORTEX_GDRIVE_SERVICE_ACCOUNT_KEY}` | `.vortex/tooling/src/vortex-push-db-gdrive` | +| `VORTEX_PUSH_DB_PROCEED` | Proceed with push.

Set to `1` in the nightly CI job only, so the storage is refreshed once a day and never from a feature branch build. | `UNDEFINED` | `.vortex/tooling/src/vortex-push-db` | | `VORTEX_PUSH_DB_S3_ACCESS_KEY` | AWS access key. | `${S3_ACCESS_KEY}` | `.vortex/tooling/src/vortex-push-db-s3` | | `VORTEX_PUSH_DB_S3_BUCKET` | S3 bucket name. | `${S3_BUCKET}` | `.vortex/tooling/src/vortex-push-db-s3` | | `VORTEX_PUSH_DB_S3_DB_DIR` | Directory with database dump file. | `./.data` | `.vortex/tooling/src/vortex-push-db-s3` | | `VORTEX_PUSH_DB_S3_DB_FILE` | Database dump file name. | `db.sql` | `.vortex/tooling/src/vortex-push-db-s3` | | `VORTEX_PUSH_DB_S3_PREFIX` | S3 prefix (path within the bucket). | `${S3_PREFIX}` | `.vortex/tooling/src/vortex-push-db-s3` | | `VORTEX_PUSH_DB_S3_REGION` | S3 region. | `${S3_REGION}` | `.vortex/tooling/src/vortex-push-db-s3` | -| `VORTEX_PUSH_DB_S3_REMOTE_FILE` | Remote database dump file name. | `db.sql` | `.vortex/tooling/src/vortex-push-db-s3` | +| `VORTEX_PUSH_DB_S3_REMOTE_FILE` | Remote database dump file name.

Defaults to the database dump file name, which is the name the database is fetched with. | `${VORTEX_PUSH_DB_S3_DB_FILE}` | `.vortex/tooling/src/vortex-push-db-s3` | | `VORTEX_PUSH_DB_S3_SECRET_KEY` | AWS secret key. | `${S3_SECRET_KEY}` | `.vortex/tooling/src/vortex-push-db-s3` | | `VORTEX_PUSH_DB_S3_STORAGE_CLASS` | S3 storage class. | `STANDARD` | `.vortex/tooling/src/vortex-push-db-s3` | | `VORTEX_REFRESH_DB_PROCEED` | Proceed with the refresh of an indexed database.

The primary database is always refreshed. An indexed database, such as the migration database with VORTEX_DB_INDEX=`2`, is refreshed only when VORTEX_REFRESH_DB2_PROCEED is set to `1`. | `UNDEFINED` | `.vortex/tooling/src/vortex-refresh-db` | -| `VORTEX_REFRESH_DB_SOURCE` | Fetch source of the database dump file to import into the base image.

Required when the fetch source is 'container_registry' and a base image is set: the database image is built from a dump file, so it needs a dump file source such as 'url', 'ftp', 'acquia', 'lagoon' or 's3'. An indexed database uses VORTEX_REFRESH_DB2_SOURCE. | `UNDEFINED` | `.vortex/tooling/src/vortex-refresh-db` | +| `VORTEX_REFRESH_DB_SOURCE` | Upstream fetch source of the database dump file.

Required when the fetch source is 'container_registry' and a base image is set: the database image is built from a dump file, so it needs a dump file source such as 'url', 'ftp', 'acquia', 'lagoon', 's3' or 'gdrive'. An indexed database uses VORTEX_REFRESH_DB2_SOURCE.

Also required when the fetch source is a dump store ('gdrive' or 's3') and VORTEX_PUSH_DB_PROCEED is set to `1`: the dump pushed to the store comes from an upstream source such as 'url', 'ftp', 'acquia' or 'lagoon'. | `UNDEFINED` | `.vortex/tooling/src/vortex-refresh-db` | | `VORTEX_RELEASE_VERSION_SCHEME` | Versioning scheme used for releases.

Can be one of: calver, semver, other @see https://www.vortextemplate.com/docs/releasing | `calver` | `.env` | | `VORTEX_SHOW_LOGIN` | Show one-time login link. | `UNDEFINED` | `.vortex/tooling/src/vortex-info` | | `VORTEX_SSH_DISABLE_STRICT_HOST_KEY_CHECKING` | Disable strict host key checking in SSH. | `0` | `.vortex/tooling/src/vortex-setup-ssh` | diff --git a/.vortex/docs/content/features.mdx b/.vortex/docs/content/features.mdx index a30b487bdf..da09bc16c2 100644 --- a/.vortex/docs/content/features.mdx +++ b/.vortex/docs/content/features.mdx @@ -169,7 +169,8 @@ import { - _Lint_ the code without provisioning a site - _Fetch the database_ from the hosting provider and cache it for the - day, with a nightly refresh of the database container image + day, with a nightly refresh of the database container image or of a + sanitized dump in S3 or Google Drive - _Build_ and test a fully provisioned site, with [Behat](https://docs.behat.org/en/latest/) tests split across parallel containers @@ -244,7 +245,8 @@ import { ### Other hosting providers - Webhook deployments for any provider that accepts a deployment URL - - Database fetch from a URL, FTP, S3 or a container registry + - Database fetch from a URL, FTP, S3, Google Drive or a container + registry :::note @@ -373,7 +375,8 @@ import { [`drevops/vortex-tooling`](https://packagist.org/packages/drevops/vortex-tooling) Composer package, run the same way in every environment - Customization through environment variables - - Database sourced from the hosting provider, a URL, FTP, S3 or a + - Database sourced from the hosting provider, a URL, FTP, S3, Google + Drive or a container image seeded with [Database data images](https://github.com/drevops/database-data-images) diff --git a/.vortex/docs/cspell.json b/.vortex/docs/cspell.json index 8a3db57a56..1e33bea1dd 100644 --- a/.vortex/docs/cspell.json +++ b/.vortex/docs/cspell.json @@ -13,6 +13,7 @@ "Buildx", "Diffy", "Dockerfiles", + "GDRIVE", "GHSA", "Incapsula", "MAILHUB", @@ -53,6 +54,7 @@ "drush", "ergebnis", "fieldsets", + "gdrive", "gherkinlint", "gitleaks", "hadolint", diff --git a/.vortex/tooling/playground/try-fetch-db-gdrive.sh b/.vortex/tooling/playground/try-fetch-db-gdrive.sh new file mode 100755 index 0000000000..43fd135f42 --- /dev/null +++ b/.vortex/tooling/playground/try-fetch-db-gdrive.sh @@ -0,0 +1,57 @@ +#!/usr/bin/env bash +## +# Manual test script for Google Drive database fetch. +# +# Fetches a database dump file from a Google Drive folder using +# fetch-db-gdrive. +# +# Usage: +# export GDRIVE_SERVICE_ACCOUNT_KEY="/path/to/service-account.json" +# export GDRIVE_FOLDER_ID="your-folder-id" +# ./try-fetch-db-gdrive.sh + +set -eu + +# Source secrets if available. +if [ -f "$HOME/.profile.secrets" ]; then + # shellcheck disable=SC1091 + . "$HOME/.profile.secrets" +fi + +# Get the directory of this script and navigate to project root. +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_ROOT="$(cd "${SCRIPT_DIR}/../../.." && pwd)" + +# Google Drive credentials and settings. +GDRIVE_SERVICE_ACCOUNT_KEY="${GDRIVE_SERVICE_ACCOUNT_KEY:-}" +GDRIVE_FOLDER_ID="${GDRIVE_FOLDER_ID:-}" + +# Validate required variables. +if [ -z "${GDRIVE_SERVICE_ACCOUNT_KEY}" ]; then + echo "Error: GDRIVE_SERVICE_ACCOUNT_KEY environment variable is required" + echo "Usage: export GDRIVE_SERVICE_ACCOUNT_KEY=\"/path/to/service-account.json\" && $0" + exit 1 +fi + +if [ -z "${GDRIVE_FOLDER_ID}" ]; then + echo "Error: GDRIVE_FOLDER_ID environment variable is required" + echo "Usage: export GDRIVE_FOLDER_ID=\"your-folder-id\" && $0" + exit 1 +fi + +echo "Testing Google Drive database fetch..." +echo "" +echo "Folder ID: ${GDRIVE_FOLDER_ID}" +echo "DB file: ${VORTEX_DB_FILE:-db.sql}" +echo "Local dir: ${VORTEX_DB_DIR:-./.data}" +echo "" + +cd "${PROJECT_ROOT}" || exit 1 + +export VORTEX_FETCH_DB_GDRIVE_SERVICE_ACCOUNT_KEY="${GDRIVE_SERVICE_ACCOUNT_KEY}" +export VORTEX_FETCH_DB_GDRIVE_FOLDER_ID="${GDRIVE_FOLDER_ID}" + +./vendor/drevops/vortex-tooling/src/vortex-fetch-db-gdrive + +echo "" +echo "Fetch complete!" diff --git a/.vortex/tooling/playground/try-push-db-gdrive.sh b/.vortex/tooling/playground/try-push-db-gdrive.sh new file mode 100755 index 0000000000..9cddbab46f --- /dev/null +++ b/.vortex/tooling/playground/try-push-db-gdrive.sh @@ -0,0 +1,65 @@ +#!/usr/bin/env bash +## +# Manual test script for Google Drive database push. +# +# Uploads a database dump file to a Google Drive folder in a shared drive +# using push-db-gdrive. Run it twice to check that the second run replaces +# the file instead of creating a duplicate. +# +# Usage: +# export GDRIVE_SERVICE_ACCOUNT_KEY="/path/to/service-account.json" +# export GDRIVE_FOLDER_ID="your-folder-id" +# ./try-push-db-gdrive.sh + +set -eu + +# Source secrets if available. +if [ -f "$HOME/.profile.secrets" ]; then + # shellcheck disable=SC1091 + . "$HOME/.profile.secrets" +fi + +# Get the directory of this script and navigate to project root. +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_ROOT="$(cd "${SCRIPT_DIR}/../../.." && pwd)" + +# Google Drive credentials and settings. +GDRIVE_SERVICE_ACCOUNT_KEY="${GDRIVE_SERVICE_ACCOUNT_KEY:-}" +GDRIVE_FOLDER_ID="${GDRIVE_FOLDER_ID:-}" + +# Validate required variables. +if [ -z "${GDRIVE_SERVICE_ACCOUNT_KEY}" ]; then + echo "Error: GDRIVE_SERVICE_ACCOUNT_KEY environment variable is required" + echo "Usage: export GDRIVE_SERVICE_ACCOUNT_KEY=\"/path/to/service-account.json\" && $0" + exit 1 +fi + +if [ -z "${GDRIVE_FOLDER_ID}" ]; then + echo "Error: GDRIVE_FOLDER_ID environment variable is required" + echo "Usage: export GDRIVE_FOLDER_ID=\"your-folder-id\" && $0" + exit 1 +fi + +cd "${PROJECT_ROOT}" || exit 1 + +# Create a test dump file when none exists. +db_dir="${VORTEX_DB_DIR:-./.data}" +db_file="${VORTEX_DB_FILE:-db.sql}" +if [ ! -f "${db_dir}/${db_file}" ]; then + mkdir -p "${db_dir}" + echo "-- Test dump created by try-push-db-gdrive.sh on $(date -u)" >"${db_dir}/${db_file}" +fi + +echo "Testing Google Drive database push..." +echo "" +echo "Folder ID: ${GDRIVE_FOLDER_ID}" +echo "Local file: ${db_dir}/${db_file}" +echo "" + +export VORTEX_PUSH_DB_GDRIVE_SERVICE_ACCOUNT_KEY="${GDRIVE_SERVICE_ACCOUNT_KEY}" +export VORTEX_PUSH_DB_GDRIVE_FOLDER_ID="${GDRIVE_FOLDER_ID}" + +./vendor/drevops/vortex-tooling/src/vortex-push-db-gdrive + +echo "" +echo "Push complete!" From 70f978dbe66c3b98ac71ad89400356414173b603 Mon Sep 17 00:00:00 2001 From: Alex Skrypnyk Date: Wed, 11 Nov 2026 10:33:45 +1100 Subject: [PATCH 04/21] Updated snapshots. --- .../db_fetch_source_container_registry/.env | 6 +- .../db_fetch_source_gdrive/.env | 45 +++++ .../db_fetch_source_gdrive/.env.local.example | 10 + .../.github/workflows/build-test-deploy.yml | 33 ++++ .../handler_process/db_fetch_source_s3/.env | 10 +- .../.github/workflows/build-test-deploy.yml | 34 ++++ .../.env | 6 +- .../migration_fetch_source_gdrive/.ahoy.yml | 51 +++++ .../migration_fetch_source_gdrive/.env | 30 +++ .../.github/workflows/build-test-deploy.yml | 22 +++ .../composer.json | 9 + .../docker-compose.yml | 52 +++++ .../scripts/provision-20-migration.sh | 187 ++++++++++++++++++ .../phpunit/Drupal/DatabaseSettingsTest.php | 110 +++++++++++ .../Drupal/EnvironmentSettingsTest.php | 14 ++ .../tests/phpunit/Drupal/SettingsTestCase.php | 8 + .../migrations/ys_migrate_categories.yml | 29 +++ .../custom/ys_migrate/ys_migrate.info.yml | 9 + .../web/sites/default/settings.migration.php | 21 ++ .../web/sites/default/settings.php | 15 ++ 20 files changed, 693 insertions(+), 8 deletions(-) create mode 100644 .vortex/installer/tests/Fixtures/handler_process/db_fetch_source_gdrive/.env create mode 100644 .vortex/installer/tests/Fixtures/handler_process/db_fetch_source_gdrive/.env.local.example create mode 100644 .vortex/installer/tests/Fixtures/handler_process/db_fetch_source_gdrive/.github/workflows/build-test-deploy.yml create mode 100644 .vortex/installer/tests/Fixtures/handler_process/db_fetch_source_s3/.github/workflows/build-test-deploy.yml create mode 100644 .vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_gdrive/.ahoy.yml create mode 100644 .vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_gdrive/.env create mode 100644 .vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_gdrive/.github/workflows/build-test-deploy.yml create mode 100644 .vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_gdrive/composer.json create mode 100644 .vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_gdrive/docker-compose.yml create mode 100755 .vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_gdrive/scripts/provision-20-migration.sh create mode 100644 .vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_gdrive/tests/phpunit/Drupal/DatabaseSettingsTest.php create mode 100644 .vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_gdrive/tests/phpunit/Drupal/EnvironmentSettingsTest.php create mode 100644 .vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_gdrive/tests/phpunit/Drupal/SettingsTestCase.php create mode 100644 .vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_gdrive/web/modules/custom/ys_migrate/migrations/ys_migrate_categories.yml create mode 100644 .vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_gdrive/web/modules/custom/ys_migrate/ys_migrate.info.yml create mode 100644 .vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_gdrive/web/sites/default/settings.migration.php create mode 100644 .vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_gdrive/web/sites/default/settings.php diff --git a/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_container_registry/.env b/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_container_registry/.env index f98032044d..e6357bf6c3 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_container_registry/.env +++ b/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_container_registry/.env @@ -14,9 +14,9 @@ +VORTEX_DB_IMAGE=the_empire/star_wars:latest +# Source of the database dump file the nightly CI job imports into a clean -+# base image to build the database container image: url, ftp, acquia, lagoon -+# or s3. The fetch script of that source reads its own variables, such as -+# VORTEX_FETCH_DB_URL for url. Add them, and keep secrets in CI. ++# base image to build the database container image: url, ftp, acquia, lagoon, ++# s3 or gdrive. The fetch script of that source reads its own variables, such ++# as VORTEX_FETCH_DB_URL for url. Add them, and keep secrets in CI. +# VORTEX_REFRESH_DB_SOURCE=url + # Environment to fetch the database from. diff --git a/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_gdrive/.env b/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_gdrive/.env new file mode 100644 index 0000000000..eb9fa4209c --- /dev/null +++ b/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_gdrive/.env @@ -0,0 +1,45 @@ +@@ -180,13 +180,22 @@ + VORTEX_DB_FILE=db.sql + + # Database fetch source. +-VORTEX_FETCH_DB_SOURCE=url ++VORTEX_FETCH_DB_SOURCE=gdrive + +-# Database dump file sourced from a URL. ++# Database dump file sourced from Google Drive. ++ ++# Google Drive folder ID with the database dump file. + # +-# HTTP Basic Authentication credentials should be embedded into the value. +-VORTEX_FETCH_DB_URL= ++# The ID is the last segment of the folder URL. The folder must be in a shared ++# drive and shared with the service account. ++VORTEX_FETCH_DB_GDRIVE_FOLDER_ID= + ++# Source of the database dump file the nightly CI job sanitizes and pushes to ++# the database fetch source: url, ftp, acquia or lagoon. The fetch script of ++# that source reads its own variables, such as VORTEX_FETCH_DB_URL for url. ++# Add them, and keep secrets in CI. ++# VORTEX_REFRESH_DB_SOURCE=url ++ + # Environment to fetch the database from. + # + # Applies to hosting environments. +@@ -254,17 +263,3 @@ + # with optional names in the format "email|name". + # Example: "to1@example.com|Jane Doe, to2@example.com|John Doe" + VORTEX_NOTIFY_EMAIL_RECIPIENTS="webmaster@star-wars.com|Webmaster" +- +-################################################################################ +-# DEMO # +-################################################################################ +-# # +-# Override project-specific values for demonstration purposes. # +-# Used to showcase Vortex without asking users to perform additional steps. # +-# # +-# Remove this section after completing database fetch integration. # +-# # +-################################################################################ +- +-# URL of the database used for demonstration with URL database fetch type. +-VORTEX_FETCH_DB_URL=https://github.com/drevops/vortex/releases/download/__VERSION__/db.demo.sql diff --git a/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_gdrive/.env.local.example b/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_gdrive/.env.local.example new file mode 100644 index 0000000000..e62dbc8c32 --- /dev/null +++ b/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_gdrive/.env.local.example @@ -0,0 +1,10 @@ +@@ -32,3 +32,9 @@ + + # Always override existing fetched DB dump. + VORTEX_FETCH_DB_FORCE=1 ++ ++# Database dump file sourced from Google Drive. ++# Service account key: Google Cloud console -> IAM & Admin -> Service accounts -> Keys -> Add key ++ ++# Google Cloud service account key: the JSON content or a path to the JSON key file. ++VORTEX_GDRIVE_SERVICE_ACCOUNT_KEY= diff --git a/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_gdrive/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_gdrive/.github/workflows/build-test-deploy.yml new file mode 100644 index 0000000000..88c93392b7 --- /dev/null +++ b/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_gdrive/.github/workflows/build-test-deploy.yml @@ -0,0 +1,33 @@ +@@ -191,6 +191,7 @@ + VORTEX_CI_DB_CACHE_FALLBACK: "yes" + # Which branch to use as a source of DB caches. + VORTEX_CI_DB_CACHE_BRANCH: "develop" ++ VORTEX_GDRIVE_SERVICE_ACCOUNT_KEY: ${{ secrets.VORTEX_GDRIVE_SERVICE_ACCOUNT_KEY }} + + steps: + # Frees disk space by removing preinstalled toolchains unused by this project. Steps run in a container, so the host is reached via the Docker socket. +@@ -241,6 +242,8 @@ + { + echo "VORTEX_CI_DB_CACHE_FALLBACK=no" + echo "VORTEX_FRONTEND_BUILD_SKIP=1" ++ # Refresh the database dump in the database fetch source. ++ echo "VORTEX_PUSH_DB_PROCEED=1" + } >> "$GITHUB_ENV" + + - name: Create cache keys files for database caching +@@ -274,6 +277,15 @@ + run: | + if [ ! -f /tmp/fetch-db-fresh ]; then echo "==> Database fetch semaphore file is missing. DB export will not proceed."; exit 0; fi + ./vendor/bin/vortex-refresh-db ++ timeout-minutes: 30 ++ ++ # Push the freshly-exported sanitized database dump to the database fetch ++ # source. No-op unless a fresh DB was fetched (the fetch semaphore) and the ++ # push was requested (VORTEX_PUSH_DB_PROCEED, in the script). ++ - name: Push DB ++ run: | ++ if [ ! -f /tmp/fetch-db-fresh ]; then echo "==> No freshly fetched database. DB push will not proceed."; exit 0; fi ++ ./vendor/bin/vortex-push-db + timeout-minutes: 30 + + # Save cache per default branch and the timestamp. diff --git a/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_s3/.env b/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_s3/.env index 5d9d274041..b26eb7cb6a 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_s3/.env +++ b/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_s3/.env @@ -1,4 +1,4 @@ -@@ -180,13 +180,16 @@ +@@ -180,13 +180,22 @@ VORTEX_DB_FILE=db.sql # Database fetch source. @@ -16,11 +16,17 @@ + +# AWS S3 region. +VORTEX_FETCH_DB_S3_REGION=ap-southeast-2 ++ ++# Source of the database dump file the nightly CI job sanitizes and pushes to ++# the database fetch source: url, ftp, acquia or lagoon. The fetch script of ++# that source reads its own variables, such as VORTEX_FETCH_DB_URL for url. ++# Add them, and keep secrets in CI. ++# VORTEX_REFRESH_DB_SOURCE=url + # Environment to fetch the database from. # # Applies to hosting environments. -@@ -254,17 +257,3 @@ +@@ -254,17 +263,3 @@ # with optional names in the format "email|name". # Example: "to1@example.com|Jane Doe, to2@example.com|John Doe" VORTEX_NOTIFY_EMAIL_RECIPIENTS="webmaster@star-wars.com|Webmaster" diff --git a/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_s3/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_s3/.github/workflows/build-test-deploy.yml new file mode 100644 index 0000000000..d8ec3cb540 --- /dev/null +++ b/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_s3/.github/workflows/build-test-deploy.yml @@ -0,0 +1,34 @@ +@@ -191,6 +191,8 @@ + VORTEX_CI_DB_CACHE_FALLBACK: "yes" + # Which branch to use as a source of DB caches. + VORTEX_CI_DB_CACHE_BRANCH: "develop" ++ VORTEX_FETCH_DB_S3_ACCESS_KEY: ${{ secrets.VORTEX_FETCH_DB_S3_ACCESS_KEY }} ++ VORTEX_FETCH_DB_S3_SECRET_KEY: ${{ secrets.VORTEX_FETCH_DB_S3_SECRET_KEY }} + + steps: + # Frees disk space by removing preinstalled toolchains unused by this project. Steps run in a container, so the host is reached via the Docker socket. +@@ -241,6 +243,8 @@ + { + echo "VORTEX_CI_DB_CACHE_FALLBACK=no" + echo "VORTEX_FRONTEND_BUILD_SKIP=1" ++ # Refresh the database dump in the database fetch source. ++ echo "VORTEX_PUSH_DB_PROCEED=1" + } >> "$GITHUB_ENV" + + - name: Create cache keys files for database caching +@@ -274,6 +278,15 @@ + run: | + if [ ! -f /tmp/fetch-db-fresh ]; then echo "==> Database fetch semaphore file is missing. DB export will not proceed."; exit 0; fi + ./vendor/bin/vortex-refresh-db ++ timeout-minutes: 30 ++ ++ # Push the freshly-exported sanitized database dump to the database fetch ++ # source. No-op unless a fresh DB was fetched (the fetch semaphore) and the ++ # push was requested (VORTEX_PUSH_DB_PROCEED, in the script). ++ - name: Push DB ++ run: | ++ if [ ! -f /tmp/fetch-db-fresh ]; then echo "==> No freshly fetched database. DB push will not proceed."; exit 0; fi ++ ./vendor/bin/vortex-push-db + timeout-minutes: 30 + + # Save cache per default branch and the timestamp. diff --git a/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_container_registry/.env b/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_container_registry/.env index cdfd843852..fc0db2960c 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_container_registry/.env +++ b/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_container_registry/.env @@ -24,9 +24,9 @@ + +# Source of the second database dump file the nightly CI job imports into a +# clean base image to rebuild the migration database container image: url, -+# ftp, acquia, lagoon or s3. The fetch script of that source reads its own -+# variables, such as VORTEX_FETCH_DB2_URL for url. Add them, and keep secrets -+# in CI. ++# ftp, acquia, lagoon, s3 or gdrive. The fetch script of that source reads its ++# own variables, such as VORTEX_FETCH_DB2_URL for url. Add them, and keep ++# secrets in CI. +# VORTEX_REFRESH_DB2_SOURCE=url + +# Environment to fetch the second database from. diff --git a/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_gdrive/.ahoy.yml b/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_gdrive/.ahoy.yml new file mode 100644 index 0000000000..11f7a073e6 --- /dev/null +++ b/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_gdrive/.ahoy.yml @@ -0,0 +1,51 @@ +@@ -32,6 +32,7 @@ + cmd: | + export COMPOSE_PROJECT_NAME=${COMPOSE_PROJECT_NAME:-${PWD##*/}} + export VORTEX_HOST_DB_PORT=$(docker compose port database 3306 2>/dev/null | cut -d : -f 2) ++ export VORTEX_HOST_DB2_PORT=$(docker compose port database2 3306 2>/dev/null | cut -d : -f 2) + export VORTEX_HOST_SOLR_PORT=$(docker compose port solr 8983 2>/dev/null | cut -d : -f 2) + export VORTEX_HOST_SELENIUM_VNC_PORT=$(docker compose port chrome 7900 2>/dev/null | cut -d : -f 2) + export VORTEX_HOST_HAS_SEQUELACE=$(uname -a | grep -i -q darwin && mdfind -name 'Sequel Ace' 2>/dev/null | grep -q "Ace" && echo 1 || true) +@@ -45,6 +46,14 @@ + open "mysql://${DATABASE_USERNAME:-drupal}:${DATABASE_PASSWORD:-drupal}@127.0.0.1:${VORTEX_HOST_DB_PORT}/drupal" -a "Sequel Ace" \ + || echo "Not a supported OS or Sequel Ace is not installed." + ++ db2: ++ usage: Open DB2 in Sequel Ace. ++ cmd: | ++ uname -a | grep -i -q darwin && test -d "${VORTEX_HOST_SEQUELACE_PATH:-/Applications/Sequel Ace.app}" && \ ++ VORTEX_HOST_DB2_PORT=$(docker compose port database2 3306 2>/dev/null | cut -d : -f 2) && \ ++ open "mysql://${DATABASE2_USERNAME:-drupal}:${DATABASE2_PASSWORD:-drupal}@127.0.0.1:${VORTEX_HOST_DB2_PORT}/drupal" -a "Sequel Ace" \ ++ || echo "Not a supported OS or Sequel Ace is not installed." ++ + # ---------------------------------------------------------------------------- + # Container commands. + # ---------------------------------------------------------------------------- +@@ -122,6 +131,13 @@ + case " $* " in *" --fresh "*) export VORTEX_FETCH_DB_FRESH=1 VORTEX_FETCH_DB_FORCE=1;; esac + ./vendor/bin/vortex-fetch-db + ++ fetch-db2: ++ usage: Fetch second database (migration). Run with "--fresh" to force. ++ aliases: [download-db2, db2-download, db2-fetch] ++ cmd: | ++ case " $* " in *" --fresh "*) export VORTEX_FETCH_DB2_FRESH=1 VORTEX_FETCH_DB2_FORCE=1;; esac ++ VORTEX_DB_INDEX=2 ./vendor/bin/vortex-fetch-db ++ + fetch-files: + usage: Fetch public files. + aliases: [download-files, files-download, files-fetch] +@@ -134,6 +150,13 @@ + docker compose rm --force --stop --volumes database + docker compose build database + ahoy up ++ ++ reload-db2: ++ usage: Reload the migration database container using local database image. ++ cmd: | ++ ahoy confirm "Running this command will replace your current database. Are you sure?" || exit 0 ++ docker compose rm --force --stop --volumes database2 ++ ahoy up -- --build database2 + + provision: + usage: Provision a site from the database dump or profile. diff --git a/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_gdrive/.env b/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_gdrive/.env new file mode 100644 index 0000000000..bce43c946e --- /dev/null +++ b/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_gdrive/.env @@ -0,0 +1,30 @@ +@@ -195,6 +195,29 @@ + VORTEX_FETCH_DB_ENVIRONMENT=prod + + ################################################################################ ++# SECOND DATABASE (MIGRATION) # ++################################################################################ ++ ++# Second database for migrations. ++# See settings.migration.php for database credentials. ++ ++# Second database dump file name. ++VORTEX_FETCH_DB2_FILE=db2.sql ++ ++# Second database fetch source. ++VORTEX_FETCH_DB2_SOURCE=gdrive ++ ++# Second database dump file sourced from Google Drive. ++ ++# Google Drive folder ID with the second database dump file. ++VORTEX_FETCH_DB2_GDRIVE_FOLDER_ID= ++ ++# Environment to fetch the second database from. ++# ++# Applies to hosting environments. ++VORTEX_FETCH_DB2_ENVIRONMENT=prod ++ ++################################################################################ + # FILES SOURCE # + ################################################################################ + diff --git a/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_gdrive/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_gdrive/.github/workflows/build-test-deploy.yml new file mode 100644 index 0000000000..4292b4f549 --- /dev/null +++ b/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_gdrive/.github/workflows/build-test-deploy.yml @@ -0,0 +1,22 @@ +@@ -270,6 +270,9 @@ + echo "db_hash=${{ hashFiles('.data') }}" >> "$GITHUB_ENV" + timeout-minutes: 30 + ++ - name: Fetch migration DB ++ run: VORTEX_DB_INDEX=2 ./vendor/bin/vortex-fetch-db ++ + - name: Export DB + run: | + if [ ! -f /tmp/fetch-db-fresh ]; then echo "==> Database fetch semaphore file is missing. DB export will not proceed."; exit 0; fi +@@ -431,6 +434,11 @@ + docker compose exec cli mkdir -p .data + docker compose --progress quiet cp -L .data/db.sql cli:/app/.data/db.sql + rm -f .data/db.sql ++ fi ++ if [ -f ".data/${VORTEX_FETCH_DB2_FILE:-db2.sql}" ]; then ++ docker compose exec -T cli mkdir -p .data ++ docker compose --progress quiet cp -L ".data/${VORTEX_FETCH_DB2_FILE:-db2.sql}" cli:"/app/.data/${VORTEX_FETCH_DB2_FILE:-db2.sql}" ++ rm -f ".data/${VORTEX_FETCH_DB2_FILE:-db2.sql}" + fi + docker compose exec $(env | cut -f1 -d= | sed 's/^/-e /') -T cli ./vendor/bin/vortex-provision + timeout-minutes: 30 diff --git a/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_gdrive/composer.json b/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_gdrive/composer.json new file mode 100644 index 0000000000..b90a23c71c --- /dev/null +++ b/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_gdrive/composer.json @@ -0,0 +1,9 @@ +@@ -26,6 +26,8 @@ + "drupal/linkit": "__VERSION__", + "drupal/menu_trail_by_path": "__VERSION__", + "drupal/metatag": "__VERSION__", ++ "drupal/migrate_plus": "__VERSION__", ++ "drupal/migrate_tools": "__VERSION__", + "drupal/navigation_extra_tools": "__VERSION__", + "drupal/pathauto": "__VERSION__", + "drupal/redirect": "__VERSION__", diff --git a/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_gdrive/docker-compose.yml b/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_gdrive/docker-compose.yml new file mode 100644 index 0000000000..70ac2fcbc8 --- /dev/null +++ b/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_gdrive/docker-compose.yml @@ -0,0 +1,52 @@ +@@ -57,6 +57,16 @@ + DRUPAL_SHIELD_PASS: ${DRUPAL_SHIELD_PASS:-} + # Redis integration flag. + DRUPAL_REDIS_ENABLED: ${DRUPAL_REDIS_ENABLED:-} ++ # Local migration database host (not used in production). ++ DATABASE2_HOST: database2 ++ # Local migration database name (not used in production). ++ DATABASE2_NAME: drupal ++ # Local migration database user (not used in production). ++ DATABASE2_USERNAME: drupal ++ # Local migration database password (not used in production). ++ DATABASE2_PASSWORD: drupal ++ # Local migration database port (not used in production). ++ DATABASE2_PORT: 3306 + + # The default user under which the containers should run. + x-user: &default-user +@@ -155,6 +165,20 @@ + ports: + - '3306' # Database port in a container. Find port on host with `ahoy info` or `docker compose port database 3306`. + ++ database2: ++ image: "${VORTEX_DB2_IMAGE:-uselagoon/mysql-8.4:__VERSION__}" # Use custom database image (if defined) or fallback to standard database image. ++ environment: ++ <<: *default-environment ++ MYSQL_DATABASE: drupal ++ MYSQL_USER: drupal ++ MYSQL_PASSWORD: drupal ++ <<: *default-user ++ # dclint disable-line no-duplicate-exported-ports # container-only port, published on a random host port ++ ports: ++ - '3306' # Database port in a container. Find port on host with `ahoy info` or `docker compose port database2 3306`. ++ labels: ++ lagoon.type: mariadb # See https://docs.lagoon.sh/using-lagoon-advanced/service-types/ ++ + redis: + image: uselagoon/valkey-8:__VERSION__ + +@@ -198,11 +222,13 @@ + depends_on: + - clamav + - database ++ - database2 + - redis + - solr + command: + - clamav:3310 + - database:3306 ++ - database2:3306 + - redis:6379 + - solr:8983 + - curl -s "http://solr:8983/solr/drupal/select?q=*:*&rows=0&wt=json" | grep response diff --git a/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_gdrive/scripts/provision-20-migration.sh b/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_gdrive/scripts/provision-20-migration.sh new file mode 100755 index 0000000000..3000594760 --- /dev/null +++ b/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_gdrive/scripts/provision-20-migration.sh @@ -0,0 +1,187 @@ +#!/usr/bin/env bash +## +# Import migration database and run migrations. +# +# This script is called during site provisioning via provision.sh. +# Customize the migration names at the bottom of this file. +# +# shellcheck disable=SC2086 + +set -eu +[ "${VORTEX_DEBUG-}" = "1" ] && set -x + +# Skip all migrations. +DRUPAL_MIGRATION_SKIP="${DRUPAL_MIGRATION_SKIP:-0}" + +# Skip rollback of migrations before import. +DRUPAL_MIGRATION_ROLLBACK_SKIP="${DRUPAL_MIGRATION_ROLLBACK_SKIP:-1}" + +# Limit the number of entities to import. Set to 'all' to import all. +DRUPAL_MIGRATION_IMPORT_LIMIT="${DRUPAL_MIGRATION_IMPORT_LIMIT:-50}" + +# Update already imported entities during migration. +DRUPAL_MIGRATION_UPDATE="${DRUPAL_MIGRATION_UPDATE:-0}" + +# Feedback frequency for migration progress. +DRUPAL_MIGRATION_FEEDBACK="${DRUPAL_MIGRATION_FEEDBACK:-50}" + +# Import migration source database. Set to 1 to import, 0 to skip. +DRUPAL_MIGRATION_SOURCE_DB_IMPORT="${DRUPAL_MIGRATION_SOURCE_DB_IMPORT:-${VORTEX_PROVISION_OVERRIDE_DB:-0}}" + +# Table name to probe in the source database to verify it is not corrupted. +DRUPAL_MIGRATION_SOURCE_DB_PROBE_TABLE="${DRUPAL_MIGRATION_SOURCE_DB_PROBE_TABLE:-categories}" + +# Disable the search server while migrations run. Set to 0 to keep indexing. +DRUPAL_MIGRATION_SEARCH_DISABLE="${DRUPAL_MIGRATION_SEARCH_DISABLE:-1}" + +# Directory with database dump file. +VORTEX_DB_DIR="${VORTEX_DB_DIR:-./.data}" + +# Migration database dump file name. +VORTEX_FETCH_DB2_FILE="${VORTEX_FETCH_DB2_FILE:-db2.sql}" + +# ------------------------------------------------------------------------------ + +# @formatter:off +info() { printf " ==> %s\n" "${1}"; } +note() { printf " %s\n" "${1}"; } +task() { printf " > %s\n" "${1}"; } +pass() { printf " + %s\n" "${1}"; } +fail() { printf " ! %s\n" "${1}"; exit "${2:-1}"; } +# @formatter:on + +drush() { ./vendor/bin/drush -y "$@"; } + +# ------------------------------------------------------------------------------ + +info "Started migration operations." + +environment="$(drush php:eval "print \Drupal\Core\Site\Settings::get('environment');")" +note "Environment: ${environment}" + +# Skip migrations in production. +if [ "${environment}" = "prod" ]; then + DRUPAL_MIGRATION_SKIP=1 +fi + +note "Migration skip: ${DRUPAL_MIGRATION_SKIP}" +note "Migration limit: ${DRUPAL_MIGRATION_IMPORT_LIMIT}" +note "Migration skip rollback: ${DRUPAL_MIGRATION_ROLLBACK_SKIP}" +note "Migration update: ${DRUPAL_MIGRATION_UPDATE}" +note "Migration feedback: ${DRUPAL_MIGRATION_FEEDBACK}" +note "Source DB import: ${DRUPAL_MIGRATION_SOURCE_DB_IMPORT}" +note "Search disable: ${DRUPAL_MIGRATION_SEARCH_DISABLE}" +echo + +if [ "${DRUPAL_MIGRATION_SKIP}" = "1" ]; then + info "Skipped migrations. DRUPAL_MIGRATION_SKIP is set to 1." + exit 0 +fi + +# Helper function to run a single migration with configured options. +run_migration() { + local migration_name="${1}" + shift + + drush migrate:reset-status "${migration_name}" || fail "Failed to reset migration status for ${migration_name}." + + task "Running migration: ${migration_name}." + local opts=() + + opts+=("--feedback=${DRUPAL_MIGRATION_FEEDBACK}") + + if printf '%s\n' "${DRUPAL_MIGRATION_IMPORT_LIMIT}" | grep -q '^[0-9][0-9]*$' && [ "${DRUPAL_MIGRATION_IMPORT_LIMIT}" -gt 0 ]; then + opts+=("--limit=${DRUPAL_MIGRATION_IMPORT_LIMIT}") + fi + + if [ "${DRUPAL_MIGRATION_UPDATE}" = "1" ]; then + opts+=("--update") + fi + + # Add any additional arguments passed to the function. + opts+=("$@") + + drush migrate:import "${opts[@]}" "${migration_name}" || { + drush migrate:messages "${migration_name}" + fail "Failed to run migration ${migration_name}." + } + + pass "Migrated: ${migration_name}." +} + +# Detect if existing migration source database is corrupted. +if [ "${DRUPAL_MIGRATION_SOURCE_DB_IMPORT}" != "1" ]; then + note "Source database import is set to be skipped. Checking existing database." + task "Probing for '${DRUPAL_MIGRATION_SOURCE_DB_PROBE_TABLE}' table in the source database." + if drush sql:query --database=migrate "SELECT COUNT(*) FROM ${DRUPAL_MIGRATION_SOURCE_DB_PROBE_TABLE}" >/dev/null 2>&1; then + pass "Source database is intact." + else + note "Migration source database is corrupted or empty. Re-importing." + DRUPAL_MIGRATION_SOURCE_DB_IMPORT=1 + pass "Marked the source database for re-import." + fi +fi + +# Import the migration source database from the dump file. +if [ "${DRUPAL_MIGRATION_SOURCE_DB_IMPORT}" = "1" ]; then + task "Importing migration source database." + + [ ! -f "${VORTEX_DB_DIR}/${VORTEX_FETCH_DB2_FILE}" ] && fail "Migration source database file not found. Please run 'ahoy fetch-db2'." + + drush sql:drop --database=migrate + # shellcheck disable=SC2091 + $(drush sql:connect --database=migrate) <"${VORTEX_DB_DIR}/${VORTEX_FETCH_DB2_FILE}" + + pass "Imported migration source database." +else + note "Using existing migration source database." +fi + +task "Verifying migration source database." +if ! drush sql:query --database=migrate "SELECT COUNT(*) FROM ${DRUPAL_MIGRATION_SOURCE_DB_PROBE_TABLE}" >/dev/null 2>&1; then + drush sql:query --database=migrate "SHOW TABLES;" + fail "Migration source database is corrupted." +fi +pass "Verified migration source database." + +# Enable custom migration modules. +task "Enabling migration modules." +drush pm:install ys_migrate +pass "Enabled migration modules." + +info "Starting migrations." + +if [ "${DRUPAL_MIGRATION_SEARCH_DISABLE}" = "1" ]; then + task "Disabling Search API Solr server." + drush search-api:server-disable solr || true + pass "Disabled Search API Solr server." +fi + +if [ "${DRUPAL_MIGRATION_ROLLBACK_SKIP}" = "1" ]; then + note "Skipped rollback of all migrations." +else + task "Rolling back all migrations." + drush migrate:rollback --all || true + pass "Rolled back all migrations." +fi +echo + +# ------------------------------------------------------------------------------ +# Add your migrations below. +# ------------------------------------------------------------------------------ + +run_migration ys_migrate_categories + +echo +note "Finished migrations." + +if [ "${DRUPAL_MIGRATION_SEARCH_DISABLE}" = "1" ]; then + task "Enabling Search API Solr server." + drush search-api:server-enable solr || true + drush search-api:enable-all || true + pass "Enabled Search API Solr server." +fi + +drush migrate:status + +info "Finished migration operations." diff --git a/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_gdrive/tests/phpunit/Drupal/DatabaseSettingsTest.php b/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_gdrive/tests/phpunit/Drupal/DatabaseSettingsTest.php new file mode 100644 index 0000000000..956cd535d4 --- /dev/null +++ b/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_gdrive/tests/phpunit/Drupal/DatabaseSettingsTest.php @@ -0,0 +1,110 @@ +@@ -47,6 +47,17 @@ + 'prefix' => '', + ], + ], ++ 'migrate' => [ ++ 'default' => [ ++ 'database' => 'drupal', ++ 'username' => 'drupal', ++ 'password' => 'drupal', ++ 'host' => 'localhost', ++ 'port' => '', ++ 'prefix' => '', ++ 'driver' => 'mysql', ++ ], ++ ], + ], + ]; + +@@ -74,6 +85,17 @@ + 'prefix' => '', + ], + ], ++ 'migrate' => [ ++ 'default' => [ ++ 'database' => 'drupal', ++ 'username' => 'drupal', ++ 'password' => 'drupal', ++ 'host' => 'localhost', ++ 'port' => '', ++ 'prefix' => '', ++ 'driver' => 'mysql', ++ ], ++ ], + ], + ]; + +@@ -101,6 +123,17 @@ + 'prefix' => '', + ], + ], ++ 'migrate' => [ ++ 'default' => [ ++ 'database' => 'drupal', ++ 'username' => 'drupal', ++ 'password' => 'drupal', ++ 'host' => 'localhost', ++ 'port' => '', ++ 'prefix' => '', ++ 'driver' => 'mysql', ++ ], ++ ], + ], + ]; + +@@ -128,9 +161,55 @@ + 'prefix' => '', + ], + ], ++ 'migrate' => [ ++ 'default' => [ ++ 'database' => 'drupal', ++ 'username' => 'drupal', ++ 'password' => 'drupal', ++ 'host' => 'localhost', ++ 'port' => '', ++ 'prefix' => '', ++ 'driver' => 'mysql', ++ ], ++ ], + ], + ]; + ++ yield [ ++ [ ++ 'DATABASE2_NAME' => 'migrate_db_name', ++ 'DATABASE2_USERNAME' => 'migrate_db_user', ++ 'DATABASE2_PASSWORD' => 'migrate_db_pass', ++ 'DATABASE2_HOST' => 'migrate_db_host', ++ 'DATABASE2_PORT' => '3307', ++ ], ++ [ ++ 'default' => [ ++ 'default' => [ ++ 'database' => 'drupal', ++ 'username' => 'drupal', ++ 'password' => 'drupal', ++ 'host' => 'localhost', ++ 'port' => '3306', ++ 'charset' => 'utf8mb4', ++ 'collation' => 'utf8mb4_general_ci', ++ 'driver' => 'mysql', ++ 'prefix' => '', ++ ], ++ ], ++ 'migrate' => [ ++ 'default' => [ ++ 'database' => 'migrate_db_name', ++ 'username' => 'migrate_db_user', ++ 'password' => 'migrate_db_pass', ++ 'host' => 'migrate_db_host', ++ 'port' => '3307', ++ 'prefix' => '', ++ 'driver' => 'mysql', ++ ], ++ ], ++ ], ++ ]; + } + + } diff --git a/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_gdrive/tests/phpunit/Drupal/EnvironmentSettingsTest.php b/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_gdrive/tests/phpunit/Drupal/EnvironmentSettingsTest.php new file mode 100644 index 0000000000..8adb9246e8 --- /dev/null +++ b/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_gdrive/tests/phpunit/Drupal/EnvironmentSettingsTest.php @@ -0,0 +1,14 @@ +@@ -163,6 +163,13 @@ + $databases['default']['default']['collation'] = 'utf8_general_ci'; + $databases['default']['default']['driver'] = 'mysql'; + $databases['default']['default']['prefix'] = ''; ++ $databases['migrate']['default']['database'] = 'drupal'; ++ $databases['migrate']['default']['username'] = 'drupal'; ++ $databases['migrate']['default']['password'] = 'drupal'; ++ $databases['migrate']['default']['host'] = 'localhost'; ++ $databases['migrate']['default']['port'] = ''; ++ $databases['migrate']['default']['prefix'] = ''; ++ $databases['migrate']['default']['driver'] = 'mysql'; + $this->assertEquals($databases, $this->databases); + + // Verify key config overrides. diff --git a/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_gdrive/tests/phpunit/Drupal/SettingsTestCase.php b/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_gdrive/tests/phpunit/Drupal/SettingsTestCase.php new file mode 100644 index 0000000000..350cc85c04 --- /dev/null +++ b/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_gdrive/tests/phpunit/Drupal/SettingsTestCase.php @@ -0,0 +1,8 @@ +@@ -62,6 +62,7 @@ + const ALLOWED_ENV_VARS = [ + // Service variables. + 'DATABASE_', ++ 'DATABASE2_', + 'REDIS_', + 'CLAMAV_', + 'COMPOSE_', diff --git a/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_gdrive/web/modules/custom/ys_migrate/migrations/ys_migrate_categories.yml b/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_gdrive/web/modules/custom/ys_migrate/migrations/ys_migrate_categories.yml new file mode 100644 index 0000000000..0494698e5d --- /dev/null +++ b/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_gdrive/web/modules/custom/ys_migrate/migrations/ys_migrate_categories.yml @@ -0,0 +1,29 @@ +id: ys_migrate_categories +label: 'Migrate categories to tags' +migration_tags: + - ys_migrate + +source: + plugin: table + key: migrate + table_name: categories + id_fields: + id: + type: integer + fields: + id: id + name: name + description: description + +process: + name: name + description/value: description + description/format: + plugin: default_value + default_value: plain_text + vid: + plugin: default_value + default_value: tags + +destination: + plugin: 'entity:taxonomy_term' diff --git a/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_gdrive/web/modules/custom/ys_migrate/ys_migrate.info.yml b/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_gdrive/web/modules/custom/ys_migrate/ys_migrate.info.yml new file mode 100644 index 0000000000..96c67179ac --- /dev/null +++ b/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_gdrive/web/modules/custom/ys_migrate/ys_migrate.info.yml @@ -0,0 +1,9 @@ +name: star wars Migrate +type: module +description: Migrations for star wars site. +core_version_requirement: ^11 +package: star_wars +dependencies: + - drupal:migrate + - migrate_plus:migrate_plus + - migrate_tools:migrate_tools diff --git a/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_gdrive/web/sites/default/settings.migration.php b/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_gdrive/web/sites/default/settings.migration.php new file mode 100644 index 0000000000..660869b39f --- /dev/null +++ b/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_gdrive/web/sites/default/settings.migration.php @@ -0,0 +1,21 @@ + (getenv('DATABASE2_NAME') ?: getenv('DATABASE2_DATABASE')) ?: 'drupal', + 'username' => getenv('DATABASE2_USERNAME') ?: 'drupal', + 'password' => getenv('DATABASE2_PASSWORD') ?: 'drupal', + 'host' => getenv('DATABASE2_HOST') ?: 'localhost', + 'port' => getenv('DATABASE2_PORT') ?: '', + 'prefix' => '', + 'driver' => 'mysql', +]; diff --git a/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_gdrive/web/sites/default/settings.php b/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_gdrive/web/sites/default/settings.php new file mode 100644 index 0000000000..c04331d75e --- /dev/null +++ b/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_gdrive/web/sites/default/settings.php @@ -0,0 +1,15 @@ +@@ -167,6 +167,14 @@ + } + + //////////////////////////////////////////////////////////////////////////////// ++/// MIGRATION SETTINGS /// ++//////////////////////////////////////////////////////////////////////////////// ++// @todo Migration. Remove when migration configuration is no longer required. ++if (file_exists($app_root . '/' . $site_path . '/settings.migration.php')) { ++ require $app_root . '/' . $site_path . '/settings.migration.php'; ++} ++ ++//////////////////////////////////////////////////////////////////////////////// + /// LOCAL OVERRIDE /// + //////////////////////////////////////////////////////////////////////////////// + // @see https://www.vortextemplate.com/docs/development/settings#local-overrides From ac2bda685194085deb008a9ce4f7668f33e1b10b Mon Sep 17 00:00:00 2001 From: Alex Skrypnyk Date: Wed, 11 Nov 2026 10:39:02 +1100 Subject: [PATCH 05/21] [#154] Fetched the dump from the upstream source in 'vortex-fetch-db' so the first push to an empty dump store succeeds. --- .env.local.example | 2 +- .../content/continuous-integration/README.mdx | 4 +- .vortex/tooling/src/vortex-fetch-db | 15 +++ .vortex/tooling/src/vortex-refresh-db | 28 ++---- .vortex/tooling/tests/unit/fetch-db.bats | 85 +++++++++++++++++ .vortex/tooling/tests/unit/refresh-db.bats | 94 ++++--------------- 6 files changed, 126 insertions(+), 102 deletions(-) diff --git a/.env.local.example b/.env.local.example index e876e08dd8..4f87f00489 100644 --- a/.env.local.example +++ b/.env.local.example @@ -73,7 +73,7 @@ VORTEX_FETCH_DB_S3_SECRET_KEY= # Database dump file sourced from Google Drive. # Service account key: Google Cloud console -> IAM & Admin -> Service accounts -> Keys -> Add key -# Google Cloud service account key: the JSON content or a path to the JSON key file. +# Path to the Google Cloud service account JSON key file. Keep the file outside the project directory. VORTEX_GDRIVE_SERVICE_ACCOUNT_KEY= #;> DB_FETCH_SOURCE_GDRIVE diff --git a/.vortex/docs/content/continuous-integration/README.mdx b/.vortex/docs/content/continuous-integration/README.mdx index a7b34f8b52..2f47a89386 100644 --- a/.vortex/docs/content/continuous-integration/README.mdx +++ b/.vortex/docs/content/continuous-integration/README.mdx @@ -433,7 +433,7 @@ To refresh a dump store every night: | Source | Secrets | |---|---| | `s3` | `VORTEX_FETCH_DB_S3_ACCESS_KEY` and `VORTEX_FETCH_DB_S3_SECRET_KEY` | - | `gdrive` | `VORTEX_GDRIVE_SERVICE_ACCOUNT_KEY`, with the content of the JSON key file | + | `gdrive` | `VORTEX_GDRIVE_SERVICE_ACCOUNT_KEY`, with the content of the JSON key file on 1 line, such as the output of `jq -c . key.json` | GitHub Actions passes these secrets to the `database` job. Add the secrets of the hosting source to the `env` block of that job. @@ -458,7 +458,7 @@ shared drive. To set up the folder: `VORTEX_FETCH_DB_GDRIVE_FOLDER_ID` in `.env`. 4. Give developers the key of a second service account with the **Viewer** role. They set `VORTEX_GDRIVE_SERVICE_ACCOUNT_KEY` in - `.env.local` to the path of the key file or to its content. + `.env.local` to the path of the key file. ## Trigger conditions diff --git a/.vortex/tooling/src/vortex-fetch-db b/.vortex/tooling/src/vortex-fetch-db index faf32b4ac9..f1abf35058 100755 --- a/.vortex/tooling/src/vortex-fetch-db +++ b/.vortex/tooling/src/vortex-fetch-db @@ -55,6 +55,21 @@ info "Started database${_db_index:+ ${_db_index}} fetch." [ "${VORTEX_FETCH_DB_PROCEED}" != "1" ] && pass "Skipped database fetch as VORTEX_FETCH_DB_PROCEED is not set to 1." && exit 0 +# A dump store ('gdrive' or 's3') receives the dump that the nightly refresh +# pushes, so with a push requested the dump comes from its upstream source. +# The store may hold no dump yet on the first push. +if [ -z "${_db_index}" ] && [ "${VORTEX_PUSH_DB_PROCEED:-}" = "1" ]; then + case "${VORTEX_FETCH_DB_SOURCE}" in + gdrive | s3) + case "${VORTEX_REFRESH_DB_SOURCE:-}" in + "" | gdrive | s3 | container_registry | none) fail "The database dump file pushed to ${VORTEX_FETCH_DB_SOURCE} is fetched from an upstream source, but VORTEX_REFRESH_DB_SOURCE is \"${VORTEX_REFRESH_DB_SOURCE:-}\". Set it to the source of the dump file: url, ftp, acquia or lagoon." ;; + esac + note "Fetching from the ${VORTEX_REFRESH_DB_SOURCE} source in VORTEX_REFRESH_DB_SOURCE as VORTEX_PUSH_DB_PROCEED is set to 1." + VORTEX_FETCH_DB_SOURCE="${VORTEX_REFRESH_DB_SOURCE}" + ;; + esac +fi + # The fetch script is resolved before the existing dump check, so an # unsupported source fails even when a cached dump would be reused. case "${VORTEX_FETCH_DB_SOURCE}" in diff --git a/.vortex/tooling/src/vortex-refresh-db b/.vortex/tooling/src/vortex-refresh-db index 9b2f01e2e5..df899cafd5 100755 --- a/.vortex/tooling/src/vortex-refresh-db +++ b/.vortex/tooling/src/vortex-refresh-db @@ -16,9 +16,8 @@ # provisioning. This runs only when VORTEX_REFRESH_DB2_PROCEED is set to 1. # # With a dump store fetch source ('gdrive' or 's3') and VORTEX_PUSH_DB_PROCEED -# set to 1, the dump file is fetched from VORTEX_REFRESH_DB_SOURCE before -# provisioning and the database is sanitized after it, so the exported dump -# can be pushed to the store with 'vortex-push-db'. +# set to 1, the database is sanitized after provisioning, so the exported +# dump can be pushed to the store with 'vortex-push-db'. # # Every variable of the calling environment is passed to the provisioning in # the 'cli' container, so the script is intended to run in CI. @@ -52,8 +51,8 @@ VORTEX_DB_IMAGE_BASE="${VORTEX_DB_IMAGE_BASE:-}" # indexed database uses VORTEX_REFRESH_DB2_SOURCE. # # Also required when the fetch source is a dump store ('gdrive' or 's3') and -# VORTEX_PUSH_DB_PROCEED is set to 1: the dump pushed to the store comes from -# an upstream source such as 'url', 'ftp', 'acquia' or 'lagoon'. +# VORTEX_PUSH_DB_PROCEED is set to 1: 'vortex-fetch-db' then fetches the dump +# from this source, such as 'url', 'ftp', 'acquia' or 'lagoon'. _v="VORTEX_REFRESH_DB${_db_index}_SOURCE" VORTEX_REFRESH_DB_SOURCE="${!_v:-}" @@ -108,7 +107,7 @@ rebuild_image=0 [ -n "${db_image}" ] && [ -n "${db_image_base}" ] && rebuild_image=1 # A dump store ('gdrive' or 's3') receives the dump this refresh exports, so -# with a push requested the dump is fetched again from its upstream source. +# with a push requested the exported dump must be sanitized. refresh_store=0 if [ -z "${_db_index}" ] && [ -z "${db_image}" ] && [ "${VORTEX_PUSH_DB_PROCEED:-}" = "1" ]; then case "${fetch_source}" in @@ -116,11 +115,8 @@ if [ -z "${_db_index}" ] && [ -z "${db_image}" ] && [ "${VORTEX_PUSH_DB_PROCEED: esac fi -if [ "${refresh_store}" -eq 1 ]; then - case "${VORTEX_REFRESH_DB_SOURCE}" in - "" | gdrive | s3 | container_registry | none) fail "The database dump file pushed to ${fetch_source} is fetched from an upstream source, but VORTEX_REFRESH_DB_SOURCE is \"${VORTEX_REFRESH_DB_SOURCE}\". Set it to the source of the dump file: url, ftp, acquia or lagoon." ;; - esac - [ "${VORTEX_PROVISION_SANITIZE_DB_SKIP:-}" = "1" ] && fail "The database dump file pushed to ${fetch_source} must be sanitized, but VORTEX_PROVISION_SANITIZE_DB_SKIP is set to 1." +if [ "${refresh_store}" -eq 1 ] && [ "${VORTEX_PROVISION_SANITIZE_DB_SKIP:-}" = "1" ]; then + fail "The database dump file pushed to ${fetch_source} must be sanitized, but VORTEX_PROVISION_SANITIZE_DB_SKIP is set to 1." fi if [ "${rebuild_image}" -eq 1 ] && [ "${fetch_source}" = "container_registry" ]; then @@ -131,16 +127,6 @@ fi "$(dirname "${BASH_SOURCE[0]}")/vortex-login-container-registry" -if [ "${refresh_store}" -eq 1 ]; then - # The dump fetched from the store is removed, so the check below proves the - # upstream source wrote a new one. - rm -f "${db_file}" - # CircleCI re-sources the job variables from BASH_ENV in every bash - # script, which would replace the prefixed values. - env BASH_ENV='' "VORTEX_FETCH_DB_SOURCE=${VORTEX_REFRESH_DB_SOURCE}" "VORTEX_FETCH_DB_FORCE=1" "$(dirname "${BASH_SOURCE[0]}")/vortex-fetch-db" - [ -f "${db_file}" ] || fail "The database dump file was not fetched to ${db_file}. Remove the output directory and file name overrides of the database fetch source." -fi - # The value of VORTEX_DB_IMAGE passed to the provisioning. An empty value # makes the provisioning import the dump file. provision_db_image="${db_image}" diff --git a/.vortex/tooling/tests/unit/fetch-db.bats b/.vortex/tooling/tests/unit/fetch-db.bats index c274716c30..17adbc8f7e 100644 --- a/.vortex/tooling/tests/unit/fetch-db.bats +++ b/.vortex/tooling/tests/unit/fetch-db.bats @@ -326,6 +326,91 @@ EOF popd >/dev/null } +@test "fetch-db: Fetch a dump store source from the refresh source with a push requested" { + pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 + + stub_fetch_scripts + + export VORTEX_FETCH_DB_PROCEED="1" + export VORTEX_PUSH_DB_PROCEED="1" + export VORTEX_REFRESH_DB_SOURCE="lagoon" + + for db_source in gdrive s3; do + export VORTEX_FETCH_DB_SOURCE="${db_source}" + + run .vortex/tooling/src/vortex-fetch-db + assert_success + assert_output_contains "Fetching from the lagoon source in VORTEX_REFRESH_DB_SOURCE as VORTEX_PUSH_DB_PROCEED is set to 1." + assert_output_contains "Fetched from lagoon." + assert_output_not_contains "Fetched from ${db_source}." + assert_output_contains "Finished database fetch." + done + + popd >/dev/null +} + +@test "fetch-db: Fetch a dump store source from the store without a push requested" { + pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 + + stub_fetch_scripts + + export VORTEX_FETCH_DB_SOURCE="gdrive" + export VORTEX_FETCH_DB_PROCEED="1" + export VORTEX_REFRESH_DB_SOURCE="lagoon" + unset VORTEX_PUSH_DB_PROCEED + + run .vortex/tooling/src/vortex-fetch-db + assert_success + assert_output_contains "Fetched from gdrive." + assert_output_not_contains "Fetched from lagoon." + + popd >/dev/null +} + +@test "fetch-db: Fetch an indexed dump store source from the store with a push requested" { + pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 + + stub_fetch_scripts + + export VORTEX_DB_INDEX=3 + export VORTEX_FETCH_DB3_SOURCE="gdrive" + export VORTEX_FETCH_DB3_PROCEED="1" + export VORTEX_FETCH_DB3_DIR="${BUILD_DIR}/db3" + export VORTEX_PUSH_DB_PROCEED="1" + export VORTEX_REFRESH_DB_SOURCE="lagoon" + + run .vortex/tooling/src/vortex-fetch-db + assert_success + # Only the primary database is pushed to a dump store. + assert_output_contains "Fetched from gdrive." + assert_output_not_contains "Fetched from lagoon." + + popd >/dev/null +} + +@test "fetch-db: Fail without an upstream source for a dump store source with a push requested" { + pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 + + stub_fetch_scripts + + export VORTEX_FETCH_DB_SOURCE="gdrive" + export VORTEX_FETCH_DB_PROCEED="1" + export VORTEX_PUSH_DB_PROCEED="1" + export VORTEX_FETCH_DB_SEMAPHORE="${BUILD_DIR}/fetch-db-fresh" + + for refresh_source in "" gdrive s3 container_registry none; do + export VORTEX_REFRESH_DB_SOURCE="${refresh_source}" + + run .vortex/tooling/src/vortex-fetch-db + assert_failure + assert_output_contains "[FAIL] The database dump file pushed to gdrive is fetched from an upstream source, but VORTEX_REFRESH_DB_SOURCE is \"${refresh_source}\". Set it to the source of the dump file: url, ftp, acquia or lagoon." + assert_output_not_contains "Fetched from" + assert_file_not_exists "${BUILD_DIR}/fetch-db-fresh" + done + + popd >/dev/null +} + @test "fetch-db: Fetch nothing with the none source" { pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 diff --git a/.vortex/tooling/tests/unit/refresh-db.bats b/.vortex/tooling/tests/unit/refresh-db.bats index 338b9a7e8a..785087de4a 100644 --- a/.vortex/tooling/tests/unit/refresh-db.bats +++ b/.vortex/tooling/tests/unit/refresh-db.bats @@ -246,23 +246,20 @@ load ../_helper.bash popd >/dev/null } -@test "refresh-db: Fetches the dump file from the refresh source for a dump store source with a push requested" { +@test "refresh-db: Sanitizes the database for a dump store source with a push requested" { pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 + mkdir -p .data + echo "upstream" >.data/db.sql + unset VORTEX_DB_IMAGE VORTEX_DB_IMAGE_BASE export VORTEX_PUSH_DB_PROCEED=1 - export VORTEX_REFRESH_DB_SOURCE="lagoon" - touch "${BUILD_DIR}/bash_env" - export BASH_ENV="${BUILD_DIR}/bash_env" stub_sibling "vortex-login-container-registry" "Logged in to the registry." + stub_sibling "vortex-fetch-db" "Fetched the dump file." stub_sibling "vortex-export-db" 'Exported the database to ${1}.' - printf '#!/usr/bin/env bash\nmkdir -p .data\necho upstream >.data/db.sql\necho "Fetched from ${VORTEX_FETCH_DB_SOURCE} with force ${VORTEX_FETCH_DB_FORCE} and BASH_ENV [${BASH_ENV}]."\n' >.vortex/tooling/src/vortex-fetch-db - chmod +x .vortex/tooling/src/vortex-fetch-db export VORTEX_FETCH_DB_SOURCE="gdrive" - mkdir -p .data - echo "store" >.data/db.sql mock_command "sleep" >/dev/null mock_docker=$(mock_command "docker") @@ -275,7 +272,8 @@ load ../_helper.bash run .vortex/tooling/src/vortex-refresh-db assert_success - assert_output_contains "Fetched from lagoon with force 1 and BASH_ENV []." + # The fetch step already fetched the dump from the upstream source. + assert_output_not_contains "Fetched the dump file." assert_output_contains "[ OK ] Copied database dump file ./.data/db.sql into the cli container." assert_output_contains "Sanitized the database." assert_output_contains "Exported the database to db.sql." @@ -288,22 +286,19 @@ load ../_helper.bash assert_string_contains "$(mock_get_call_args "${mock_docker}" 6)" "compose --progress quiet cp -L cli:/app/.data/db.sql ./.data/db.sql" assert_equal "6" "$(mock_get_call_num "${mock_docker}")" - assert_file_contains .data/db.sql "upstream" - popd >/dev/null } -@test "refresh-db: Provisions from the dump store without the refresh source when a push is not requested" { +@test "refresh-db: Exports the dump store database unsanitized when a push is not requested" { pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 mkdir -p .data echo "store" >.data/db.sql - unset VORTEX_DB_IMAGE VORTEX_DB_IMAGE_BASE VORTEX_PUSH_DB_PROCEED VORTEX_REFRESH_DB_SOURCE + unset VORTEX_DB_IMAGE VORTEX_DB_IMAGE_BASE VORTEX_PUSH_DB_PROCEED export VORTEX_FETCH_DB_SOURCE="gdrive" stub_sibling "vortex-login-container-registry" "Logged in to the registry." - stub_sibling "vortex-fetch-db" "Fetched the dump file." stub_sibling "vortex-export-db" 'Exported the database to ${1}.' mock_command "sleep" >/dev/null @@ -311,39 +306,12 @@ load ../_helper.bash run .vortex/tooling/src/vortex-refresh-db assert_success - assert_output_not_contains "Fetched the dump file." assert_output_contains "Exported the database to db.sql." assert_output_contains "[ OK ] Finished database refresh." + # The dump from the store is already sanitized. + assert_string_contains "$(mock_get_call_args "${mock_docker}" 5)" "compose --progress quiet cp -L cli:/app/.data/db.sql ./.data/db.sql" assert_equal "5" "$(mock_get_call_num "${mock_docker}")" - assert_file_contains .data/db.sql "store" - - popd >/dev/null -} - -@test "refresh-db: Fails without an upstream source for a dump store source with a push requested" { - pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 - - unset VORTEX_DB_IMAGE VORTEX_DB_IMAGE_BASE - export VORTEX_FETCH_DB_SOURCE="gdrive" - export VORTEX_PUSH_DB_PROCEED=1 - - stub_sibling "vortex-login-container-registry" "Logged in to the registry." - stub_sibling "vortex-fetch-db" "Fetched the dump file." - - mock_docker=$(mock_command "docker") - - for refresh_source in "" gdrive s3 container_registry none; do - export VORTEX_REFRESH_DB_SOURCE="${refresh_source}" - - run .vortex/tooling/src/vortex-refresh-db - assert_failure - assert_output_contains "[FAIL] The database dump file pushed to gdrive is fetched from an upstream source, but VORTEX_REFRESH_DB_SOURCE is \"${refresh_source}\". Set it to the source of the dump file: url, ftp, acquia or lagoon." - assert_output_not_contains "Logged in to the registry." - assert_output_not_contains "Fetched the dump file." - done - - assert_equal "0" "$(mock_get_call_num "${mock_docker}")" popd >/dev/null } @@ -354,18 +322,16 @@ load ../_helper.bash unset VORTEX_DB_IMAGE VORTEX_DB_IMAGE_BASE export VORTEX_FETCH_DB_SOURCE="gdrive" export VORTEX_PUSH_DB_PROCEED=1 - export VORTEX_REFRESH_DB_SOURCE="lagoon" export VORTEX_PROVISION_SANITIZE_DB_SKIP=1 stub_sibling "vortex-login-container-registry" "Logged in to the registry." - stub_sibling "vortex-fetch-db" "Fetched the dump file." mock_docker=$(mock_command "docker") run .vortex/tooling/src/vortex-refresh-db assert_failure assert_output_contains "[FAIL] The database dump file pushed to gdrive must be sanitized, but VORTEX_PROVISION_SANITIZE_DB_SKIP is set to 1." - assert_output_not_contains "Fetched the dump file." + assert_output_not_contains "Logged in to the registry." assert_equal "0" "$(mock_get_call_num "${mock_docker}")" popd >/dev/null @@ -374,15 +340,15 @@ load ../_helper.bash @test "refresh-db: Fails when the sanitization fails for a dump store source" { pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 + mkdir -p .data + echo "upstream" >.data/db.sql + unset VORTEX_DB_IMAGE VORTEX_DB_IMAGE_BASE - export VORTEX_FETCH_DB_SOURCE="gdrive" + export VORTEX_FETCH_DB_SOURCE="s3" export VORTEX_PUSH_DB_PROCEED=1 - export VORTEX_REFRESH_DB_SOURCE="url" stub_sibling "vortex-login-container-registry" "Logged in to the registry." stub_sibling "vortex-export-db" 'Exported the database to ${1}.' - printf '#!/usr/bin/env bash\nmkdir -p .data\necho upstream >.data/db.sql\n' >.vortex/tooling/src/vortex-fetch-db - chmod +x .vortex/tooling/src/vortex-fetch-db mock_command "sleep" >/dev/null mock_docker=$(mock_command "docker") @@ -400,34 +366,6 @@ load ../_helper.bash popd >/dev/null } -@test "refresh-db: Fails when the refresh source does not fetch the dump file for a dump store source" { - pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 - - mkdir -p .data - echo "store" >.data/db.sql - - unset VORTEX_DB_IMAGE VORTEX_DB_IMAGE_BASE - export VORTEX_FETCH_DB_SOURCE="s3" - export VORTEX_PUSH_DB_PROCEED=1 - export VORTEX_REFRESH_DB_SOURCE="url" - - stub_sibling "vortex-login-container-registry" "Logged in to the registry." - stub_sibling "vortex-fetch-db" "Fetched the dump file elsewhere." - - mock_docker=$(mock_command "docker") - - run .vortex/tooling/src/vortex-refresh-db - assert_failure - assert_output_contains "Fetched the dump file elsewhere." - assert_output_contains "[FAIL] The database dump file was not fetched to ./.data/db.sql. Remove the output directory and file name overrides of the database fetch source." - - # The stale dump from the store is never provisioned or pushed. - assert_file_not_exists .data/db.sql - assert_equal "0" "$(mock_get_call_num "${mock_docker}")" - - popd >/dev/null -} - @test "refresh-db: Fails when the exported dump file cannot be copied from the cli container" { pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 From ec6aea52fe243117250e45180f14b8626bb704b6 Mon Sep 17 00:00:00 2001 From: Alex Skrypnyk Date: Wed, 11 Nov 2026 10:42:46 +1100 Subject: [PATCH 06/21] [#154] Narrowed the fetch-db upstream source failure assertion. --- .vortex/tooling/tests/unit/fetch-db.bats | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.vortex/tooling/tests/unit/fetch-db.bats b/.vortex/tooling/tests/unit/fetch-db.bats index 17adbc8f7e..bd8d08e665 100644 --- a/.vortex/tooling/tests/unit/fetch-db.bats +++ b/.vortex/tooling/tests/unit/fetch-db.bats @@ -404,7 +404,8 @@ EOF run .vortex/tooling/src/vortex-fetch-db assert_failure assert_output_contains "[FAIL] The database dump file pushed to gdrive is fetched from an upstream source, but VORTEX_REFRESH_DB_SOURCE is \"${refresh_source}\". Set it to the source of the dump file: url, ftp, acquia or lagoon." - assert_output_not_contains "Fetched from" + assert_output_not_contains "Fetched from gdrive." + assert_output_not_contains "Finished database fetch." assert_file_not_exists "${BUILD_DIR}/fetch-db-fresh" done From f472a72fd647ac70618eaf1ab9b6bb42e880e133 Mon Sep 17 00:00:00 2001 From: Alex Skrypnyk Date: Wed, 11 Nov 2026 10:44:14 +1100 Subject: [PATCH 07/21] [#154] Regenerated the variables reference. --- .vortex/docs/content/development/variables.mdx | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.vortex/docs/content/development/variables.mdx b/.vortex/docs/content/development/variables.mdx index ac37b24c5c..a111857861 100644 --- a/.vortex/docs/content/development/variables.mdx +++ b/.vortex/docs/content/development/variables.mdx @@ -397,7 +397,7 @@ The list below is automatically generated with [Shellvar](https://github.com/ale | `VORTEX_FETCH_FILES_SSH_FINGERPRINT` | SSH key fingerprint used to connect to a remote. | `UNDEFINED` | `.vortex/tooling/src/vortex-fetch-files-acquia`, `.vortex/tooling/src/vortex-fetch-files-lagoon` | | `VORTEX_FETCH_FILES_URL` | Public files archive sourced from a URL.

A tar archive, optionally compressed, with the content of the public files directory at its root. HTTP Basic Authentication credentials should be embedded into the value. | `UNDEFINED` | `.env`, `.vortex/tooling/src/vortex-fetch-files-url` | | `VORTEX_FRONTEND_BUILD_SKIP` | Skip building of the front-end assets inside the container image.

Set to `1` to build theme assets on the host or as part of deployment instead of during the container image build. | `UNDEFINED` | `.env` | -| `VORTEX_GDRIVE_SERVICE_ACCOUNT_KEY` | Google Cloud service account key: the JSON content or a path to the JSON key file. | `UNDEFINED` | `.env.local.example` | +| `VORTEX_GDRIVE_SERVICE_ACCOUNT_KEY` | Path to the Google Cloud service account JSON key file. Keep the file outside the project directory. | `UNDEFINED` | `.env.local.example` | | `VORTEX_IMPORT_DB_FILE` | Database dump file name. | `db.sql` | `.vortex/tooling/src/vortex-import-db-file` | | `VORTEX_IMPORT_DB_FILE_DIR` | Directory with database dump file. | `./.data` | `.vortex/tooling/src/vortex-import-db-file` | | `VORTEX_INSTALLER_INTERACTIVE` | Run installer in interactive mode. | `0` | `.vortex/tooling/src/vortex-update` | @@ -557,7 +557,6 @@ The list below is automatically generated with [Shellvar](https://github.com/ale | `VORTEX_PUSH_DB_S3_SECRET_KEY` | AWS secret key. | `${S3_SECRET_KEY}` | `.vortex/tooling/src/vortex-push-db-s3` | | `VORTEX_PUSH_DB_S3_STORAGE_CLASS` | S3 storage class. | `STANDARD` | `.vortex/tooling/src/vortex-push-db-s3` | | `VORTEX_REFRESH_DB_PROCEED` | Proceed with the refresh of an indexed database.

The primary database is always refreshed. An indexed database, such as the migration database with VORTEX_DB_INDEX=`2`, is refreshed only when VORTEX_REFRESH_DB2_PROCEED is set to `1`. | `UNDEFINED` | `.vortex/tooling/src/vortex-refresh-db` | -| `VORTEX_REFRESH_DB_SOURCE` | Upstream fetch source of the database dump file.

Required when the fetch source is 'container_registry' and a base image is set: the database image is built from a dump file, so it needs a dump file source such as 'url', 'ftp', 'acquia', 'lagoon', 's3' or 'gdrive'. An indexed database uses VORTEX_REFRESH_DB2_SOURCE.

Also required when the fetch source is a dump store ('gdrive' or 's3') and VORTEX_PUSH_DB_PROCEED is set to `1`: the dump pushed to the store comes from an upstream source such as 'url', 'ftp', 'acquia' or 'lagoon'. | `UNDEFINED` | `.vortex/tooling/src/vortex-refresh-db` | | `VORTEX_RELEASE_VERSION_SCHEME` | Versioning scheme used for releases.

Can be one of: calver, semver, other @see https://www.vortextemplate.com/docs/releasing | `calver` | `.env` | | `VORTEX_SHOW_LOGIN` | Show one-time login link. | `UNDEFINED` | `.vortex/tooling/src/vortex-info` | | `VORTEX_SSH_DISABLE_STRICT_HOST_KEY_CHECKING` | Disable strict host key checking in SSH. | `0` | `.vortex/tooling/src/vortex-setup-ssh` | From ee301e6180d80c9598d829c34480fe6ae31342a3 Mon Sep 17 00:00:00 2001 From: Alex Skrypnyk Date: Wed, 11 Nov 2026 10:46:18 +1100 Subject: [PATCH 08/21] Updated snapshots. --- .../handler_process/db_fetch_source_gdrive/.env.local.example | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_gdrive/.env.local.example b/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_gdrive/.env.local.example index e62dbc8c32..0f1fe53eb1 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_gdrive/.env.local.example +++ b/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_gdrive/.env.local.example @@ -6,5 +6,5 @@ +# Database dump file sourced from Google Drive. +# Service account key: Google Cloud console -> IAM & Admin -> Service accounts -> Keys -> Add key + -+# Google Cloud service account key: the JSON content or a path to the JSON key file. ++# Path to the Google Cloud service account JSON key file. Keep the file outside the project directory. +VORTEX_GDRIVE_SERVICE_ACCOUNT_KEY= From d7f7c5e72ccd3c9b0c77b392b73fb161b0b24806 Mon Sep 17 00:00:00 2001 From: Alex Skrypnyk Date: Wed, 11 Nov 2026 11:19:17 +1100 Subject: [PATCH 09/21] Addressed code review: defaulted the Google Drive and S3 push file name to 'VORTEX_FETCH_DB_FILE'. --- .vortex/tooling/src/vortex-push-db-gdrive | 5 ++- .vortex/tooling/src/vortex-push-db-s3 | 4 +- .../tooling/tests/unit/push-db-gdrive.bats | 31 +++++++++++++++ .vortex/tooling/tests/unit/push-db-s3.bats | 38 +++++++++++++++++++ 4 files changed, 75 insertions(+), 3 deletions(-) diff --git a/.vortex/tooling/src/vortex-push-db-gdrive b/.vortex/tooling/src/vortex-push-db-gdrive index 0f153bc016..c2a975752a 100755 --- a/.vortex/tooling/src/vortex-push-db-gdrive +++ b/.vortex/tooling/src/vortex-push-db-gdrive @@ -32,8 +32,9 @@ VORTEX_PUSH_DB_GDRIVE_DB_DIR="${VORTEX_PUSH_DB_GDRIVE_DB_DIR:-${VORTEX_DB_DIR:-. # Database dump file name. # -# The same name is used for the file in the Google Drive folder. -VORTEX_PUSH_DB_GDRIVE_DB_FILE="${VORTEX_PUSH_DB_GDRIVE_DB_FILE:-${VORTEX_DB_FILE:-db.sql}}" +# Defaults to the name the database is fetched and refreshed with. The same +# name is used for the file in the Google Drive folder. +VORTEX_PUSH_DB_GDRIVE_DB_FILE="${VORTEX_PUSH_DB_GDRIVE_DB_FILE:-${VORTEX_FETCH_DB_FILE:-${VORTEX_DB_FILE:-db.sql}}}" # ------------------------------------------------------------------------------ diff --git a/.vortex/tooling/src/vortex-push-db-s3 b/.vortex/tooling/src/vortex-push-db-s3 index 569555f344..fd6986b1ff 100755 --- a/.vortex/tooling/src/vortex-push-db-s3 +++ b/.vortex/tooling/src/vortex-push-db-s3 @@ -35,7 +35,9 @@ VORTEX_PUSH_DB_S3_PREFIX="${VORTEX_PUSH_DB_S3_PREFIX:-${VORTEX_FETCH_DB_S3_PREFI VORTEX_PUSH_DB_S3_DB_DIR="${VORTEX_PUSH_DB_S3_DB_DIR:-${VORTEX_DB_DIR:-./.data}}" # Database dump file name. -VORTEX_PUSH_DB_S3_DB_FILE="${VORTEX_PUSH_DB_S3_DB_FILE:-${VORTEX_DB_FILE:-db.sql}}" +# +# Defaults to the name the database is fetched and refreshed with. +VORTEX_PUSH_DB_S3_DB_FILE="${VORTEX_PUSH_DB_S3_DB_FILE:-${VORTEX_FETCH_DB_FILE:-${VORTEX_DB_FILE:-db.sql}}}" # Remote database dump file name. # diff --git a/.vortex/tooling/tests/unit/push-db-gdrive.bats b/.vortex/tooling/tests/unit/push-db-gdrive.bats index 0d7e6b4cc6..35773af729 100644 --- a/.vortex/tooling/tests/unit/push-db-gdrive.bats +++ b/.vortex/tooling/tests/unit/push-db-gdrive.bats @@ -295,3 +295,34 @@ fixture_gdrive_key() { popd >/dev/null } + +@test "push-db-gdrive: Push the database dump file the database is fetched with" { + pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 + + mkdir -p .data + echo "database content" >.data/db_fetched.sql + fixture_gdrive_key + + export VORTEX_PUSH_DB_GDRIVE_SERVICE_ACCOUNT_KEY="${BUILD_DIR}/gdrive.json" + export VORTEX_PUSH_DB_GDRIVE_FOLDER_ID="folder-123" + export VORTEX_FETCH_DB_FILE="db_fetched.sql" + unset VORTEX_PUSH_DB_GDRIVE_DB_DIR VORTEX_PUSH_DB_GDRIVE_DB_FILE + + mock_curl=$(mock_command "curl") + mock_set_output "${mock_curl}" '{"access_token":"test-token"}' 1 + mock_set_output "${mock_curl}" '{"files":[]}' 2 + mock_set_output "${mock_curl}" $'HTTP/2 200\r\nlocation: https://www.googleapis.com/upload/drive/v3/files?upload_id=session-5\r\n' 3 + mock_set_output "${mock_curl}" $'{"id":"file-new"}\n200' 4 + + run .vortex/tooling/src/vortex-push-db-gdrive + assert_success + assert_output_contains "Local file: ./.data/db_fetched.sql" + assert_output_contains "Remote file: db_fetched.sql" + + assert_string_contains "$(mock_get_call_args "${mock_curl}" 2)" "q=name = 'db_fetched.sql' and" + assert_string_contains "$(mock_get_call_args "${mock_curl}" 4)" "--upload-file ./.data/db_fetched.sql" + + rm -rf .data + + popd >/dev/null +} diff --git a/.vortex/tooling/tests/unit/push-db-s3.bats b/.vortex/tooling/tests/unit/push-db-s3.bats index 6c7ccbe3d3..c467452983 100644 --- a/.vortex/tooling/tests/unit/push-db-s3.bats +++ b/.vortex/tooling/tests/unit/push-db-s3.bats @@ -471,3 +471,41 @@ load ../_helper.bash popd >/dev/null } + +@test "push-db-s3: Push the database dump file the database is fetched with" { + pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 + + mkdir -p .data + echo "database content" >.data/db_fetched.sql + + declare -a STEPS=( + "[INFO] Started database dump push to S3." + + "Local file: ./.data/db_fetched.sql" + "Remote file: db_fetched.sql" + + "@curl * # 0 # 200" + + "[ OK ] Finished database dump push to S3." + ) + + unset S3_ACCESS_KEY S3_SECRET_KEY S3_BUCKET S3_REGION S3_PREFIX + export VORTEX_FETCH_DB_S3_ACCESS_KEY="" VORTEX_FETCH_DB_S3_SECRET_KEY="" VORTEX_FETCH_DB_S3_BUCKET="" VORTEX_FETCH_DB_S3_REGION="" VORTEX_FETCH_DB_S3_PREFIX="" + + export VORTEX_PUSH_DB_S3_ACCESS_KEY="testaccesskey" + export VORTEX_PUSH_DB_S3_SECRET_KEY="testsecretkey" + export VORTEX_PUSH_DB_S3_BUCKET="test-bucket" + export VORTEX_PUSH_DB_S3_REGION="ap-southeast-2" + export VORTEX_FETCH_DB_FILE="db_fetched.sql" + unset VORTEX_PUSH_DB_S3_DB_DIR VORTEX_PUSH_DB_S3_DB_FILE VORTEX_PUSH_DB_S3_REMOTE_FILE + + mocks="$(steps_run "setup")" + run .vortex/tooling/src/vortex-push-db-s3 + steps_run "assert" "${mocks[@]}" + + assert_success + + rm -rf .data + + popd >/dev/null +} From c4abf20a703f931d6e9e820db2133a36c5ff17e2 Mon Sep 17 00:00:00 2001 From: Alex Skrypnyk Date: Wed, 11 Nov 2026 11:21:07 +1100 Subject: [PATCH 10/21] Addressed code review: regenerated the variables reference. --- .vortex/docs/content/development/variables.mdx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.vortex/docs/content/development/variables.mdx b/.vortex/docs/content/development/variables.mdx index a111857861..34081e0e34 100644 --- a/.vortex/docs/content/development/variables.mdx +++ b/.vortex/docs/content/development/variables.mdx @@ -543,14 +543,14 @@ The list below is automatically generated with [Shellvar](https://github.com/ale | `VORTEX_PUSH_CONTAINER_REGISTRY_PASS` | The password to login into the container registry. | `${VORTEX_CONTAINER_REGISTRY_PASS}` | `.vortex/tooling/src/vortex-push-container-registry` | | `VORTEX_PUSH_CONTAINER_REGISTRY_USER` | The username to login into the container registry. | `${VORTEX_CONTAINER_REGISTRY_USER}` | `.vortex/tooling/src/vortex-push-container-registry` | | `VORTEX_PUSH_DB_GDRIVE_DB_DIR` | Directory with database dump file. | `./.data` | `.vortex/tooling/src/vortex-push-db-gdrive` | -| `VORTEX_PUSH_DB_GDRIVE_DB_FILE` | Database dump file name.

The same name is used for the file in the Google Drive folder. | `db.sql` | `.vortex/tooling/src/vortex-push-db-gdrive` | +| `VORTEX_PUSH_DB_GDRIVE_DB_FILE` | Database dump file name.

Defaults to the name the database is fetched and refreshed with. The same name is used for the file in the Google Drive folder. | `db.sql` | `.vortex/tooling/src/vortex-push-db-gdrive` | | `VORTEX_PUSH_DB_GDRIVE_FOLDER_ID` | Google Drive folder ID to push the database dump file to.

Defaults to the folder the database is fetched from. | `${VORTEX_FETCH_DB_GDRIVE_FOLDER_ID}` | `.vortex/tooling/src/vortex-push-db-gdrive` | | `VORTEX_PUSH_DB_GDRIVE_SERVICE_ACCOUNT_KEY` | Google Cloud service account key: the JSON content or a path to the JSON key file. | `${VORTEX_GDRIVE_SERVICE_ACCOUNT_KEY}` | `.vortex/tooling/src/vortex-push-db-gdrive` | | `VORTEX_PUSH_DB_PROCEED` | Proceed with push.

Set to `1` in the nightly CI job only, so the storage is refreshed once a day and never from a feature branch build. | `UNDEFINED` | `.vortex/tooling/src/vortex-push-db` | | `VORTEX_PUSH_DB_S3_ACCESS_KEY` | AWS access key. | `${S3_ACCESS_KEY}` | `.vortex/tooling/src/vortex-push-db-s3` | | `VORTEX_PUSH_DB_S3_BUCKET` | S3 bucket name. | `${S3_BUCKET}` | `.vortex/tooling/src/vortex-push-db-s3` | | `VORTEX_PUSH_DB_S3_DB_DIR` | Directory with database dump file. | `./.data` | `.vortex/tooling/src/vortex-push-db-s3` | -| `VORTEX_PUSH_DB_S3_DB_FILE` | Database dump file name. | `db.sql` | `.vortex/tooling/src/vortex-push-db-s3` | +| `VORTEX_PUSH_DB_S3_DB_FILE` | Database dump file name.

Defaults to the name the database is fetched and refreshed with. | `db.sql` | `.vortex/tooling/src/vortex-push-db-s3` | | `VORTEX_PUSH_DB_S3_PREFIX` | S3 prefix (path within the bucket). | `${S3_PREFIX}` | `.vortex/tooling/src/vortex-push-db-s3` | | `VORTEX_PUSH_DB_S3_REGION` | S3 region. | `${S3_REGION}` | `.vortex/tooling/src/vortex-push-db-s3` | | `VORTEX_PUSH_DB_S3_REMOTE_FILE` | Remote database dump file name.

Defaults to the database dump file name, which is the name the database is fetched with. | `${VORTEX_PUSH_DB_S3_DB_FILE}` | `.vortex/tooling/src/vortex-push-db-s3` | From cc0ef0628c5a32c64cb7a1160622df57f6b940b0 Mon Sep 17 00:00:00 2001 From: Alex Skrypnyk Date: Wed, 11 Nov 2026 14:34:51 +1100 Subject: [PATCH 11/21] Replaced the Google Drive push with an FTP push that keeps the current and the previous dump, for each database. --- .../db_fetch_source_gdrive/.env | 45 --- .../db_fetch_source_gdrive/.env.local.example | 10 - .../.github/workflows/build-test-deploy.yml | 33 -- .../migration_fetch_source_gdrive/.ahoy.yml | 51 --- .../migration_fetch_source_gdrive/.env | 30 -- .../.github/workflows/build-test-deploy.yml | 22 -- .../composer.json | 9 - .../docker-compose.yml | 52 --- .../scripts/provision-20-migration.sh | 187 ---------- .../phpunit/Drupal/DatabaseSettingsTest.php | 110 ------ .../Drupal/EnvironmentSettingsTest.php | 14 - .../tests/phpunit/Drupal/SettingsTestCase.php | 8 - .../migrations/ys_migrate_categories.yml | 29 -- .../custom/ys_migrate/ys_migrate.info.yml | 9 - .../web/sites/default/settings.migration.php | 21 -- .../web/sites/default/settings.php | 15 - .../tooling/playground/try-fetch-db-gdrive.sh | 57 --- .vortex/tooling/playground/try-push-db-ftp.sh | 71 ++++ .../tooling/playground/try-push-db-gdrive.sh | 65 ---- .vortex/tooling/src/vortex-fetch-db | 24 +- .vortex/tooling/src/vortex-fetch-db-gdrive | 135 ------- .vortex/tooling/src/vortex-push-db | 39 ++- .vortex/tooling/src/vortex-push-db-ftp | 143 ++++++++ .vortex/tooling/src/vortex-push-db-gdrive | 161 --------- .vortex/tooling/src/vortex-push-db-s3 | 218 ++++++++---- .vortex/tooling/src/vortex-refresh-db | 28 +- .vortex/tooling/tests/_helper.bash | 1 + .../tooling/tests/unit/fetch-db-gdrive.bats | 296 ---------------- .vortex/tooling/tests/unit/fetch-db.bats | 63 ++-- .vortex/tooling/tests/unit/push-db-ftp.bats | 204 +++++++++++ .../tooling/tests/unit/push-db-gdrive.bats | 328 ------------------ .vortex/tooling/tests/unit/push-db-s3.bats | 145 ++++++++ .vortex/tooling/tests/unit/push-db.bats | 86 ++++- .vortex/tooling/tests/unit/refresh-db.bats | 40 ++- 34 files changed, 916 insertions(+), 1833 deletions(-) delete mode 100644 .vortex/installer/tests/Fixtures/handler_process/db_fetch_source_gdrive/.env delete mode 100644 .vortex/installer/tests/Fixtures/handler_process/db_fetch_source_gdrive/.env.local.example delete mode 100644 .vortex/installer/tests/Fixtures/handler_process/db_fetch_source_gdrive/.github/workflows/build-test-deploy.yml delete mode 100644 .vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_gdrive/.ahoy.yml delete mode 100644 .vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_gdrive/.env delete mode 100644 .vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_gdrive/.github/workflows/build-test-deploy.yml delete mode 100644 .vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_gdrive/composer.json delete mode 100644 .vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_gdrive/docker-compose.yml delete mode 100755 .vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_gdrive/scripts/provision-20-migration.sh delete mode 100644 .vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_gdrive/tests/phpunit/Drupal/DatabaseSettingsTest.php delete mode 100644 .vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_gdrive/tests/phpunit/Drupal/EnvironmentSettingsTest.php delete mode 100644 .vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_gdrive/tests/phpunit/Drupal/SettingsTestCase.php delete mode 100644 .vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_gdrive/web/modules/custom/ys_migrate/migrations/ys_migrate_categories.yml delete mode 100644 .vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_gdrive/web/modules/custom/ys_migrate/ys_migrate.info.yml delete mode 100644 .vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_gdrive/web/sites/default/settings.migration.php delete mode 100644 .vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_gdrive/web/sites/default/settings.php delete mode 100755 .vortex/tooling/playground/try-fetch-db-gdrive.sh create mode 100755 .vortex/tooling/playground/try-push-db-ftp.sh delete mode 100755 .vortex/tooling/playground/try-push-db-gdrive.sh delete mode 100755 .vortex/tooling/src/vortex-fetch-db-gdrive create mode 100755 .vortex/tooling/src/vortex-push-db-ftp delete mode 100755 .vortex/tooling/src/vortex-push-db-gdrive delete mode 100644 .vortex/tooling/tests/unit/fetch-db-gdrive.bats create mode 100644 .vortex/tooling/tests/unit/push-db-ftp.bats delete mode 100644 .vortex/tooling/tests/unit/push-db-gdrive.bats diff --git a/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_gdrive/.env b/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_gdrive/.env deleted file mode 100644 index eb9fa4209c..0000000000 --- a/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_gdrive/.env +++ /dev/null @@ -1,45 +0,0 @@ -@@ -180,13 +180,22 @@ - VORTEX_DB_FILE=db.sql - - # Database fetch source. --VORTEX_FETCH_DB_SOURCE=url -+VORTEX_FETCH_DB_SOURCE=gdrive - --# Database dump file sourced from a URL. -+# Database dump file sourced from Google Drive. -+ -+# Google Drive folder ID with the database dump file. - # --# HTTP Basic Authentication credentials should be embedded into the value. --VORTEX_FETCH_DB_URL= -+# The ID is the last segment of the folder URL. The folder must be in a shared -+# drive and shared with the service account. -+VORTEX_FETCH_DB_GDRIVE_FOLDER_ID= - -+# Source of the database dump file the nightly CI job sanitizes and pushes to -+# the database fetch source: url, ftp, acquia or lagoon. The fetch script of -+# that source reads its own variables, such as VORTEX_FETCH_DB_URL for url. -+# Add them, and keep secrets in CI. -+# VORTEX_REFRESH_DB_SOURCE=url -+ - # Environment to fetch the database from. - # - # Applies to hosting environments. -@@ -254,17 +263,3 @@ - # with optional names in the format "email|name". - # Example: "to1@example.com|Jane Doe, to2@example.com|John Doe" - VORTEX_NOTIFY_EMAIL_RECIPIENTS="webmaster@star-wars.com|Webmaster" -- --################################################################################ --# DEMO # --################################################################################ --# # --# Override project-specific values for demonstration purposes. # --# Used to showcase Vortex without asking users to perform additional steps. # --# # --# Remove this section after completing database fetch integration. # --# # --################################################################################ -- --# URL of the database used for demonstration with URL database fetch type. --VORTEX_FETCH_DB_URL=https://github.com/drevops/vortex/releases/download/__VERSION__/db.demo.sql diff --git a/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_gdrive/.env.local.example b/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_gdrive/.env.local.example deleted file mode 100644 index 0f1fe53eb1..0000000000 --- a/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_gdrive/.env.local.example +++ /dev/null @@ -1,10 +0,0 @@ -@@ -32,3 +32,9 @@ - - # Always override existing fetched DB dump. - VORTEX_FETCH_DB_FORCE=1 -+ -+# Database dump file sourced from Google Drive. -+# Service account key: Google Cloud console -> IAM & Admin -> Service accounts -> Keys -> Add key -+ -+# Path to the Google Cloud service account JSON key file. Keep the file outside the project directory. -+VORTEX_GDRIVE_SERVICE_ACCOUNT_KEY= diff --git a/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_gdrive/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_gdrive/.github/workflows/build-test-deploy.yml deleted file mode 100644 index 88c93392b7..0000000000 --- a/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_gdrive/.github/workflows/build-test-deploy.yml +++ /dev/null @@ -1,33 +0,0 @@ -@@ -191,6 +191,7 @@ - VORTEX_CI_DB_CACHE_FALLBACK: "yes" - # Which branch to use as a source of DB caches. - VORTEX_CI_DB_CACHE_BRANCH: "develop" -+ VORTEX_GDRIVE_SERVICE_ACCOUNT_KEY: ${{ secrets.VORTEX_GDRIVE_SERVICE_ACCOUNT_KEY }} - - steps: - # Frees disk space by removing preinstalled toolchains unused by this project. Steps run in a container, so the host is reached via the Docker socket. -@@ -241,6 +242,8 @@ - { - echo "VORTEX_CI_DB_CACHE_FALLBACK=no" - echo "VORTEX_FRONTEND_BUILD_SKIP=1" -+ # Refresh the database dump in the database fetch source. -+ echo "VORTEX_PUSH_DB_PROCEED=1" - } >> "$GITHUB_ENV" - - - name: Create cache keys files for database caching -@@ -274,6 +277,15 @@ - run: | - if [ ! -f /tmp/fetch-db-fresh ]; then echo "==> Database fetch semaphore file is missing. DB export will not proceed."; exit 0; fi - ./vendor/bin/vortex-refresh-db -+ timeout-minutes: 30 -+ -+ # Push the freshly-exported sanitized database dump to the database fetch -+ # source. No-op unless a fresh DB was fetched (the fetch semaphore) and the -+ # push was requested (VORTEX_PUSH_DB_PROCEED, in the script). -+ - name: Push DB -+ run: | -+ if [ ! -f /tmp/fetch-db-fresh ]; then echo "==> No freshly fetched database. DB push will not proceed."; exit 0; fi -+ ./vendor/bin/vortex-push-db - timeout-minutes: 30 - - # Save cache per default branch and the timestamp. diff --git a/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_gdrive/.ahoy.yml b/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_gdrive/.ahoy.yml deleted file mode 100644 index 11f7a073e6..0000000000 --- a/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_gdrive/.ahoy.yml +++ /dev/null @@ -1,51 +0,0 @@ -@@ -32,6 +32,7 @@ - cmd: | - export COMPOSE_PROJECT_NAME=${COMPOSE_PROJECT_NAME:-${PWD##*/}} - export VORTEX_HOST_DB_PORT=$(docker compose port database 3306 2>/dev/null | cut -d : -f 2) -+ export VORTEX_HOST_DB2_PORT=$(docker compose port database2 3306 2>/dev/null | cut -d : -f 2) - export VORTEX_HOST_SOLR_PORT=$(docker compose port solr 8983 2>/dev/null | cut -d : -f 2) - export VORTEX_HOST_SELENIUM_VNC_PORT=$(docker compose port chrome 7900 2>/dev/null | cut -d : -f 2) - export VORTEX_HOST_HAS_SEQUELACE=$(uname -a | grep -i -q darwin && mdfind -name 'Sequel Ace' 2>/dev/null | grep -q "Ace" && echo 1 || true) -@@ -45,6 +46,14 @@ - open "mysql://${DATABASE_USERNAME:-drupal}:${DATABASE_PASSWORD:-drupal}@127.0.0.1:${VORTEX_HOST_DB_PORT}/drupal" -a "Sequel Ace" \ - || echo "Not a supported OS or Sequel Ace is not installed." - -+ db2: -+ usage: Open DB2 in Sequel Ace. -+ cmd: | -+ uname -a | grep -i -q darwin && test -d "${VORTEX_HOST_SEQUELACE_PATH:-/Applications/Sequel Ace.app}" && \ -+ VORTEX_HOST_DB2_PORT=$(docker compose port database2 3306 2>/dev/null | cut -d : -f 2) && \ -+ open "mysql://${DATABASE2_USERNAME:-drupal}:${DATABASE2_PASSWORD:-drupal}@127.0.0.1:${VORTEX_HOST_DB2_PORT}/drupal" -a "Sequel Ace" \ -+ || echo "Not a supported OS or Sequel Ace is not installed." -+ - # ---------------------------------------------------------------------------- - # Container commands. - # ---------------------------------------------------------------------------- -@@ -122,6 +131,13 @@ - case " $* " in *" --fresh "*) export VORTEX_FETCH_DB_FRESH=1 VORTEX_FETCH_DB_FORCE=1;; esac - ./vendor/bin/vortex-fetch-db - -+ fetch-db2: -+ usage: Fetch second database (migration). Run with "--fresh" to force. -+ aliases: [download-db2, db2-download, db2-fetch] -+ cmd: | -+ case " $* " in *" --fresh "*) export VORTEX_FETCH_DB2_FRESH=1 VORTEX_FETCH_DB2_FORCE=1;; esac -+ VORTEX_DB_INDEX=2 ./vendor/bin/vortex-fetch-db -+ - fetch-files: - usage: Fetch public files. - aliases: [download-files, files-download, files-fetch] -@@ -134,6 +150,13 @@ - docker compose rm --force --stop --volumes database - docker compose build database - ahoy up -+ -+ reload-db2: -+ usage: Reload the migration database container using local database image. -+ cmd: | -+ ahoy confirm "Running this command will replace your current database. Are you sure?" || exit 0 -+ docker compose rm --force --stop --volumes database2 -+ ahoy up -- --build database2 - - provision: - usage: Provision a site from the database dump or profile. diff --git a/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_gdrive/.env b/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_gdrive/.env deleted file mode 100644 index bce43c946e..0000000000 --- a/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_gdrive/.env +++ /dev/null @@ -1,30 +0,0 @@ -@@ -195,6 +195,29 @@ - VORTEX_FETCH_DB_ENVIRONMENT=prod - - ################################################################################ -+# SECOND DATABASE (MIGRATION) # -+################################################################################ -+ -+# Second database for migrations. -+# See settings.migration.php for database credentials. -+ -+# Second database dump file name. -+VORTEX_FETCH_DB2_FILE=db2.sql -+ -+# Second database fetch source. -+VORTEX_FETCH_DB2_SOURCE=gdrive -+ -+# Second database dump file sourced from Google Drive. -+ -+# Google Drive folder ID with the second database dump file. -+VORTEX_FETCH_DB2_GDRIVE_FOLDER_ID= -+ -+# Environment to fetch the second database from. -+# -+# Applies to hosting environments. -+VORTEX_FETCH_DB2_ENVIRONMENT=prod -+ -+################################################################################ - # FILES SOURCE # - ################################################################################ - diff --git a/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_gdrive/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_gdrive/.github/workflows/build-test-deploy.yml deleted file mode 100644 index 4292b4f549..0000000000 --- a/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_gdrive/.github/workflows/build-test-deploy.yml +++ /dev/null @@ -1,22 +0,0 @@ -@@ -270,6 +270,9 @@ - echo "db_hash=${{ hashFiles('.data') }}" >> "$GITHUB_ENV" - timeout-minutes: 30 - -+ - name: Fetch migration DB -+ run: VORTEX_DB_INDEX=2 ./vendor/bin/vortex-fetch-db -+ - - name: Export DB - run: | - if [ ! -f /tmp/fetch-db-fresh ]; then echo "==> Database fetch semaphore file is missing. DB export will not proceed."; exit 0; fi -@@ -431,6 +434,11 @@ - docker compose exec cli mkdir -p .data - docker compose --progress quiet cp -L .data/db.sql cli:/app/.data/db.sql - rm -f .data/db.sql -+ fi -+ if [ -f ".data/${VORTEX_FETCH_DB2_FILE:-db2.sql}" ]; then -+ docker compose exec -T cli mkdir -p .data -+ docker compose --progress quiet cp -L ".data/${VORTEX_FETCH_DB2_FILE:-db2.sql}" cli:"/app/.data/${VORTEX_FETCH_DB2_FILE:-db2.sql}" -+ rm -f ".data/${VORTEX_FETCH_DB2_FILE:-db2.sql}" - fi - docker compose exec $(env | cut -f1 -d= | sed 's/^/-e /') -T cli ./vendor/bin/vortex-provision - timeout-minutes: 30 diff --git a/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_gdrive/composer.json b/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_gdrive/composer.json deleted file mode 100644 index b90a23c71c..0000000000 --- a/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_gdrive/composer.json +++ /dev/null @@ -1,9 +0,0 @@ -@@ -26,6 +26,8 @@ - "drupal/linkit": "__VERSION__", - "drupal/menu_trail_by_path": "__VERSION__", - "drupal/metatag": "__VERSION__", -+ "drupal/migrate_plus": "__VERSION__", -+ "drupal/migrate_tools": "__VERSION__", - "drupal/navigation_extra_tools": "__VERSION__", - "drupal/pathauto": "__VERSION__", - "drupal/redirect": "__VERSION__", diff --git a/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_gdrive/docker-compose.yml b/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_gdrive/docker-compose.yml deleted file mode 100644 index 70ac2fcbc8..0000000000 --- a/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_gdrive/docker-compose.yml +++ /dev/null @@ -1,52 +0,0 @@ -@@ -57,6 +57,16 @@ - DRUPAL_SHIELD_PASS: ${DRUPAL_SHIELD_PASS:-} - # Redis integration flag. - DRUPAL_REDIS_ENABLED: ${DRUPAL_REDIS_ENABLED:-} -+ # Local migration database host (not used in production). -+ DATABASE2_HOST: database2 -+ # Local migration database name (not used in production). -+ DATABASE2_NAME: drupal -+ # Local migration database user (not used in production). -+ DATABASE2_USERNAME: drupal -+ # Local migration database password (not used in production). -+ DATABASE2_PASSWORD: drupal -+ # Local migration database port (not used in production). -+ DATABASE2_PORT: 3306 - - # The default user under which the containers should run. - x-user: &default-user -@@ -155,6 +165,20 @@ - ports: - - '3306' # Database port in a container. Find port on host with `ahoy info` or `docker compose port database 3306`. - -+ database2: -+ image: "${VORTEX_DB2_IMAGE:-uselagoon/mysql-8.4:__VERSION__}" # Use custom database image (if defined) or fallback to standard database image. -+ environment: -+ <<: *default-environment -+ MYSQL_DATABASE: drupal -+ MYSQL_USER: drupal -+ MYSQL_PASSWORD: drupal -+ <<: *default-user -+ # dclint disable-line no-duplicate-exported-ports # container-only port, published on a random host port -+ ports: -+ - '3306' # Database port in a container. Find port on host with `ahoy info` or `docker compose port database2 3306`. -+ labels: -+ lagoon.type: mariadb # See https://docs.lagoon.sh/using-lagoon-advanced/service-types/ -+ - redis: - image: uselagoon/valkey-8:__VERSION__ - -@@ -198,11 +222,13 @@ - depends_on: - - clamav - - database -+ - database2 - - redis - - solr - command: - - clamav:3310 - - database:3306 -+ - database2:3306 - - redis:6379 - - solr:8983 - - curl -s "http://solr:8983/solr/drupal/select?q=*:*&rows=0&wt=json" | grep response diff --git a/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_gdrive/scripts/provision-20-migration.sh b/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_gdrive/scripts/provision-20-migration.sh deleted file mode 100755 index 3000594760..0000000000 --- a/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_gdrive/scripts/provision-20-migration.sh +++ /dev/null @@ -1,187 +0,0 @@ -#!/usr/bin/env bash -## -# Import migration database and run migrations. -# -# This script is called during site provisioning via provision.sh. -# Customize the migration names at the bottom of this file. -# -# shellcheck disable=SC2086 - -set -eu -[ "${VORTEX_DEBUG-}" = "1" ] && set -x - -# Skip all migrations. -DRUPAL_MIGRATION_SKIP="${DRUPAL_MIGRATION_SKIP:-0}" - -# Skip rollback of migrations before import. -DRUPAL_MIGRATION_ROLLBACK_SKIP="${DRUPAL_MIGRATION_ROLLBACK_SKIP:-1}" - -# Limit the number of entities to import. Set to 'all' to import all. -DRUPAL_MIGRATION_IMPORT_LIMIT="${DRUPAL_MIGRATION_IMPORT_LIMIT:-50}" - -# Update already imported entities during migration. -DRUPAL_MIGRATION_UPDATE="${DRUPAL_MIGRATION_UPDATE:-0}" - -# Feedback frequency for migration progress. -DRUPAL_MIGRATION_FEEDBACK="${DRUPAL_MIGRATION_FEEDBACK:-50}" - -# Import migration source database. Set to 1 to import, 0 to skip. -DRUPAL_MIGRATION_SOURCE_DB_IMPORT="${DRUPAL_MIGRATION_SOURCE_DB_IMPORT:-${VORTEX_PROVISION_OVERRIDE_DB:-0}}" - -# Table name to probe in the source database to verify it is not corrupted. -DRUPAL_MIGRATION_SOURCE_DB_PROBE_TABLE="${DRUPAL_MIGRATION_SOURCE_DB_PROBE_TABLE:-categories}" - -# Disable the search server while migrations run. Set to 0 to keep indexing. -DRUPAL_MIGRATION_SEARCH_DISABLE="${DRUPAL_MIGRATION_SEARCH_DISABLE:-1}" - -# Directory with database dump file. -VORTEX_DB_DIR="${VORTEX_DB_DIR:-./.data}" - -# Migration database dump file name. -VORTEX_FETCH_DB2_FILE="${VORTEX_FETCH_DB2_FILE:-db2.sql}" - -# ------------------------------------------------------------------------------ - -# @formatter:off -info() { printf " ==> %s\n" "${1}"; } -note() { printf " %s\n" "${1}"; } -task() { printf " > %s\n" "${1}"; } -pass() { printf " + %s\n" "${1}"; } -fail() { printf " ! %s\n" "${1}"; exit "${2:-1}"; } -# @formatter:on - -drush() { ./vendor/bin/drush -y "$@"; } - -# ------------------------------------------------------------------------------ - -info "Started migration operations." - -environment="$(drush php:eval "print \Drupal\Core\Site\Settings::get('environment');")" -note "Environment: ${environment}" - -# Skip migrations in production. -if [ "${environment}" = "prod" ]; then - DRUPAL_MIGRATION_SKIP=1 -fi - -note "Migration skip: ${DRUPAL_MIGRATION_SKIP}" -note "Migration limit: ${DRUPAL_MIGRATION_IMPORT_LIMIT}" -note "Migration skip rollback: ${DRUPAL_MIGRATION_ROLLBACK_SKIP}" -note "Migration update: ${DRUPAL_MIGRATION_UPDATE}" -note "Migration feedback: ${DRUPAL_MIGRATION_FEEDBACK}" -note "Source DB import: ${DRUPAL_MIGRATION_SOURCE_DB_IMPORT}" -note "Search disable: ${DRUPAL_MIGRATION_SEARCH_DISABLE}" -echo - -if [ "${DRUPAL_MIGRATION_SKIP}" = "1" ]; then - info "Skipped migrations. DRUPAL_MIGRATION_SKIP is set to 1." - exit 0 -fi - -# Helper function to run a single migration with configured options. -run_migration() { - local migration_name="${1}" - shift - - drush migrate:reset-status "${migration_name}" || fail "Failed to reset migration status for ${migration_name}." - - task "Running migration: ${migration_name}." - local opts=() - - opts+=("--feedback=${DRUPAL_MIGRATION_FEEDBACK}") - - if printf '%s\n' "${DRUPAL_MIGRATION_IMPORT_LIMIT}" | grep -q '^[0-9][0-9]*$' && [ "${DRUPAL_MIGRATION_IMPORT_LIMIT}" -gt 0 ]; then - opts+=("--limit=${DRUPAL_MIGRATION_IMPORT_LIMIT}") - fi - - if [ "${DRUPAL_MIGRATION_UPDATE}" = "1" ]; then - opts+=("--update") - fi - - # Add any additional arguments passed to the function. - opts+=("$@") - - drush migrate:import "${opts[@]}" "${migration_name}" || { - drush migrate:messages "${migration_name}" - fail "Failed to run migration ${migration_name}." - } - - pass "Migrated: ${migration_name}." -} - -# Detect if existing migration source database is corrupted. -if [ "${DRUPAL_MIGRATION_SOURCE_DB_IMPORT}" != "1" ]; then - note "Source database import is set to be skipped. Checking existing database." - task "Probing for '${DRUPAL_MIGRATION_SOURCE_DB_PROBE_TABLE}' table in the source database." - if drush sql:query --database=migrate "SELECT COUNT(*) FROM ${DRUPAL_MIGRATION_SOURCE_DB_PROBE_TABLE}" >/dev/null 2>&1; then - pass "Source database is intact." - else - note "Migration source database is corrupted or empty. Re-importing." - DRUPAL_MIGRATION_SOURCE_DB_IMPORT=1 - pass "Marked the source database for re-import." - fi -fi - -# Import the migration source database from the dump file. -if [ "${DRUPAL_MIGRATION_SOURCE_DB_IMPORT}" = "1" ]; then - task "Importing migration source database." - - [ ! -f "${VORTEX_DB_DIR}/${VORTEX_FETCH_DB2_FILE}" ] && fail "Migration source database file not found. Please run 'ahoy fetch-db2'." - - drush sql:drop --database=migrate - # shellcheck disable=SC2091 - $(drush sql:connect --database=migrate) <"${VORTEX_DB_DIR}/${VORTEX_FETCH_DB2_FILE}" - - pass "Imported migration source database." -else - note "Using existing migration source database." -fi - -task "Verifying migration source database." -if ! drush sql:query --database=migrate "SELECT COUNT(*) FROM ${DRUPAL_MIGRATION_SOURCE_DB_PROBE_TABLE}" >/dev/null 2>&1; then - drush sql:query --database=migrate "SHOW TABLES;" - fail "Migration source database is corrupted." -fi -pass "Verified migration source database." - -# Enable custom migration modules. -task "Enabling migration modules." -drush pm:install ys_migrate -pass "Enabled migration modules." - -info "Starting migrations." - -if [ "${DRUPAL_MIGRATION_SEARCH_DISABLE}" = "1" ]; then - task "Disabling Search API Solr server." - drush search-api:server-disable solr || true - pass "Disabled Search API Solr server." -fi - -if [ "${DRUPAL_MIGRATION_ROLLBACK_SKIP}" = "1" ]; then - note "Skipped rollback of all migrations." -else - task "Rolling back all migrations." - drush migrate:rollback --all || true - pass "Rolled back all migrations." -fi -echo - -# ------------------------------------------------------------------------------ -# Add your migrations below. -# ------------------------------------------------------------------------------ - -run_migration ys_migrate_categories - -echo -note "Finished migrations." - -if [ "${DRUPAL_MIGRATION_SEARCH_DISABLE}" = "1" ]; then - task "Enabling Search API Solr server." - drush search-api:server-enable solr || true - drush search-api:enable-all || true - pass "Enabled Search API Solr server." -fi - -drush migrate:status - -info "Finished migration operations." diff --git a/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_gdrive/tests/phpunit/Drupal/DatabaseSettingsTest.php b/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_gdrive/tests/phpunit/Drupal/DatabaseSettingsTest.php deleted file mode 100644 index 956cd535d4..0000000000 --- a/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_gdrive/tests/phpunit/Drupal/DatabaseSettingsTest.php +++ /dev/null @@ -1,110 +0,0 @@ -@@ -47,6 +47,17 @@ - 'prefix' => '', - ], - ], -+ 'migrate' => [ -+ 'default' => [ -+ 'database' => 'drupal', -+ 'username' => 'drupal', -+ 'password' => 'drupal', -+ 'host' => 'localhost', -+ 'port' => '', -+ 'prefix' => '', -+ 'driver' => 'mysql', -+ ], -+ ], - ], - ]; - -@@ -74,6 +85,17 @@ - 'prefix' => '', - ], - ], -+ 'migrate' => [ -+ 'default' => [ -+ 'database' => 'drupal', -+ 'username' => 'drupal', -+ 'password' => 'drupal', -+ 'host' => 'localhost', -+ 'port' => '', -+ 'prefix' => '', -+ 'driver' => 'mysql', -+ ], -+ ], - ], - ]; - -@@ -101,6 +123,17 @@ - 'prefix' => '', - ], - ], -+ 'migrate' => [ -+ 'default' => [ -+ 'database' => 'drupal', -+ 'username' => 'drupal', -+ 'password' => 'drupal', -+ 'host' => 'localhost', -+ 'port' => '', -+ 'prefix' => '', -+ 'driver' => 'mysql', -+ ], -+ ], - ], - ]; - -@@ -128,9 +161,55 @@ - 'prefix' => '', - ], - ], -+ 'migrate' => [ -+ 'default' => [ -+ 'database' => 'drupal', -+ 'username' => 'drupal', -+ 'password' => 'drupal', -+ 'host' => 'localhost', -+ 'port' => '', -+ 'prefix' => '', -+ 'driver' => 'mysql', -+ ], -+ ], - ], - ]; - -+ yield [ -+ [ -+ 'DATABASE2_NAME' => 'migrate_db_name', -+ 'DATABASE2_USERNAME' => 'migrate_db_user', -+ 'DATABASE2_PASSWORD' => 'migrate_db_pass', -+ 'DATABASE2_HOST' => 'migrate_db_host', -+ 'DATABASE2_PORT' => '3307', -+ ], -+ [ -+ 'default' => [ -+ 'default' => [ -+ 'database' => 'drupal', -+ 'username' => 'drupal', -+ 'password' => 'drupal', -+ 'host' => 'localhost', -+ 'port' => '3306', -+ 'charset' => 'utf8mb4', -+ 'collation' => 'utf8mb4_general_ci', -+ 'driver' => 'mysql', -+ 'prefix' => '', -+ ], -+ ], -+ 'migrate' => [ -+ 'default' => [ -+ 'database' => 'migrate_db_name', -+ 'username' => 'migrate_db_user', -+ 'password' => 'migrate_db_pass', -+ 'host' => 'migrate_db_host', -+ 'port' => '3307', -+ 'prefix' => '', -+ 'driver' => 'mysql', -+ ], -+ ], -+ ], -+ ]; - } - - } diff --git a/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_gdrive/tests/phpunit/Drupal/EnvironmentSettingsTest.php b/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_gdrive/tests/phpunit/Drupal/EnvironmentSettingsTest.php deleted file mode 100644 index 8adb9246e8..0000000000 --- a/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_gdrive/tests/phpunit/Drupal/EnvironmentSettingsTest.php +++ /dev/null @@ -1,14 +0,0 @@ -@@ -163,6 +163,13 @@ - $databases['default']['default']['collation'] = 'utf8_general_ci'; - $databases['default']['default']['driver'] = 'mysql'; - $databases['default']['default']['prefix'] = ''; -+ $databases['migrate']['default']['database'] = 'drupal'; -+ $databases['migrate']['default']['username'] = 'drupal'; -+ $databases['migrate']['default']['password'] = 'drupal'; -+ $databases['migrate']['default']['host'] = 'localhost'; -+ $databases['migrate']['default']['port'] = ''; -+ $databases['migrate']['default']['prefix'] = ''; -+ $databases['migrate']['default']['driver'] = 'mysql'; - $this->assertEquals($databases, $this->databases); - - // Verify key config overrides. diff --git a/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_gdrive/tests/phpunit/Drupal/SettingsTestCase.php b/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_gdrive/tests/phpunit/Drupal/SettingsTestCase.php deleted file mode 100644 index 350cc85c04..0000000000 --- a/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_gdrive/tests/phpunit/Drupal/SettingsTestCase.php +++ /dev/null @@ -1,8 +0,0 @@ -@@ -62,6 +62,7 @@ - const ALLOWED_ENV_VARS = [ - // Service variables. - 'DATABASE_', -+ 'DATABASE2_', - 'REDIS_', - 'CLAMAV_', - 'COMPOSE_', diff --git a/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_gdrive/web/modules/custom/ys_migrate/migrations/ys_migrate_categories.yml b/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_gdrive/web/modules/custom/ys_migrate/migrations/ys_migrate_categories.yml deleted file mode 100644 index 0494698e5d..0000000000 --- a/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_gdrive/web/modules/custom/ys_migrate/migrations/ys_migrate_categories.yml +++ /dev/null @@ -1,29 +0,0 @@ -id: ys_migrate_categories -label: 'Migrate categories to tags' -migration_tags: - - ys_migrate - -source: - plugin: table - key: migrate - table_name: categories - id_fields: - id: - type: integer - fields: - id: id - name: name - description: description - -process: - name: name - description/value: description - description/format: - plugin: default_value - default_value: plain_text - vid: - plugin: default_value - default_value: tags - -destination: - plugin: 'entity:taxonomy_term' diff --git a/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_gdrive/web/modules/custom/ys_migrate/ys_migrate.info.yml b/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_gdrive/web/modules/custom/ys_migrate/ys_migrate.info.yml deleted file mode 100644 index 96c67179ac..0000000000 --- a/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_gdrive/web/modules/custom/ys_migrate/ys_migrate.info.yml +++ /dev/null @@ -1,9 +0,0 @@ -name: star wars Migrate -type: module -description: Migrations for star wars site. -core_version_requirement: ^11 -package: star_wars -dependencies: - - drupal:migrate - - migrate_plus:migrate_plus - - migrate_tools:migrate_tools diff --git a/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_gdrive/web/sites/default/settings.migration.php b/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_gdrive/web/sites/default/settings.migration.php deleted file mode 100644 index 660869b39f..0000000000 --- a/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_gdrive/web/sites/default/settings.migration.php +++ /dev/null @@ -1,21 +0,0 @@ - (getenv('DATABASE2_NAME') ?: getenv('DATABASE2_DATABASE')) ?: 'drupal', - 'username' => getenv('DATABASE2_USERNAME') ?: 'drupal', - 'password' => getenv('DATABASE2_PASSWORD') ?: 'drupal', - 'host' => getenv('DATABASE2_HOST') ?: 'localhost', - 'port' => getenv('DATABASE2_PORT') ?: '', - 'prefix' => '', - 'driver' => 'mysql', -]; diff --git a/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_gdrive/web/sites/default/settings.php b/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_gdrive/web/sites/default/settings.php deleted file mode 100644 index c04331d75e..0000000000 --- a/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_gdrive/web/sites/default/settings.php +++ /dev/null @@ -1,15 +0,0 @@ -@@ -167,6 +167,14 @@ - } - - //////////////////////////////////////////////////////////////////////////////// -+/// MIGRATION SETTINGS /// -+//////////////////////////////////////////////////////////////////////////////// -+// @todo Migration. Remove when migration configuration is no longer required. -+if (file_exists($app_root . '/' . $site_path . '/settings.migration.php')) { -+ require $app_root . '/' . $site_path . '/settings.migration.php'; -+} -+ -+//////////////////////////////////////////////////////////////////////////////// - /// LOCAL OVERRIDE /// - //////////////////////////////////////////////////////////////////////////////// - // @see https://www.vortextemplate.com/docs/development/settings#local-overrides diff --git a/.vortex/tooling/playground/try-fetch-db-gdrive.sh b/.vortex/tooling/playground/try-fetch-db-gdrive.sh deleted file mode 100755 index 43fd135f42..0000000000 --- a/.vortex/tooling/playground/try-fetch-db-gdrive.sh +++ /dev/null @@ -1,57 +0,0 @@ -#!/usr/bin/env bash -## -# Manual test script for Google Drive database fetch. -# -# Fetches a database dump file from a Google Drive folder using -# fetch-db-gdrive. -# -# Usage: -# export GDRIVE_SERVICE_ACCOUNT_KEY="/path/to/service-account.json" -# export GDRIVE_FOLDER_ID="your-folder-id" -# ./try-fetch-db-gdrive.sh - -set -eu - -# Source secrets if available. -if [ -f "$HOME/.profile.secrets" ]; then - # shellcheck disable=SC1091 - . "$HOME/.profile.secrets" -fi - -# Get the directory of this script and navigate to project root. -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -PROJECT_ROOT="$(cd "${SCRIPT_DIR}/../../.." && pwd)" - -# Google Drive credentials and settings. -GDRIVE_SERVICE_ACCOUNT_KEY="${GDRIVE_SERVICE_ACCOUNT_KEY:-}" -GDRIVE_FOLDER_ID="${GDRIVE_FOLDER_ID:-}" - -# Validate required variables. -if [ -z "${GDRIVE_SERVICE_ACCOUNT_KEY}" ]; then - echo "Error: GDRIVE_SERVICE_ACCOUNT_KEY environment variable is required" - echo "Usage: export GDRIVE_SERVICE_ACCOUNT_KEY=\"/path/to/service-account.json\" && $0" - exit 1 -fi - -if [ -z "${GDRIVE_FOLDER_ID}" ]; then - echo "Error: GDRIVE_FOLDER_ID environment variable is required" - echo "Usage: export GDRIVE_FOLDER_ID=\"your-folder-id\" && $0" - exit 1 -fi - -echo "Testing Google Drive database fetch..." -echo "" -echo "Folder ID: ${GDRIVE_FOLDER_ID}" -echo "DB file: ${VORTEX_DB_FILE:-db.sql}" -echo "Local dir: ${VORTEX_DB_DIR:-./.data}" -echo "" - -cd "${PROJECT_ROOT}" || exit 1 - -export VORTEX_FETCH_DB_GDRIVE_SERVICE_ACCOUNT_KEY="${GDRIVE_SERVICE_ACCOUNT_KEY}" -export VORTEX_FETCH_DB_GDRIVE_FOLDER_ID="${GDRIVE_FOLDER_ID}" - -./vendor/drevops/vortex-tooling/src/vortex-fetch-db-gdrive - -echo "" -echo "Fetch complete!" diff --git a/.vortex/tooling/playground/try-push-db-ftp.sh b/.vortex/tooling/playground/try-push-db-ftp.sh new file mode 100755 index 0000000000..dda38af769 --- /dev/null +++ b/.vortex/tooling/playground/try-push-db-ftp.sh @@ -0,0 +1,71 @@ +#!/usr/bin/env bash +## +# Manual test script for FTP database push. +# +# Uploads a database dump file to an FTP server using push-db-ftp. Run it +# twice to check that the second run keeps the first dump as +# 'db-previous.sql' next to the new 'db.sql'. +# +# Usage: +# export FTP_HOST="ftp.example.com" +# export FTP_PORT="21" +# export FTP_USER="your-user" +# export FTP_PASS="your-password" +# export FTP_FILE="path/to/db.sql" +# ./try-push-db-ftp.sh + +set -eu + +# Source secrets if available. +if [ -f "$HOME/.profile.secrets" ]; then + # shellcheck disable=SC1091 + . "$HOME/.profile.secrets" +fi + +# Get the directory of this script and navigate to project root. +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_ROOT="$(cd "${SCRIPT_DIR}/../../.." && pwd)" + +# FTP credentials and settings. +FTP_HOST="${FTP_HOST:-}" +FTP_PORT="${FTP_PORT:-21}" +FTP_USER="${FTP_USER:-}" +FTP_PASS="${FTP_PASS:-}" +FTP_FILE="${FTP_FILE:-db.sql}" + +# Validate required variables. +for name in FTP_HOST FTP_USER FTP_PASS; do + if [ -z "${!name}" ]; then + echo "Error: ${name} environment variable is required" + echo "Usage: export ${name}=\"...\" && $0" + exit 1 + fi +done + +cd "${PROJECT_ROOT}" || exit 1 + +# Create a test dump file when none exists. +db_dir="${VORTEX_DB_DIR:-./.data}" +db_file="${VORTEX_DB_FILE:-db.sql}" +if [ ! -f "${db_dir}/${db_file}" ]; then + mkdir -p "${db_dir}" + echo "-- Test dump created by try-push-db-ftp.sh on $(date -u)" >"${db_dir}/${db_file}" +fi + +echo "Testing FTP database push..." +echo "" +echo "FTP host: ${FTP_HOST}:${FTP_PORT}" +echo "FTP file: ${FTP_FILE}" +echo "Local file: ${db_dir}/${db_file}" +echo "" + +export VORTEX_PUSH_DB_FTP_HOST="${FTP_HOST}" +export VORTEX_PUSH_DB_FTP_PORT="${FTP_PORT}" +export VORTEX_PUSH_DB_FTP_USER="${FTP_USER}" +export VORTEX_PUSH_DB_FTP_PASS="${FTP_PASS}" +export VORTEX_PUSH_DB_FTP_FILE="${FTP_FILE}" + +./vendor/drevops/vortex-tooling/src/vortex-push-db-ftp + +echo "" +echo "Push complete!" diff --git a/.vortex/tooling/playground/try-push-db-gdrive.sh b/.vortex/tooling/playground/try-push-db-gdrive.sh deleted file mode 100755 index 9cddbab46f..0000000000 --- a/.vortex/tooling/playground/try-push-db-gdrive.sh +++ /dev/null @@ -1,65 +0,0 @@ -#!/usr/bin/env bash -## -# Manual test script for Google Drive database push. -# -# Uploads a database dump file to a Google Drive folder in a shared drive -# using push-db-gdrive. Run it twice to check that the second run replaces -# the file instead of creating a duplicate. -# -# Usage: -# export GDRIVE_SERVICE_ACCOUNT_KEY="/path/to/service-account.json" -# export GDRIVE_FOLDER_ID="your-folder-id" -# ./try-push-db-gdrive.sh - -set -eu - -# Source secrets if available. -if [ -f "$HOME/.profile.secrets" ]; then - # shellcheck disable=SC1091 - . "$HOME/.profile.secrets" -fi - -# Get the directory of this script and navigate to project root. -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -PROJECT_ROOT="$(cd "${SCRIPT_DIR}/../../.." && pwd)" - -# Google Drive credentials and settings. -GDRIVE_SERVICE_ACCOUNT_KEY="${GDRIVE_SERVICE_ACCOUNT_KEY:-}" -GDRIVE_FOLDER_ID="${GDRIVE_FOLDER_ID:-}" - -# Validate required variables. -if [ -z "${GDRIVE_SERVICE_ACCOUNT_KEY}" ]; then - echo "Error: GDRIVE_SERVICE_ACCOUNT_KEY environment variable is required" - echo "Usage: export GDRIVE_SERVICE_ACCOUNT_KEY=\"/path/to/service-account.json\" && $0" - exit 1 -fi - -if [ -z "${GDRIVE_FOLDER_ID}" ]; then - echo "Error: GDRIVE_FOLDER_ID environment variable is required" - echo "Usage: export GDRIVE_FOLDER_ID=\"your-folder-id\" && $0" - exit 1 -fi - -cd "${PROJECT_ROOT}" || exit 1 - -# Create a test dump file when none exists. -db_dir="${VORTEX_DB_DIR:-./.data}" -db_file="${VORTEX_DB_FILE:-db.sql}" -if [ ! -f "${db_dir}/${db_file}" ]; then - mkdir -p "${db_dir}" - echo "-- Test dump created by try-push-db-gdrive.sh on $(date -u)" >"${db_dir}/${db_file}" -fi - -echo "Testing Google Drive database push..." -echo "" -echo "Folder ID: ${GDRIVE_FOLDER_ID}" -echo "Local file: ${db_dir}/${db_file}" -echo "" - -export VORTEX_PUSH_DB_GDRIVE_SERVICE_ACCOUNT_KEY="${GDRIVE_SERVICE_ACCOUNT_KEY}" -export VORTEX_PUSH_DB_GDRIVE_FOLDER_ID="${GDRIVE_FOLDER_ID}" - -./vendor/drevops/vortex-tooling/src/vortex-push-db-gdrive - -echo "" -echo "Push complete!" diff --git a/.vortex/tooling/src/vortex-fetch-db b/.vortex/tooling/src/vortex-fetch-db index f1abf35058..b3f93c2f6e 100755 --- a/.vortex/tooling/src/vortex-fetch-db +++ b/.vortex/tooling/src/vortex-fetch-db @@ -55,17 +55,20 @@ info "Started database${_db_index:+ ${_db_index}} fetch." [ "${VORTEX_FETCH_DB_PROCEED}" != "1" ] && pass "Skipped database fetch as VORTEX_FETCH_DB_PROCEED is not set to 1." && exit 0 -# A dump store ('gdrive' or 's3') receives the dump that the nightly refresh -# pushes, so with a push requested the dump comes from its upstream source. -# The store may hold no dump yet on the first push. -if [ -z "${_db_index}" ] && [ "${VORTEX_PUSH_DB_PROCEED:-}" = "1" ]; then +# An 'ftp' or 's3' source with an upstream source in VORTEX_REFRESH_DB_SOURCE +# is a dump store: it receives the dump that the nightly refresh pushes, so +# with a push requested the dump comes from the upstream source. The store +# may hold no dump yet on the first push. +_v="VORTEX_REFRESH_DB${_db_index}_SOURCE" +refresh_source="${!_v:-}" +if [ "${VORTEX_PUSH_DB_PROCEED:-}" = "1" ] && [ -n "${refresh_source}" ]; then case "${VORTEX_FETCH_DB_SOURCE}" in - gdrive | s3) - case "${VORTEX_REFRESH_DB_SOURCE:-}" in - "" | gdrive | s3 | container_registry | none) fail "The database dump file pushed to ${VORTEX_FETCH_DB_SOURCE} is fetched from an upstream source, but VORTEX_REFRESH_DB_SOURCE is \"${VORTEX_REFRESH_DB_SOURCE:-}\". Set it to the source of the dump file: url, ftp, acquia or lagoon." ;; + ftp | s3) + case "${refresh_source}" in + "${VORTEX_FETCH_DB_SOURCE}" | container_registry | none) fail "The database dump file pushed to ${VORTEX_FETCH_DB_SOURCE} is fetched from an upstream source, but VORTEX_REFRESH_DB${_db_index}_SOURCE is \"${refresh_source}\". Set it to another source of the dump file: url, ftp, acquia, lagoon or s3." ;; esac - note "Fetching from the ${VORTEX_REFRESH_DB_SOURCE} source in VORTEX_REFRESH_DB_SOURCE as VORTEX_PUSH_DB_PROCEED is set to 1." - VORTEX_FETCH_DB_SOURCE="${VORTEX_REFRESH_DB_SOURCE}" + note "Fetching from the ${refresh_source} source in VORTEX_REFRESH_DB${_db_index}_SOURCE as VORTEX_PUSH_DB_PROCEED is set to 1." + VORTEX_FETCH_DB_SOURCE="${refresh_source}" ;; esac fi @@ -79,9 +82,8 @@ case "${VORTEX_FETCH_DB_SOURCE}" in lagoon) fetch_script="vortex-fetch-db-lagoon" ;; container_registry) fetch_script="vortex-fetch-db-container-registry" ;; s3) fetch_script="vortex-fetch-db-s3" ;; - gdrive) fetch_script="vortex-fetch-db-gdrive" ;; none) fetch_script="" ;; - *) fail "Unsupported database source \"${VORTEX_FETCH_DB_SOURCE}\" in VORTEX_FETCH_DB${_db_index}_SOURCE. Use one of: url, ftp, acquia, lagoon, container_registry, s3, gdrive, none." ;; + *) fail "Unsupported database source \"${VORTEX_FETCH_DB_SOURCE}\" in VORTEX_FETCH_DB${_db_index}_SOURCE. Use one of: url, ftp, acquia, lagoon, container_registry, s3, none." ;; esac # Skip file existence check for container_registry source as the database is diff --git a/.vortex/tooling/src/vortex-fetch-db-gdrive b/.vortex/tooling/src/vortex-fetch-db-gdrive deleted file mode 100755 index 546b40dc53..0000000000 --- a/.vortex/tooling/src/vortex-fetch-db-gdrive +++ /dev/null @@ -1,135 +0,0 @@ -#!/usr/bin/env bash -## -# Fetch DB dump from Google Drive. -# -# Authenticates with a Google Cloud service account key and downloads the file -# with the database dump file name from a Google Drive folder. The folder may -# be in a shared drive. -# -# IMPORTANT! This script runs outside the container on the host system. -# -# shellcheck disable=SC1090,SC1091 - -t=$(mktemp) && export -p >"${t}" && set -a && . ./.env && if [ -f ./.env.local ]; then . ./.env.local; fi && set +a && . "${t}" && rm "${t}" && unset t - -set -eu - -# Database index suffix. When set (e.g., "2"), all DB-related variable lookups -# use the indexed variant. -_db_index="${VORTEX_DB_INDEX:-}" - -# Google Cloud service account key: the JSON content or a path to the JSON -# key file. -_v="VORTEX_FETCH_DB${_db_index}_GDRIVE_SERVICE_ACCOUNT_KEY" -VORTEX_FETCH_DB_GDRIVE_SERVICE_ACCOUNT_KEY="${!_v:-${VORTEX_GDRIVE_SERVICE_ACCOUNT_KEY:-}}" - -# Google Drive folder ID with the database dump file. -# -# The ID is the last segment of the folder URL. The folder must be shared with -# the service account. -_v="VORTEX_FETCH_DB${_db_index}_GDRIVE_FOLDER_ID" -VORTEX_FETCH_DB_GDRIVE_FOLDER_ID="${!_v:-}" - -# Directory with database dump file. -_v="VORTEX_FETCH_DB${_db_index}_GDRIVE_DB_DIR" -_vs="VORTEX_FETCH_DB${_db_index}_DIR" -_vss="VORTEX_DB${_db_index}_DIR" -VORTEX_FETCH_DB_GDRIVE_DB_DIR="${!_v:-${!_vs:-${!_vss:-${VORTEX_DB_DIR:-./.data}}}}" - -# Database dump file name. -# -# The same name is used for the file in the Google Drive folder. -_v="VORTEX_FETCH_DB${_db_index}_GDRIVE_DB_FILE" -_vs="VORTEX_FETCH_DB${_db_index}_FILE" -_vss="VORTEX_DB${_db_index}_FILE" -VORTEX_FETCH_DB_GDRIVE_DB_FILE="${!_v:-${!_vs:-${!_vss:-${VORTEX_DB_FILE:-db.sql}}}}" - -# ------------------------------------------------------------------------------ - -# @formatter:off -info() { [ "${TERM:-}" != "dumb" ] && tput colors >/dev/null 2>&1 && printf "\033[36m[INFO] %s\033[0m\n" "${1}" || printf "[INFO] %s\n" "${1}"; } -note() { printf " %s\n" "${1}"; } -task() { _TASK_START=$(date +%s); [ "${TERM:-}" != "dumb" ] && tput colors >/dev/null 2>&1 && printf "\033[34m[TASK] %s\033[0m\n" "${1}" || printf "[TASK] %s\n" "${1}"; } -pass() { _d=""; [ -n "${_TASK_START:-}" ] && _d=" ($(($(date +%s) - _TASK_START))s)" && unset _TASK_START; [ "${TERM:-}" != "dumb" ] && tput colors >/dev/null 2>&1 && printf "\033[32m[ OK ] %s%s\033[0m\n" "${1}" "${_d}" || printf "[ OK ] %s%s\n" "${1}" "${_d}"; } -fail() { [ "${TERM:-}" != "dumb" ] && tput colors >/dev/null 2>&1 && printf "\033[31m[FAIL] %s\033[0m\n" "${1}" || printf "[FAIL] %s\n" "${1}"; exit "${2:-1}"; } -# @formatter:on - -extract_json_last_value() { - local key="${1}" - php -r "\$data=json_decode(file_get_contents('php://stdin'), TRUE); \$last=array_pop(\$data); isset(\$last[\"${key}\"]) ? print trim(json_encode(\$last[\"${key}\"], JSON_UNESCAPED_SLASHES), '\"') : exit(1);" -} - -extract_json_value() { - local key="${1}" - php -r "\$data=json_decode(file_get_contents('php://stdin'), TRUE); isset(\$data[\"${key}\"]) ? print trim(json_encode(\$data[\"${key}\"], JSON_UNESCAPED_SLASHES), '\"') : exit(1);" -} - -# Print a Google OAuth access token for a service account key and a scope. -# -# The key is passed to PHP in the environment, so it never appears in the -# process list. -gdrive_access_token() { - local key="${1}" - local scope="${2}" - - [ -f "${key}" ] && key="$(cat "${key}")" - - local assertion - # shellcheck disable=SC2016 - assertion=$(GDRIVE_KEY="${key}" GDRIVE_SCOPE="${scope}" php -r '$k=json_decode((string) getenv("GDRIVE_KEY"), TRUE); if (!is_array($k) || empty($k["client_email"]) || empty($k["private_key"])) { exit(1); } $b=fn($s) => rtrim(strtr(base64_encode($s), "+/", "-_"), "="); $t=time(); $u=$b(json_encode(["alg" => "RS256", "typ" => "JWT"])) . "." . $b(json_encode(["iss" => $k["client_email"], "scope" => getenv("GDRIVE_SCOPE"), "aud" => "https://oauth2.googleapis.com/token", "iat" => $t, "exp" => $t + 3600])); if (!@openssl_sign($u, $s, $k["private_key"], OPENSSL_ALGO_SHA256)) { exit(1); } print $u . "." . $b($s);') || return 1 - - local token_json - token_json=$(curl -sS -L https://oauth2.googleapis.com/token --data-urlencode "grant_type=urn:ietf:params:oauth:grant-type:jwt-bearer" --data-urlencode "assertion=${assertion}") || return 1 - - echo "${token_json}" | extract_json_value "access_token" -} - -# Print the ID of the most recently modified file with a name in a folder. -# -# Prints nothing when the folder has no such file, and fails when the folder -# cannot be listed. -gdrive_find_file() { - local token="${1}" - local folder_id="${2}" - local name="${3}" - - local files_json - files_json=$(curl -sS -G https://www.googleapis.com/drive/v3/files -H "Authorization: Bearer ${token}" --data-urlencode "q=name = '${name//\'/\\\'}' and '${folder_id}' in parents and trashed = false" --data-urlencode "fields=files(id)" --data-urlencode "orderBy=modifiedTime desc" --data-urlencode "pageSize=1" --data-urlencode "supportsAllDrives=true" --data-urlencode "includeItemsFromAllDrives=true") || return 1 - - local files - files=$(echo "${files_json}" | extract_json_value "files") || return 1 - - echo "${files}" | extract_json_last_value "id" || true -} - -for cmd in php curl; do command -v "${cmd}" >/dev/null || fail "Command ${cmd} is not available."; done - -[ -z "${VORTEX_FETCH_DB_GDRIVE_SERVICE_ACCOUNT_KEY}" ] && fail "Missing required value for VORTEX_FETCH_DB${_db_index}_GDRIVE_SERVICE_ACCOUNT_KEY or VORTEX_GDRIVE_SERVICE_ACCOUNT_KEY." -[ -z "${VORTEX_FETCH_DB_GDRIVE_FOLDER_ID}" ] && fail "Missing required value for VORTEX_FETCH_DB${_db_index}_GDRIVE_FOLDER_ID." - -info "Started database dump fetch from Google Drive." - -note "Remote file: ${VORTEX_FETCH_DB_GDRIVE_DB_FILE}" -note "Local path: ${VORTEX_FETCH_DB_GDRIVE_DB_DIR}/${VORTEX_FETCH_DB_GDRIVE_DB_FILE}" -note "Folder ID: ${VORTEX_FETCH_DB_GDRIVE_FOLDER_ID}" - -task "Authenticating with the service account." -token=$(gdrive_access_token "${VORTEX_FETCH_DB_GDRIVE_SERVICE_ACCOUNT_KEY}" "https://www.googleapis.com/auth/drive.readonly") || fail "Unable to authenticate with the service account. Check that VORTEX_FETCH_DB${_db_index}_GDRIVE_SERVICE_ACCOUNT_KEY or VORTEX_GDRIVE_SERVICE_ACCOUNT_KEY is a valid service account JSON key." -pass "Authenticated with the service account." - -task "Finding the database dump file in the folder." -file_id=$(gdrive_find_file "${token}" "${VORTEX_FETCH_DB_GDRIVE_FOLDER_ID}" "${VORTEX_FETCH_DB_GDRIVE_DB_FILE}") || fail "Unable to list the ${VORTEX_FETCH_DB_GDRIVE_FOLDER_ID} folder. Check that the folder ID is correct and that the folder is shared with the service account." -[ -z "${file_id}" ] && fail "Unable to find the ${VORTEX_FETCH_DB_GDRIVE_DB_FILE} file in the ${VORTEX_FETCH_DB_GDRIVE_FOLDER_ID} folder." -pass "Found the database dump file with ID ${file_id}." - -mkdir -p "${VORTEX_FETCH_DB_GDRIVE_DB_DIR}" - -task "Downloading the database dump file." -curl -sS -f -L -H "Authorization: Bearer ${token}" -o "${VORTEX_FETCH_DB_GDRIVE_DB_DIR}/${VORTEX_FETCH_DB_GDRIVE_DB_FILE}" "https://www.googleapis.com/drive/v3/files/${file_id}?alt=media&supportsAllDrives=true" || fail "Unable to download the database dump file." -pass "Downloaded the database dump file." - -# Enable xtrace only after the authenticated requests so the access token is -# never written to the trace output. -[ "${VORTEX_DEBUG-}" = "1" ] && set -x - -pass "Finished database dump fetch from Google Drive." diff --git a/.vortex/tooling/src/vortex-push-db b/.vortex/tooling/src/vortex-push-db index 4ba20e8c4d..9a36c9629f 100755 --- a/.vortex/tooling/src/vortex-push-db +++ b/.vortex/tooling/src/vortex-push-db @@ -5,8 +5,13 @@ # This is a router script to call relevant scripts based on type. # # The database dump file is pushed to the storage the database is fetched -# from, set in VORTEX_FETCH_DB_SOURCE. Only the 'gdrive' and 's3' sources -# accept a push; the push is skipped for other sources. +# from, set in VORTEX_FETCH_DB_SOURCE. Only an 'ftp' or 's3' source with an +# upstream source in VORTEX_REFRESH_DB_SOURCE accepts a push (a dump store); +# the push is skipped for other sources. +# +# With VORTEX_DB_INDEX set (e.g., "2" for the migration database), the +# indexed database pushes its own dump file with the indexed variables, so +# several databases share 1 storage. # # IMPORTANT! This script runs outside the container on the host system. # @@ -17,6 +22,10 @@ t=$(mktemp) && export -p >"${t}" && set -a && . ./.env && if [ -f ./.env.local ] set -eu [ "${VORTEX_DEBUG-}" = "1" ] && set -x +# Database index suffix. When set (e.g., "2"), all DB-related variable lookups +# use the indexed variant. +_db_index="${VORTEX_DB_INDEX:-}" + # Proceed with push. # # Set to 1 in the nightly CI job only, so the storage is refreshed once a day @@ -24,7 +33,12 @@ set -eu VORTEX_PUSH_DB_PROCEED="${VORTEX_PUSH_DB_PROCEED:-}" # Database fetch source, which is also the push destination. -VORTEX_FETCH_DB_SOURCE="${VORTEX_FETCH_DB_SOURCE:-url}" +_v="VORTEX_FETCH_DB${_db_index}_SOURCE" +VORTEX_FETCH_DB_SOURCE="${!_v:-url}" + +# Upstream source of the pushed database dump file. +_v="VORTEX_REFRESH_DB${_db_index}_SOURCE" +VORTEX_REFRESH_DB_SOURCE="${!_v:-}" # ------------------------------------------------------------------------------ @@ -36,19 +50,24 @@ pass() { _d=""; [ -n "${_TASK_START:-}" ] && _d=" ($(($(date +%s) - _TASK_START) fail() { [ "${TERM:-}" != "dumb" ] && tput colors >/dev/null 2>&1 && printf "\033[31m[FAIL] %s\033[0m\n" "${1}" || printf "[FAIL] %s\n" "${1}"; exit "${2:-1}"; } # @formatter:on -info "Started database push." +info "Started database${_db_index:+ ${_db_index}} push." [ "${VORTEX_PUSH_DB_PROCEED}" != "1" ] && pass "Skipped database push as VORTEX_PUSH_DB_PROCEED is not set to 1." && exit 0 case "${VORTEX_FETCH_DB_SOURCE}" in - gdrive) push_script="vortex-push-db-gdrive" ;; + ftp) push_script="vortex-push-db-ftp" ;; s3) push_script="vortex-push-db-s3" ;; - *) pass "Skipped database push as the \"${VORTEX_FETCH_DB_SOURCE}\" database source in VORTEX_FETCH_DB_SOURCE does not accept a push. Use one of: gdrive, s3." && exit 0 ;; + *) pass "Skipped database push as the \"${VORTEX_FETCH_DB_SOURCE}\" database source in VORTEX_FETCH_DB${_db_index}_SOURCE does not accept a push. Use one of: ftp, s3." && exit 0 ;; esac -# The pushed dump leaves the hosting and CI, so it must be sanitized. -[ "${VORTEX_PROVISION_SANITIZE_DB_SKIP:-}" = "1" ] && fail "Refused to push the database as VORTEX_PROVISION_SANITIZE_DB_SKIP is set to 1 and the database dump is not sanitized." +# Without an upstream source, the fetch source is where the dump comes from, +# and a push would replace that dump with its own export. +[ -z "${VORTEX_REFRESH_DB_SOURCE}" ] && pass "Skipped database push as VORTEX_REFRESH_DB${_db_index}_SOURCE is not set, so the ${VORTEX_FETCH_DB_SOURCE} source is not a dump store." && exit 0 + +# The pushed dump leaves the hosting and CI, so it must be sanitized. Only the +# primary database is sanitized: an indexed database is pushed as fetched. +[ -z "${_db_index}" ] && [ "${VORTEX_PROVISION_SANITIZE_DB_SKIP:-}" = "1" ] && fail "Refused to push the database as VORTEX_PROVISION_SANITIZE_DB_SKIP is set to 1 and the database dump is not sanitized." -"$(dirname "${BASH_SOURCE[0]}")/${push_script}" +env VORTEX_DB_INDEX="${_db_index}" "$(dirname "${BASH_SOURCE[0]}")/${push_script}" -pass "Finished database push." +pass "Finished database${_db_index:+ ${_db_index}} push." diff --git a/.vortex/tooling/src/vortex-push-db-ftp b/.vortex/tooling/src/vortex-push-db-ftp new file mode 100755 index 0000000000..2979a1e1ab --- /dev/null +++ b/.vortex/tooling/src/vortex-push-db-ftp @@ -0,0 +1,143 @@ +#!/usr/bin/env bash +## +# Push DB dump to FTP. +# +# Keeps 2 copies of the database dump file on the FTP server: the current +# dump under the remote file name and the dump it replaced with a '-previous' +# suffix, such as 'db.sql' and 'db-previous.sql'. The fetch always finds the +# current dump under the remote file name. +# +# The dump is uploaded under a temporary name and renamed into place only +# after the upload succeeds, so a failed upload keeps the current dump. +# +# IMPORTANT! This script runs outside the container on the host system. +# +# shellcheck disable=SC1090,SC1091 + +t=$(mktemp) && export -p >"${t}" && set -a && . ./.env && if [ -f ./.env.local ]; then . ./.env.local; fi && set +a && . "${t}" && rm "${t}" && unset t + +set -eu + +# Database index suffix. When set (e.g., "2"), all DB-related variable lookups +# use the indexed variant, so each database pushes its own dump file. +_db_index="${VORTEX_DB_INDEX:-}" + +# The FTP values fall back to the values the database is fetched with, so a +# server used as the fetch source is configured once. + +# The FTP user. +_v="VORTEX_PUSH_DB${_db_index}_FTP_USER" +_vs="VORTEX_FETCH_DB${_db_index}_FTP_USER" +VORTEX_PUSH_DB_FTP_USER="${!_v:-${!_vs:-}}" + +# The FTP password. +_v="VORTEX_PUSH_DB${_db_index}_FTP_PASS" +_vs="VORTEX_FETCH_DB${_db_index}_FTP_PASS" +VORTEX_PUSH_DB_FTP_PASS="${!_v:-${!_vs:-}}" + +# The FTP host. +_v="VORTEX_PUSH_DB${_db_index}_FTP_HOST" +_vs="VORTEX_FETCH_DB${_db_index}_FTP_HOST" +VORTEX_PUSH_DB_FTP_HOST="${!_v:-${!_vs:-}}" + +# The FTP port. +_v="VORTEX_PUSH_DB${_db_index}_FTP_PORT" +_vs="VORTEX_FETCH_DB${_db_index}_FTP_PORT" +VORTEX_PUSH_DB_FTP_PORT="${!_v:-${!_vs:-21}}" + +# The remote file name, including any directories. +_v="VORTEX_PUSH_DB${_db_index}_FTP_FILE" +_vs="VORTEX_FETCH_DB${_db_index}_FTP_FILE" +VORTEX_PUSH_DB_FTP_FILE="${!_v:-${!_vs:-}}" + +# Directory with database dump file. +_v="VORTEX_PUSH_DB${_db_index}_FTP_DB_DIR" +_vs="VORTEX_FETCH_DB${_db_index}_DIR" +_vss="VORTEX_DB${_db_index}_DIR" +VORTEX_PUSH_DB_FTP_DB_DIR="${!_v:-${!_vs:-${!_vss:-${VORTEX_DB_DIR:-./.data}}}}" + +# Database dump file name. +# +# Defaults to the name the database is fetched and refreshed with. +_v="VORTEX_PUSH_DB${_db_index}_FTP_DB_FILE" +_vs="VORTEX_FETCH_DB${_db_index}_FILE" +_vss="VORTEX_DB${_db_index}_FILE" +VORTEX_PUSH_DB_FTP_DB_FILE="${!_v:-${!_vs:-${!_vss:-${VORTEX_DB_FILE:-db.sql}}}}" + +# Prefix for the names of the pushed database dump files. +# +# Use a different prefix in each place that pushes to the same storage, such +# as 'gha-' and 'circleci-', to tell their dumps apart. A fetch reads the +# prefixed name only when its remote file name includes the prefix. +VORTEX_PUSH_DB_FILE_PREFIX="${VORTEX_PUSH_DB_FILE_PREFIX:-}" + +# ------------------------------------------------------------------------------ + +# @formatter:off +info() { [ "${TERM:-}" != "dumb" ] && tput colors >/dev/null 2>&1 && printf "\033[36m[INFO] %s\033[0m\n" "${1}" || printf "[INFO] %s\n" "${1}"; } +note() { printf " %s\n" "${1}"; } +task() { _TASK_START=$(date +%s); [ "${TERM:-}" != "dumb" ] && tput colors >/dev/null 2>&1 && printf "\033[34m[TASK] %s\033[0m\n" "${1}" || printf "[TASK] %s\n" "${1}"; } +pass() { _d=""; [ -n "${_TASK_START:-}" ] && _d=" ($(($(date +%s) - _TASK_START))s)" && unset _TASK_START; [ "${TERM:-}" != "dumb" ] && tput colors >/dev/null 2>&1 && printf "\033[32m[ OK ] %s%s\033[0m\n" "${1}" "${_d}" || printf "[ OK ] %s%s\n" "${1}" "${_d}"; } +fail() { [ "${TERM:-}" != "dumb" ] && tput colors >/dev/null 2>&1 && printf "\033[31m[FAIL] %s\033[0m\n" "${1}" || printf "[FAIL] %s\n" "${1}"; exit "${2:-1}"; } +# @formatter:on + +# Print a value quoted for a curl config file. +curl_config_quote() { + local value="${1//\\/\\\\}" + printf '"%s"' "${value//\"/\\\"}" +} + +# shellcheck disable=SC2043 +for cmd in curl; do command -v "${cmd}" >/dev/null || fail "Command ${cmd} is not available."; done + +[ -z "${VORTEX_PUSH_DB_FTP_USER}" ] && fail "Missing required value for VORTEX_PUSH_DB${_db_index}_FTP_USER or VORTEX_FETCH_DB${_db_index}_FTP_USER." +[ -z "${VORTEX_PUSH_DB_FTP_PASS}" ] && fail "Missing required value for VORTEX_PUSH_DB${_db_index}_FTP_PASS or VORTEX_FETCH_DB${_db_index}_FTP_PASS." +[ -z "${VORTEX_PUSH_DB_FTP_HOST}" ] && fail "Missing required value for VORTEX_PUSH_DB${_db_index}_FTP_HOST or VORTEX_FETCH_DB${_db_index}_FTP_HOST." +[ -z "${VORTEX_PUSH_DB_FTP_FILE}" ] && fail "Missing required value for VORTEX_PUSH_DB${_db_index}_FTP_FILE or VORTEX_FETCH_DB${_db_index}_FTP_FILE." + +local_file="${VORTEX_PUSH_DB_FTP_DB_DIR}/${VORTEX_PUSH_DB_FTP_DB_FILE}" +[ ! -f "${local_file}" ] && fail "Database dump file ${local_file} does not exist." + +remote_dir="" +remote_name="${VORTEX_PUSH_DB_FTP_FILE#/}" +if [[ ${remote_name} == */* ]]; then + remote_dir="${remote_name%/*}/" + remote_name="${remote_name##*/}" +fi + +current_name="${VORTEX_PUSH_DB_FILE_PREFIX}${remote_name}" +if [[ ${remote_name} == *.* ]]; then + previous_name="${VORTEX_PUSH_DB_FILE_PREFIX}${remote_name%.*}-previous.${remote_name##*.}" +else + previous_name="${VORTEX_PUSH_DB_FILE_PREFIX}${remote_name}-previous" +fi +upload_name="${current_name}.uploading" + +info "Started database${_db_index:+ ${_db_index}} dump push to FTP." + +note "Local file: ${local_file}" +note "Remote file: ${remote_dir}${current_name}" +note "Previous file: ${remote_dir}${previous_name}" +note "FTP host: ${VORTEX_PUSH_DB_FTP_HOST}:${VORTEX_PUSH_DB_FTP_PORT}" + +# The credentials go to curl on stdin, so they never appear in the process +# list. The commands after the upload run in the remote directory: the +# previous dump is replaced by the current one, and the upload takes the +# current name. Only the final rename must succeed, since the first push has +# no current or previous dump. +task "Uploading the database dump file and keeping the dump it replaces." +curl_config="user = $(curl_config_quote "${VORTEX_PUSH_DB_FTP_USER}:${VORTEX_PUSH_DB_FTP_PASS}")" +echo "${curl_config}" | curl --silent --show-error --config - --ftp-create-dirs --upload-file "${local_file}" \ + --quote "-*DELE ${previous_name}" \ + --quote "-*RNFR ${current_name}" \ + --quote "-*RNTO ${previous_name}" \ + --quote "-RNFR ${upload_name}" \ + --quote "-RNTO ${current_name}" \ + "ftp://${VORTEX_PUSH_DB_FTP_HOST}:${VORTEX_PUSH_DB_FTP_PORT}/${remote_dir}${upload_name}" || fail "Unable to push the database dump file to FTP." +pass "Uploaded the database dump file and kept the dump it replaced." + +# Enable xtrace only after the request so the credentials are never written +# to the trace output. +[ "${VORTEX_DEBUG-}" = "1" ] && set -x + +pass "Finished database${_db_index:+ ${_db_index}} dump push to FTP." diff --git a/.vortex/tooling/src/vortex-push-db-gdrive b/.vortex/tooling/src/vortex-push-db-gdrive deleted file mode 100755 index c2a975752a..0000000000 --- a/.vortex/tooling/src/vortex-push-db-gdrive +++ /dev/null @@ -1,161 +0,0 @@ -#!/usr/bin/env bash -## -# Push DB dump to Google Drive. -# -# Authenticates with a Google Cloud service account key and uploads the -# database dump file to a Google Drive folder. A file with the same name in -# the folder is replaced in place, so its ID and sharing settings are kept. -# -# Service accounts have no storage quota of their own, so the folder must be -# in a shared drive. -# -# IMPORTANT! This script runs outside the container on the host system. -# -# shellcheck disable=SC1090,SC1091 - -t=$(mktemp) && export -p >"${t}" && set -a && . ./.env && if [ -f ./.env.local ]; then . ./.env.local; fi && set +a && . "${t}" && rm "${t}" && unset t - -set -eu -set -o pipefail - -# Google Cloud service account key: the JSON content or a path to the JSON -# key file. -VORTEX_PUSH_DB_GDRIVE_SERVICE_ACCOUNT_KEY="${VORTEX_PUSH_DB_GDRIVE_SERVICE_ACCOUNT_KEY:-${VORTEX_GDRIVE_SERVICE_ACCOUNT_KEY:-}}" - -# Google Drive folder ID to push the database dump file to. -# -# Defaults to the folder the database is fetched from. -VORTEX_PUSH_DB_GDRIVE_FOLDER_ID="${VORTEX_PUSH_DB_GDRIVE_FOLDER_ID:-${VORTEX_FETCH_DB_GDRIVE_FOLDER_ID:-}}" - -# Directory with database dump file. -VORTEX_PUSH_DB_GDRIVE_DB_DIR="${VORTEX_PUSH_DB_GDRIVE_DB_DIR:-${VORTEX_DB_DIR:-./.data}}" - -# Database dump file name. -# -# Defaults to the name the database is fetched and refreshed with. The same -# name is used for the file in the Google Drive folder. -VORTEX_PUSH_DB_GDRIVE_DB_FILE="${VORTEX_PUSH_DB_GDRIVE_DB_FILE:-${VORTEX_FETCH_DB_FILE:-${VORTEX_DB_FILE:-db.sql}}}" - -# ------------------------------------------------------------------------------ - -# @formatter:off -info() { [ "${TERM:-}" != "dumb" ] && tput colors >/dev/null 2>&1 && printf "\033[36m[INFO] %s\033[0m\n" "${1}" || printf "[INFO] %s\n" "${1}"; } -note() { printf " %s\n" "${1}"; } -task() { _TASK_START=$(date +%s); [ "${TERM:-}" != "dumb" ] && tput colors >/dev/null 2>&1 && printf "\033[34m[TASK] %s\033[0m\n" "${1}" || printf "[TASK] %s\n" "${1}"; } -pass() { _d=""; [ -n "${_TASK_START:-}" ] && _d=" ($(($(date +%s) - _TASK_START))s)" && unset _TASK_START; [ "${TERM:-}" != "dumb" ] && tput colors >/dev/null 2>&1 && printf "\033[32m[ OK ] %s%s\033[0m\n" "${1}" "${_d}" || printf "[ OK ] %s%s\n" "${1}" "${_d}"; } -fail() { [ "${TERM:-}" != "dumb" ] && tput colors >/dev/null 2>&1 && printf "\033[31m[FAIL] %s\033[0m\n" "${1}" || printf "[FAIL] %s\n" "${1}"; exit "${2:-1}"; } -# @formatter:on - -extract_json_last_value() { - local key="${1}" - php -r "\$data=json_decode(file_get_contents('php://stdin'), TRUE); \$last=array_pop(\$data); isset(\$last[\"${key}\"]) ? print trim(json_encode(\$last[\"${key}\"], JSON_UNESCAPED_SLASHES), '\"') : exit(1);" -} - -extract_json_value() { - local key="${1}" - php -r "\$data=json_decode(file_get_contents('php://stdin'), TRUE); isset(\$data[\"${key}\"]) ? print trim(json_encode(\$data[\"${key}\"], JSON_UNESCAPED_SLASHES), '\"') : exit(1);" -} - -# Print a Google OAuth access token for a service account key and a scope. -# -# The key is passed to PHP in the environment, so it never appears in the -# process list. -gdrive_access_token() { - local key="${1}" - local scope="${2}" - - [ -f "${key}" ] && key="$(cat "${key}")" - - local assertion - # shellcheck disable=SC2016 - assertion=$(GDRIVE_KEY="${key}" GDRIVE_SCOPE="${scope}" php -r '$k=json_decode((string) getenv("GDRIVE_KEY"), TRUE); if (!is_array($k) || empty($k["client_email"]) || empty($k["private_key"])) { exit(1); } $b=fn($s) => rtrim(strtr(base64_encode($s), "+/", "-_"), "="); $t=time(); $u=$b(json_encode(["alg" => "RS256", "typ" => "JWT"])) . "." . $b(json_encode(["iss" => $k["client_email"], "scope" => getenv("GDRIVE_SCOPE"), "aud" => "https://oauth2.googleapis.com/token", "iat" => $t, "exp" => $t + 3600])); if (!@openssl_sign($u, $s, $k["private_key"], OPENSSL_ALGO_SHA256)) { exit(1); } print $u . "." . $b($s);') || return 1 - - local token_json - token_json=$(curl -sS -L https://oauth2.googleapis.com/token --data-urlencode "grant_type=urn:ietf:params:oauth:grant-type:jwt-bearer" --data-urlencode "assertion=${assertion}") || return 1 - - echo "${token_json}" | extract_json_value "access_token" -} - -# Print the ID of the most recently modified file with a name in a folder. -# -# Prints nothing when the folder has no such file, and fails when the folder -# cannot be listed. -gdrive_find_file() { - local token="${1}" - local folder_id="${2}" - local name="${3}" - - local files_json - files_json=$(curl -sS -G https://www.googleapis.com/drive/v3/files -H "Authorization: Bearer ${token}" --data-urlencode "q=name = '${name//\'/\\\'}' and '${folder_id}' in parents and trashed = false" --data-urlencode "fields=files(id)" --data-urlencode "orderBy=modifiedTime desc" --data-urlencode "pageSize=1" --data-urlencode "supportsAllDrives=true" --data-urlencode "includeItemsFromAllDrives=true") || return 1 - - local files - files=$(echo "${files_json}" | extract_json_value "files") || return 1 - - echo "${files}" | extract_json_last_value "id" || true -} - -for cmd in php curl; do command -v "${cmd}" >/dev/null || fail "Command ${cmd} is not available."; done - -[ -z "${VORTEX_PUSH_DB_GDRIVE_SERVICE_ACCOUNT_KEY}" ] && fail "Missing required value for VORTEX_PUSH_DB_GDRIVE_SERVICE_ACCOUNT_KEY or VORTEX_GDRIVE_SERVICE_ACCOUNT_KEY." -[ -z "${VORTEX_PUSH_DB_GDRIVE_FOLDER_ID}" ] && fail "Missing required value for VORTEX_PUSH_DB_GDRIVE_FOLDER_ID or VORTEX_FETCH_DB_GDRIVE_FOLDER_ID." - -local_file="${VORTEX_PUSH_DB_GDRIVE_DB_DIR}/${VORTEX_PUSH_DB_GDRIVE_DB_FILE}" -[ ! -f "${local_file}" ] && fail "Database dump file ${local_file} does not exist." - -info "Started database dump push to Google Drive." - -note "Local file: ${local_file}" -note "Remote file: ${VORTEX_PUSH_DB_GDRIVE_DB_FILE}" -note "Folder ID: ${VORTEX_PUSH_DB_GDRIVE_FOLDER_ID}" - -task "Authenticating with the service account." -token=$(gdrive_access_token "${VORTEX_PUSH_DB_GDRIVE_SERVICE_ACCOUNT_KEY}" "https://www.googleapis.com/auth/drive") || fail "Unable to authenticate with the service account. Check that VORTEX_PUSH_DB_GDRIVE_SERVICE_ACCOUNT_KEY or VORTEX_GDRIVE_SERVICE_ACCOUNT_KEY is a valid service account JSON key." -pass "Authenticated with the service account." - -file_id=$(gdrive_find_file "${token}" "${VORTEX_PUSH_DB_GDRIVE_FOLDER_ID}" "${VORTEX_PUSH_DB_GDRIVE_DB_FILE}") || fail "Unable to list the ${VORTEX_PUSH_DB_GDRIVE_FOLDER_ID} folder. Check that the folder ID is correct and that the folder is shared with the service account." - -upload_url="https://www.googleapis.com/upload/drive/v3/files" -if [ -n "${file_id}" ]; then - note "Replacing the existing file with ID ${file_id}." - request_type="PATCH" - upload_url="${upload_url}/${file_id}" - metadata="{}" -else - note "Creating a new file in the folder." - request_type="POST" - metadata=$(GDRIVE_NAME="${VORTEX_PUSH_DB_GDRIVE_DB_FILE}" GDRIVE_FOLDER_ID="${VORTEX_PUSH_DB_GDRIVE_FOLDER_ID}" php -r 'print json_encode(["name" => getenv("GDRIVE_NAME"), "parents" => [getenv("GDRIVE_FOLDER_ID")]]);') -fi - -# A resumable upload session accepts the whole file in a single request, so -# the dump is not limited by the 5 MB multipart upload size. -task "Starting the upload session." -session_headers=$(curl -sS -X "${request_type}" -D - -o /dev/null -H "Authorization: Bearer ${token}" -H "Content-Type: application/json; charset=UTF-8" -H "X-Upload-Content-Type: application/octet-stream" --data "${metadata}" "${upload_url}?uploadType=resumable&supportsAllDrives=true") || fail "Unable to start the upload session." -session_url=$(echo "${session_headers}" | sed -n 's/^[Ll]ocation: *//p' | tr -d '\r') -[ -z "${session_url}" ] && fail "Unable to start the upload session: the response has no upload location. Check that the folder is in a shared drive and that the service account can edit it." -pass "Started the upload session." - -task "Uploading the database dump file." -set +e -response=$(curl -sS -X PUT --upload-file "${local_file}" -H "Content-Type: application/octet-stream" --write-out "\n%{http_code}" "${session_url}") -curl_status=$? -set -e - -# Enable xtrace only after the authenticated requests so the access token and -# the upload session URL are never written to the trace output. -[ "${VORTEX_DEBUG-}" = "1" ] && set -x - -[ "${curl_status}" -ne 0 ] && fail "Google Drive upload request failed with curl exit ${curl_status}." - -http_code=$(echo "${response}" | tail -1) -response_body=$(echo "${response}" | sed '$d') - -case "${http_code}" in - 2*) ;; - *) - [ -n "${response_body}" ] && echo "Response: ${response_body}" - fail "Unable to push to Google Drive: HTTP status ${http_code}." - ;; -esac -pass "Uploaded the database dump file." - -pass "Finished database dump push to Google Drive." diff --git a/.vortex/tooling/src/vortex-push-db-s3 b/.vortex/tooling/src/vortex-push-db-s3 index fd6986b1ff..4463c14f4f 100755 --- a/.vortex/tooling/src/vortex-push-db-s3 +++ b/.vortex/tooling/src/vortex-push-db-s3 @@ -4,6 +4,14 @@ # # Uses AWS Signature Version 4 with curl (no AWS CLI required). # +# Keeps 2 copies of the database dump file in the bucket: the current dump +# under the remote file name and the dump it replaced with a '-previous' +# suffix, such as 'db.sql' and 'db-previous.sql'. The fetch always finds the +# current dump under the remote file name. +# +# The current dump is copied to the previous one on the S3 side before the +# upload, so a failed upload keeps the current dump. +# # IMPORTANT! This script runs outside the container on the host system. # # shellcheck disable=SC1090,SC1091 @@ -13,41 +21,69 @@ t=$(mktemp) && export -p >"${t}" && set -a && . ./.env && if [ -f ./.env.local ] set -eu set -o pipefail +# Database index suffix. When set (e.g., "2"), all DB-related variable lookups +# use the indexed variant, so each database pushes its own dump file. +_db_index="${VORTEX_DB_INDEX:-}" + # The S3 values fall back to the values the database is fetched with, so a # bucket used as the fetch source is configured once. # AWS access key. -VORTEX_PUSH_DB_S3_ACCESS_KEY="${VORTEX_PUSH_DB_S3_ACCESS_KEY:-${VORTEX_FETCH_DB_S3_ACCESS_KEY:-${S3_ACCESS_KEY:-}}}" +_v="VORTEX_PUSH_DB${_db_index}_S3_ACCESS_KEY" +_vs="VORTEX_FETCH_DB${_db_index}_S3_ACCESS_KEY" +VORTEX_PUSH_DB_S3_ACCESS_KEY="${!_v:-${!_vs:-${S3_ACCESS_KEY:-}}}" # AWS secret key. -VORTEX_PUSH_DB_S3_SECRET_KEY="${VORTEX_PUSH_DB_S3_SECRET_KEY:-${VORTEX_FETCH_DB_S3_SECRET_KEY:-${S3_SECRET_KEY:-}}}" +_v="VORTEX_PUSH_DB${_db_index}_S3_SECRET_KEY" +_vs="VORTEX_FETCH_DB${_db_index}_S3_SECRET_KEY" +VORTEX_PUSH_DB_S3_SECRET_KEY="${!_v:-${!_vs:-${S3_SECRET_KEY:-}}}" # S3 bucket name. -VORTEX_PUSH_DB_S3_BUCKET="${VORTEX_PUSH_DB_S3_BUCKET:-${VORTEX_FETCH_DB_S3_BUCKET:-${S3_BUCKET:-}}}" +_v="VORTEX_PUSH_DB${_db_index}_S3_BUCKET" +_vs="VORTEX_FETCH_DB${_db_index}_S3_BUCKET" +VORTEX_PUSH_DB_S3_BUCKET="${!_v:-${!_vs:-${S3_BUCKET:-}}}" # S3 region. -VORTEX_PUSH_DB_S3_REGION="${VORTEX_PUSH_DB_S3_REGION:-${VORTEX_FETCH_DB_S3_REGION:-${S3_REGION:-}}}" +_v="VORTEX_PUSH_DB${_db_index}_S3_REGION" +_vs="VORTEX_FETCH_DB${_db_index}_S3_REGION" +VORTEX_PUSH_DB_S3_REGION="${!_v:-${!_vs:-${S3_REGION:-}}}" # S3 prefix (path within the bucket). -VORTEX_PUSH_DB_S3_PREFIX="${VORTEX_PUSH_DB_S3_PREFIX:-${VORTEX_FETCH_DB_S3_PREFIX:-${S3_PREFIX:-}}}" +_v="VORTEX_PUSH_DB${_db_index}_S3_PREFIX" +_vs="VORTEX_FETCH_DB${_db_index}_S3_PREFIX" +VORTEX_PUSH_DB_S3_PREFIX="${!_v:-${!_vs:-${S3_PREFIX:-}}}" # Directory with database dump file. -VORTEX_PUSH_DB_S3_DB_DIR="${VORTEX_PUSH_DB_S3_DB_DIR:-${VORTEX_DB_DIR:-./.data}}" +_v="VORTEX_PUSH_DB${_db_index}_S3_DB_DIR" +_vs="VORTEX_DB${_db_index}_DIR" +VORTEX_PUSH_DB_S3_DB_DIR="${!_v:-${!_vs:-${VORTEX_DB_DIR:-./.data}}}" # Database dump file name. # # Defaults to the name the database is fetched and refreshed with. -VORTEX_PUSH_DB_S3_DB_FILE="${VORTEX_PUSH_DB_S3_DB_FILE:-${VORTEX_FETCH_DB_FILE:-${VORTEX_DB_FILE:-db.sql}}}" +_v="VORTEX_PUSH_DB${_db_index}_S3_DB_FILE" +_vs="VORTEX_FETCH_DB${_db_index}_FILE" +_vss="VORTEX_DB${_db_index}_FILE" +VORTEX_PUSH_DB_S3_DB_FILE="${!_v:-${!_vs:-${!_vss:-${VORTEX_DB_FILE:-db.sql}}}}" # Remote database dump file name. # -# Defaults to the database dump file name, which is the name the database is -# fetched with. -VORTEX_PUSH_DB_S3_REMOTE_FILE="${VORTEX_PUSH_DB_S3_REMOTE_FILE:-${VORTEX_PUSH_DB_S3_DB_FILE}}" +# Defaults to the remote file name the database is fetched from, then to the +# database dump file name. +_v="VORTEX_PUSH_DB${_db_index}_S3_REMOTE_FILE" +_vs="VORTEX_FETCH_DB${_db_index}_S3_DB_FILE" +VORTEX_PUSH_DB_S3_REMOTE_FILE="${!_v:-${!_vs:-${VORTEX_PUSH_DB_S3_DB_FILE}}}" # S3 storage class. VORTEX_PUSH_DB_S3_STORAGE_CLASS="${VORTEX_PUSH_DB_S3_STORAGE_CLASS:-STANDARD}" +# Prefix for the names of the pushed database dump files. +# +# Use a different prefix in each place that pushes to the same storage, such +# as 'gha-' and 'circleci-', to tell their dumps apart. A fetch reads the +# prefixed name only when its remote file name includes the prefix. +VORTEX_PUSH_DB_FILE_PREFIX="${VORTEX_PUSH_DB_FILE_PREFIX:-}" + # ------------------------------------------------------------------------------ # @formatter:off @@ -94,37 +130,123 @@ aws_signature() { hmac_sha256 "hexkey:${signing_key}" "${string_to_sign}" } +# Send a signed PUT request for an object and print the response body +# followed by the HTTP status code on the last line. +# +# Arguments: the encoded object key, the payload hash, the file to upload +# (empty for none), then any extra 'name:value' headers to sign, with +# lowercase names. +s3_put() { + local key="${1}" + local payload_hash="${2}" + local upload_file="${3}" + shift 3 + + local host="${VORTEX_PUSH_DB_S3_BUCKET}.s3.${VORTEX_PUSH_DB_S3_REGION}.amazonaws.com" + local date_short date_long + date_short=$(date -u +'%Y%m%d') + date_long=$(date -u +'%Y%m%dT%H%M%SZ') + + local canonical_headers signed_headers + canonical_headers=$(printf '%s\n' "host:${host}" "x-amz-content-sha256:${payload_hash}" "x-amz-date:${date_long}" "x-amz-server-side-encryption:AES256" "x-amz-storage-class:${VORTEX_PUSH_DB_S3_STORAGE_CLASS}" "$@" | LC_ALL=C sort) + signed_headers=$(echo "${canonical_headers}" | cut -d: -f1 | paste -sd ';' -) + + local canonical_request="PUT +/${key} + +${canonical_headers} + +${signed_headers} +${payload_hash}" + + local canonical_request_hash + canonical_request_hash=$(hash_sha256 "${canonical_request}") + + local string_to_sign="AWS4-HMAC-SHA256 +${date_long} +${date_short}/${VORTEX_PUSH_DB_S3_REGION}/s3/aws4_request +${canonical_request_hash}" + + local signature + signature=$(aws_signature "${VORTEX_PUSH_DB_S3_SECRET_KEY}" "${date_short}" "${VORTEX_PUSH_DB_S3_REGION}" "s3" "${string_to_sign}") + + local args=(--silent --show-error --location --proto-redir "=https" --connect-timeout "${VORTEX_PUSH_DB_S3_CONNECT_TIMEOUT:-10}" --max-time "${VORTEX_PUSH_DB_S3_MAX_TIME:-300}" --request PUT --write-out "\n%{http_code}") + if [ -n "${upload_file}" ]; then + args+=(--upload-file "${upload_file}") + fi + + local header + while IFS= read -r header; do + args+=(--header "${header%%:*}: ${header#*:}") + done <<<"${canonical_headers}" + args+=(--header "Authorization: AWS4-HMAC-SHA256 Credential=${VORTEX_PUSH_DB_S3_ACCESS_KEY}/${date_short}/${VORTEX_PUSH_DB_S3_REGION}/s3/aws4_request, SignedHeaders=${signed_headers}, Signature=${signature}") + + curl "${args[@]}" "https://${host}/${key}" +} + for cmd in curl openssl; do command -v "${cmd}" >/dev/null || fail "Command ${cmd} is not available."; done -[ -z "${VORTEX_PUSH_DB_S3_ACCESS_KEY}" ] && fail "Missing required value for VORTEX_PUSH_DB_S3_ACCESS_KEY." -[ -z "${VORTEX_PUSH_DB_S3_SECRET_KEY}" ] && fail "Missing required value for VORTEX_PUSH_DB_S3_SECRET_KEY." -[ -z "${VORTEX_PUSH_DB_S3_BUCKET}" ] && fail "Missing required value for VORTEX_PUSH_DB_S3_BUCKET." -[ -z "${VORTEX_PUSH_DB_S3_REGION}" ] && fail "Missing required value for VORTEX_PUSH_DB_S3_REGION." +[ -z "${VORTEX_PUSH_DB_S3_ACCESS_KEY}" ] && fail "Missing required value for VORTEX_PUSH_DB${_db_index}_S3_ACCESS_KEY." +[ -z "${VORTEX_PUSH_DB_S3_SECRET_KEY}" ] && fail "Missing required value for VORTEX_PUSH_DB${_db_index}_S3_SECRET_KEY." +[ -z "${VORTEX_PUSH_DB_S3_BUCKET}" ] && fail "Missing required value for VORTEX_PUSH_DB${_db_index}_S3_BUCKET." +[ -z "${VORTEX_PUSH_DB_S3_REGION}" ] && fail "Missing required value for VORTEX_PUSH_DB${_db_index}_S3_REGION." local_file="${VORTEX_PUSH_DB_S3_DB_DIR}/${VORTEX_PUSH_DB_S3_DB_FILE}" [ ! -f "${local_file}" ] && fail "Database dump file ${local_file} does not exist." -info "Started database dump push to S3." +info "Started database${_db_index:+ ${_db_index}} dump push to S3." [ -n "${VORTEX_PUSH_DB_S3_PREFIX}" ] && VORTEX_PUSH_DB_S3_PREFIX="${VORTEX_PUSH_DB_S3_PREFIX%/}/" -request_type="PUT" -auth_type="AWS4-HMAC-SHA256" -service="s3" -base_url=".${service}.${VORTEX_PUSH_DB_S3_REGION}.amazonaws.com" -date_short=$(date -u +'%Y%m%d') -date_long=$(date -u +'%Y%m%dT%H%M%SZ') +# The file name prefix and the '-previous' suffix apply to the file name, not +# to the directories in the remote file name. +remote_dir="" +remote_name="${VORTEX_PUSH_DB_S3_REMOTE_FILE}" +if [[ ${remote_name} == */* ]]; then + remote_dir="${remote_name%/*}/" + remote_name="${remote_name##*/}" +fi + +current_file="${remote_dir}${VORTEX_PUSH_DB_FILE_PREFIX}${remote_name}" +if [[ ${remote_name} == *.* ]]; then + previous_file="${remote_dir}${VORTEX_PUSH_DB_FILE_PREFIX}${remote_name%.*}-previous.${remote_name##*.}" +else + previous_file="${remote_dir}${VORTEX_PUSH_DB_FILE_PREFIX}${remote_name}-previous" +fi -object_key="$(uri_encode_key "${VORTEX_PUSH_DB_S3_PREFIX}${VORTEX_PUSH_DB_S3_REMOTE_FILE}")" -object_url="https://${VORTEX_PUSH_DB_S3_BUCKET}${base_url}/${object_key}" +object_key="$(uri_encode_key "${VORTEX_PUSH_DB_S3_PREFIX}${current_file}")" +previous_key="$(uri_encode_key "${VORTEX_PUSH_DB_S3_PREFIX}${previous_file}")" note "Local file: ${local_file}" -note "Remote file: ${VORTEX_PUSH_DB_S3_PREFIX}${VORTEX_PUSH_DB_S3_REMOTE_FILE}" +note "Remote file: ${VORTEX_PUSH_DB_S3_PREFIX}${current_file}" +note "Previous file: ${VORTEX_PUSH_DB_S3_PREFIX}${previous_file}" note "S3 bucket: ${VORTEX_PUSH_DB_S3_BUCKET}" note "S3 region: ${VORTEX_PUSH_DB_S3_REGION}" [ -n "${VORTEX_PUSH_DB_S3_PREFIX}" ] && note "S3 prefix: ${VORTEX_PUSH_DB_S3_PREFIX}" note "Storage class: ${VORTEX_PUSH_DB_S3_STORAGE_CLASS}" +# A copy can fail after S3 has sent a 200 status, so a success needs the copy +# result in the body. A missing current dump means this is the first push. +task "Keeping the current database dump as the previous one." +set +e +response=$(s3_put "${previous_key}" "$(hash_sha256 "")" "" "x-amz-copy-source:/${VORTEX_PUSH_DB_S3_BUCKET}/${object_key}") +curl_status=$? +set -e + +[ "${curl_status}" -ne 0 ] && fail "S3 copy request failed with curl exit ${curl_status}." + +http_code=$(echo "${response}" | tail -1) +response_body=$(echo "${response}" | sed '$d') + +if [[ ${http_code} == 2* ]] && [[ ${response_body} == *"/dev/null; then content_type="$(file --brief --mime-type "${local_file}")" else @@ -133,51 +255,14 @@ fi payload_hash=$(openssl dgst -sha256 <"${local_file}" | sed 's/^.* //') -header_list='content-type;host;x-amz-content-sha256;x-amz-date;x-amz-server-side-encryption;x-amz-storage-class' - -canonical_request="\ -${request_type} -/${object_key} - -content-type:${content_type} -host:${VORTEX_PUSH_DB_S3_BUCKET}${base_url} -x-amz-content-sha256:${payload_hash} -x-amz-date:${date_long} -x-amz-server-side-encryption:AES256 -x-amz-storage-class:${VORTEX_PUSH_DB_S3_STORAGE_CLASS} - -${header_list} -${payload_hash}" - -canonical_request_hash=$(hash_sha256 "${canonical_request}") - -string_to_sign="\ -${auth_type} -${date_long} -${date_short}/${VORTEX_PUSH_DB_S3_REGION}/${service}/aws4_request -${canonical_request_hash}" - -signature=$(aws_signature "${VORTEX_PUSH_DB_S3_SECRET_KEY}" "${date_short}" "${VORTEX_PUSH_DB_S3_REGION}" "${service}" "${string_to_sign}") - +task "Uploading the database dump file." set +e -response=$(curl --silent --show-error --location --proto-redir =https \ - --connect-timeout "${VORTEX_PUSH_DB_S3_CONNECT_TIMEOUT:-10}" \ - --max-time "${VORTEX_PUSH_DB_S3_MAX_TIME:-300}" \ - --request "${request_type}" --upload-file "${local_file}" \ - --header "Content-Type: ${content_type}" \ - --header "Host: ${VORTEX_PUSH_DB_S3_BUCKET}${base_url}" \ - --header "X-Amz-Content-SHA256: ${payload_hash}" \ - --header "X-Amz-Date: ${date_long}" \ - --header "X-Amz-Server-Side-Encryption: AES256" \ - --header "X-Amz-Storage-Class: ${VORTEX_PUSH_DB_S3_STORAGE_CLASS}" \ - --header "Authorization: ${auth_type} Credential=${VORTEX_PUSH_DB_S3_ACCESS_KEY}/${date_short}/${VORTEX_PUSH_DB_S3_REGION}/${service}/aws4_request, SignedHeaders=${header_list}, Signature=${signature}" \ - --write-out "\n%{http_code}" \ - "${object_url}") +response=$(s3_put "${object_key}" "${payload_hash}" "${local_file}" "content-type:${content_type}") curl_status=$? set -e -# Enable xtrace only after the signed request so credentials and the -# signature are never written to the trace output. +# Enable xtrace only after the signed requests so credentials and the +# signatures are never written to the trace output. [ "${VORTEX_DEBUG-}" = "1" ] && set -x [ "${curl_status}" -ne 0 ] && fail "S3 push request failed with curl exit ${curl_status}." @@ -192,5 +277,6 @@ case "${http_code}" in fail "Unable to push to S3: HTTP status ${http_code}." ;; esac +pass "Uploaded the database dump file." -pass "Finished database dump push to S3." +pass "Finished database${_db_index:+ ${_db_index}} dump push to S3." diff --git a/.vortex/tooling/src/vortex-refresh-db b/.vortex/tooling/src/vortex-refresh-db index df899cafd5..f32cf74240 100755 --- a/.vortex/tooling/src/vortex-refresh-db +++ b/.vortex/tooling/src/vortex-refresh-db @@ -15,9 +15,10 @@ # the service is exported and pushed as VORTEX_DB2_IMAGE, without # provisioning. This runs only when VORTEX_REFRESH_DB2_PROCEED is set to 1. # -# With a dump store fetch source ('gdrive' or 's3') and VORTEX_PUSH_DB_PROCEED -# set to 1, the database is sanitized after provisioning, so the exported -# dump can be pushed to the store with 'vortex-push-db'. +# With an 'ftp' or 's3' fetch source that has an upstream source in +# VORTEX_REFRESH_DB_SOURCE (a dump store) and VORTEX_PUSH_DB_PROCEED set to 1, +# the database is sanitized after provisioning, so the exported dump can be +# pushed to the store with 'vortex-push-db'. # # Every variable of the calling environment is passed to the provisioning in # the 'cli' container, so the script is intended to run in CI. @@ -47,12 +48,13 @@ VORTEX_DB_IMAGE_BASE="${VORTEX_DB_IMAGE_BASE:-}" # # Required when the fetch source is 'container_registry' and a base image is # set: the database image is built from a dump file, so it needs a dump file -# source such as 'url', 'ftp', 'acquia', 'lagoon', 's3' or 'gdrive'. An -# indexed database uses VORTEX_REFRESH_DB2_SOURCE. +# source such as 'url', 'ftp', 'acquia', 'lagoon' or 's3'. An indexed +# database uses VORTEX_REFRESH_DB2_SOURCE. # -# Also required when the fetch source is a dump store ('gdrive' or 's3') and -# VORTEX_PUSH_DB_PROCEED is set to 1: 'vortex-fetch-db' then fetches the dump -# from this source, such as 'url', 'ftp', 'acquia' or 'lagoon'. +# With an 'ftp' or 's3' fetch source, setting this value makes the fetch +# source a dump store: with VORTEX_PUSH_DB_PROCEED set to 1, 'vortex-fetch-db' +# fetches the dump from this source and 'vortex-push-db' pushes the exported +# dump to the store. _v="VORTEX_REFRESH_DB${_db_index}_SOURCE" VORTEX_REFRESH_DB_SOURCE="${!_v:-}" @@ -106,12 +108,12 @@ fi rebuild_image=0 [ -n "${db_image}" ] && [ -n "${db_image_base}" ] && rebuild_image=1 -# A dump store ('gdrive' or 's3') receives the dump this refresh exports, so -# with a push requested the exported dump must be sanitized. +# A dump store receives the dump this refresh exports, so with a push +# requested the exported dump must be sanitized. refresh_store=0 -if [ -z "${_db_index}" ] && [ -z "${db_image}" ] && [ "${VORTEX_PUSH_DB_PROCEED:-}" = "1" ]; then +if [ -z "${_db_index}" ] && [ -z "${db_image}" ] && [ "${VORTEX_PUSH_DB_PROCEED:-}" = "1" ] && [ -n "${VORTEX_REFRESH_DB_SOURCE}" ]; then case "${fetch_source}" in - gdrive | s3) refresh_store=1 ;; + ftp | s3) refresh_store=1 ;; esac fi @@ -121,7 +123,7 @@ fi if [ "${rebuild_image}" -eq 1 ] && [ "${fetch_source}" = "container_registry" ]; then case "${VORTEX_REFRESH_DB_SOURCE}" in - "" | container_registry | none) fail "The ${db_image} image is built from a database dump file, but VORTEX_REFRESH_DB${_db_index}_SOURCE is \"${VORTEX_REFRESH_DB_SOURCE}\". Set it to the source of the dump file: url, ftp, acquia, lagoon, s3 or gdrive." ;; + "" | container_registry | none) fail "The ${db_image} image is built from a database dump file, but VORTEX_REFRESH_DB${_db_index}_SOURCE is \"${VORTEX_REFRESH_DB_SOURCE}\". Set it to the source of the dump file: url, ftp, acquia, lagoon or s3." ;; esac fi diff --git a/.vortex/tooling/tests/_helper.bash b/.vortex/tooling/tests/_helper.bash index 61bb8a2d46..a22c442003 100644 --- a/.vortex/tooling/tests/_helper.bash +++ b/.vortex/tooling/tests/_helper.bash @@ -122,6 +122,7 @@ setup() { unset VORTEX_FETCH_DB_FORCE unset VORTEX_EXPORT_DB_CONTAINER_REGISTRY_PUSH_PROCEED unset VORTEX_PUSH_DB_PROCEED + unset VORTEX_PUSH_DB_FILE_PREFIX # Isolate ambient credentials from the scripts under test. Scripts fall back # to these unprefixed values whenever their own variable is unset or empty, diff --git a/.vortex/tooling/tests/unit/fetch-db-gdrive.bats b/.vortex/tooling/tests/unit/fetch-db-gdrive.bats deleted file mode 100644 index 987005923e..0000000000 --- a/.vortex/tooling/tests/unit/fetch-db-gdrive.bats +++ /dev/null @@ -1,296 +0,0 @@ -#!/usr/bin/env bats -## -# Unit tests for the 'vortex-fetch-db-gdrive' script. -# -# shellcheck disable=SC2016,SC2030,SC2031,SC2034 - -load ../_helper.bash - -# Writes a service account key file with a newly generated private key. -fixture_gdrive_key() { - openssl genpkey -algorithm RSA -pkeyopt rsa_keygen_bits:2048 -out "${BUILD_DIR}/gdrive.pem" 2>/dev/null - php -r 'print json_encode(["type" => "service_account", "client_email" => "vortex@example.iam.gserviceaccount.com", "private_key" => file_get_contents($argv[1])]);' "${BUILD_DIR}/gdrive.pem" >"${BUILD_DIR}/gdrive.json" -} - -@test "fetch-db-gdrive: Fetch database file successfully with a key file" { - pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 - - rm -rf .data - fixture_gdrive_key - - unset VORTEX_GDRIVE_SERVICE_ACCOUNT_KEY - export VORTEX_FETCH_DB_GDRIVE_SERVICE_ACCOUNT_KEY="${BUILD_DIR}/gdrive.json" - export VORTEX_FETCH_DB_GDRIVE_FOLDER_ID="folder-123" - export VORTEX_FETCH_DB_GDRIVE_DB_DIR=".data" - export VORTEX_FETCH_DB_GDRIVE_DB_FILE="db.sql" - - mock_curl=$(mock_command "curl") - mock_set_output "${mock_curl}" '{"access_token":"test-token","expires_in":3599}' 1 - mock_set_output "${mock_curl}" '{"files":[{"id":"file-456"}]}' 2 - mock_set_side_effect "${mock_curl}" "echo 'CREATE TABLE test (id INT);' > .data/db.sql" 3 - - run .vortex/tooling/src/vortex-fetch-db-gdrive - assert_success - assert_output_contains "[INFO] Started database dump fetch from Google Drive." - assert_output_contains "Remote file: db.sql" - assert_output_contains "Local path: .data/db.sql" - assert_output_contains "Folder ID: folder-123" - assert_output_contains "[ OK ] Authenticated with the service account." - assert_output_contains "[ OK ] Found the database dump file with ID file-456." - assert_output_contains "[ OK ] Downloaded the database dump file." - assert_output_contains "[ OK ] Finished database dump fetch from Google Drive." - assert_output_not_contains "test-token" - - assert_equal "3" "$(mock_get_call_num "${mock_curl}")" - - # The token request carries a signed JWT assertion for the read-only scope. - token_args="$(mock_get_call_args "${mock_curl}" 1)" - assert_string_contains "${token_args}" "https://oauth2.googleapis.com/token" - assert_string_contains "${token_args}" "grant_type=urn:ietf:params:oauth:grant-type:jwt-bearer" - assertion="${token_args##*assertion=}" - assertion="${assertion%% *}" - IFS='.' read -r jwt_header jwt_claims jwt_signature <<<"${assertion}" - assert_not_empty "${jwt_signature}" - claims="$(php -r 'print base64_decode(strtr($argv[1], "-_", "+/"));' "${jwt_claims}")" - assert_string_contains "${claims}" '"iss":"vortex@example.iam.gserviceaccount.com"' - assert_string_contains "${claims}" '"scope":"https:\/\/www.googleapis.com\/auth\/drive.readonly"' - - list_args="$(mock_get_call_args "${mock_curl}" 2)" - assert_string_contains "${list_args}" "https://www.googleapis.com/drive/v3/files" - assert_string_contains "${list_args}" "Authorization: Bearer test-token" - assert_string_contains "${list_args}" "q=name = 'db.sql' and 'folder-123' in parents and trashed = false" - assert_string_contains "${list_args}" "supportsAllDrives=true" - - download_args="$(mock_get_call_args "${mock_curl}" 3)" - assert_string_contains "${download_args}" "Authorization: Bearer test-token" - assert_string_contains "${download_args}" "-o .data/db.sql" - assert_string_contains "${download_args}" "https://www.googleapis.com/drive/v3/files/file-456?alt=media&supportsAllDrives=true" - - assert_file_contains ".data/db.sql" "CREATE TABLE test" - - rm -rf .data - - popd >/dev/null -} - -@test "fetch-db-gdrive: Fetch database file with the key content in the shared variable" { - pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 - - rm -rf .data - fixture_gdrive_key - - export VORTEX_FETCH_DB_GDRIVE_SERVICE_ACCOUNT_KEY="" - VORTEX_GDRIVE_SERVICE_ACCOUNT_KEY="$(cat "${BUILD_DIR}/gdrive.json")" - export VORTEX_GDRIVE_SERVICE_ACCOUNT_KEY - export VORTEX_FETCH_DB_GDRIVE_FOLDER_ID="folder-123" - unset VORTEX_FETCH_DB_GDRIVE_DB_DIR VORTEX_FETCH_DB_GDRIVE_DB_FILE - - mock_curl=$(mock_command "curl") - mock_set_output "${mock_curl}" '{"access_token":"test-token"}' 1 - mock_set_output "${mock_curl}" '{"files":[{"id":"file-456"}]}' 2 - mock_set_status "${mock_curl}" 0 3 - - run .vortex/tooling/src/vortex-fetch-db-gdrive - assert_success - # The default directory and file name come from .env. - assert_output_contains "Local path: ./.data/db.sql" - assert_output_contains "[ OK ] Finished database dump fetch from Google Drive." - - assert_string_contains "$(mock_get_call_args "${mock_curl}" 3)" "-o ./.data/db.sql" - - rm -rf .data - - popd >/dev/null -} - -@test "fetch-db-gdrive: Resolve indexed variables with VORTEX_DB_INDEX" { - pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 - - rm -rf .data - fixture_gdrive_key - - unset VORTEX_GDRIVE_SERVICE_ACCOUNT_KEY - export VORTEX_DB_INDEX=3 - export VORTEX_FETCH_DB3_GDRIVE_SERVICE_ACCOUNT_KEY="${BUILD_DIR}/gdrive.json" - export VORTEX_FETCH_DB3_GDRIVE_FOLDER_ID="folder-789" - export VORTEX_FETCH_DB3_FILE="db3.sql" - - mock_curl=$(mock_command "curl") - mock_set_output "${mock_curl}" '{"access_token":"test-token"}' 1 - mock_set_output "${mock_curl}" '{"files":[{"id":"file-3"}]}' 2 - mock_set_status "${mock_curl}" 0 3 - - run .vortex/tooling/src/vortex-fetch-db-gdrive - assert_success - assert_output_contains "Remote file: db3.sql" - assert_output_contains "Folder ID: folder-789" - - assert_string_contains "$(mock_get_call_args "${mock_curl}" 2)" "q=name = 'db3.sql' and 'folder-789' in parents and trashed = false" - - rm -rf .data - - popd >/dev/null -} - -@test "fetch-db-gdrive: Escape a quote in the file name of the folder query" { - pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 - - rm -rf .data - fixture_gdrive_key - - export VORTEX_FETCH_DB_GDRIVE_SERVICE_ACCOUNT_KEY="${BUILD_DIR}/gdrive.json" - export VORTEX_FETCH_DB_GDRIVE_FOLDER_ID="folder-123" - export VORTEX_FETCH_DB_GDRIVE_DB_FILE="site's.sql" - - mock_curl=$(mock_command "curl") - mock_set_output "${mock_curl}" '{"access_token":"test-token"}' 1 - mock_set_output "${mock_curl}" '{"files":[{"id":"file-456"}]}' 2 - mock_set_status "${mock_curl}" 0 3 - - run .vortex/tooling/src/vortex-fetch-db-gdrive - assert_success - - assert_string_contains "$(mock_get_call_args "${mock_curl}" 2)" "q=name = 'site\\'s.sql' and" - - rm -rf .data - - popd >/dev/null -} - -@test "fetch-db-gdrive: Fail when the service account key is missing" { - pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 - - export VORTEX_FETCH_DB_GDRIVE_SERVICE_ACCOUNT_KEY="" - export VORTEX_GDRIVE_SERVICE_ACCOUNT_KEY="" - export VORTEX_FETCH_DB_GDRIVE_FOLDER_ID="folder-123" - - mock_curl=$(mock_command "curl") - - run .vortex/tooling/src/vortex-fetch-db-gdrive - assert_failure - assert_output_contains "[FAIL] Missing required value for VORTEX_FETCH_DB_GDRIVE_SERVICE_ACCOUNT_KEY or VORTEX_GDRIVE_SERVICE_ACCOUNT_KEY." - assert_equal "0" "$(mock_get_call_num "${mock_curl}")" - - popd >/dev/null -} - -@test "fetch-db-gdrive: Fail when the folder ID is missing" { - pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 - - export VORTEX_FETCH_DB_GDRIVE_SERVICE_ACCOUNT_KEY="{}" - export VORTEX_FETCH_DB_GDRIVE_FOLDER_ID="" - - mock_curl=$(mock_command "curl") - - run .vortex/tooling/src/vortex-fetch-db-gdrive - assert_failure - assert_output_contains "[FAIL] Missing required value for VORTEX_FETCH_DB_GDRIVE_FOLDER_ID." - assert_equal "0" "$(mock_get_call_num "${mock_curl}")" - - popd >/dev/null -} - -@test "fetch-db-gdrive: Fail when the service account key is invalid" { - pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 - - export VORTEX_FETCH_DB_GDRIVE_FOLDER_ID="folder-123" - - mock_curl=$(mock_command "curl") - - for key in "not json" '{"client_email":"vortex@example.iam.gserviceaccount.com"}' '{"client_email":"vortex@example.iam.gserviceaccount.com","private_key":"not a key"}'; do - export VORTEX_FETCH_DB_GDRIVE_SERVICE_ACCOUNT_KEY="${key}" - - run .vortex/tooling/src/vortex-fetch-db-gdrive - assert_failure - assert_output_contains "[FAIL] Unable to authenticate with the service account. Check that VORTEX_FETCH_DB_GDRIVE_SERVICE_ACCOUNT_KEY or VORTEX_GDRIVE_SERVICE_ACCOUNT_KEY is a valid service account JSON key." - done - - assert_equal "0" "$(mock_get_call_num "${mock_curl}")" - - popd >/dev/null -} - -@test "fetch-db-gdrive: Fail when the token request is rejected" { - pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 - - fixture_gdrive_key - - export VORTEX_FETCH_DB_GDRIVE_SERVICE_ACCOUNT_KEY="${BUILD_DIR}/gdrive.json" - export VORTEX_FETCH_DB_GDRIVE_FOLDER_ID="folder-123" - - mock_curl=$(mock_command "curl") - mock_set_output "${mock_curl}" '{"error":"invalid_grant","error_description":"Invalid JWT Signature."}' 1 - - run .vortex/tooling/src/vortex-fetch-db-gdrive - assert_failure - assert_output_contains "[FAIL] Unable to authenticate with the service account." - assert_output_not_contains "Found the database dump file" - assert_equal "1" "$(mock_get_call_num "${mock_curl}")" - - popd >/dev/null -} - -@test "fetch-db-gdrive: Fail when the folder cannot be listed" { - pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 - - fixture_gdrive_key - - export VORTEX_FETCH_DB_GDRIVE_SERVICE_ACCOUNT_KEY="${BUILD_DIR}/gdrive.json" - export VORTEX_FETCH_DB_GDRIVE_FOLDER_ID="folder-123" - - mock_curl=$(mock_command "curl") - mock_set_output "${mock_curl}" '{"access_token":"test-token"}' 1 - mock_set_output "${mock_curl}" '{"error":{"code":404,"message":"File not found: folder-123."}}' 2 - - run .vortex/tooling/src/vortex-fetch-db-gdrive - assert_failure - assert_output_contains "[FAIL] Unable to list the folder-123 folder. Check that the folder ID is correct and that the folder is shared with the service account." - assert_equal "2" "$(mock_get_call_num "${mock_curl}")" - - popd >/dev/null -} - -@test "fetch-db-gdrive: Fail when the folder has no database dump file" { - pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 - - fixture_gdrive_key - - export VORTEX_FETCH_DB_GDRIVE_SERVICE_ACCOUNT_KEY="${BUILD_DIR}/gdrive.json" - export VORTEX_FETCH_DB_GDRIVE_FOLDER_ID="folder-123" - export VORTEX_FETCH_DB_GDRIVE_DB_FILE="db.sql" - - mock_curl=$(mock_command "curl") - mock_set_output "${mock_curl}" '{"access_token":"test-token"}' 1 - mock_set_output "${mock_curl}" '{"files":[]}' 2 - - run .vortex/tooling/src/vortex-fetch-db-gdrive - assert_failure - assert_output_contains "[FAIL] Unable to find the db.sql file in the folder-123 folder." - assert_equal "2" "$(mock_get_call_num "${mock_curl}")" - - popd >/dev/null -} - -@test "fetch-db-gdrive: Fail when the download fails" { - pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 - - rm -rf .data - fixture_gdrive_key - - export VORTEX_FETCH_DB_GDRIVE_SERVICE_ACCOUNT_KEY="${BUILD_DIR}/gdrive.json" - export VORTEX_FETCH_DB_GDRIVE_FOLDER_ID="folder-123" - - mock_curl=$(mock_command "curl") - mock_set_output "${mock_curl}" '{"access_token":"test-token"}' 1 - mock_set_output "${mock_curl}" '{"files":[{"id":"file-456"}]}' 2 - mock_set_status "${mock_curl}" 22 3 - - run .vortex/tooling/src/vortex-fetch-db-gdrive - assert_failure - assert_output_contains "[FAIL] Unable to download the database dump file." - assert_output_not_contains "Finished database dump fetch from Google Drive." - - rm -rf .data - - popd >/dev/null -} diff --git a/.vortex/tooling/tests/unit/fetch-db.bats b/.vortex/tooling/tests/unit/fetch-db.bats index bd8d08e665..0ab820b840 100644 --- a/.vortex/tooling/tests/unit/fetch-db.bats +++ b/.vortex/tooling/tests/unit/fetch-db.bats @@ -14,7 +14,6 @@ stub_fetch_scripts() { stub_sibling "vortex-fetch-db-lagoon" "Fetched from lagoon." stub_sibling "vortex-fetch-db-container-registry" "Fetched from container_registry." stub_sibling "vortex-fetch-db-s3" "Fetched from s3." - stub_sibling "vortex-fetch-db-gdrive" "Fetched from gdrive." } @test "fetch-db: Fetch with URL source" { @@ -308,7 +307,7 @@ EOF export VORTEX_FETCH_DB_PROCEED="1" - for db_source in url ftp acquia lagoon container_registry s3 gdrive; do + for db_source in url ftp acquia lagoon container_registry s3; do export VORTEX_FETCH_DB_SOURCE="${db_source}" run .vortex/tooling/src/vortex-fetch-db @@ -316,7 +315,7 @@ EOF assert_output_contains "Fetched from ${db_source}." assert_output_contains "Finished database fetch." - for other_source in url ftp acquia lagoon container_registry s3 gdrive; do + for other_source in url ftp acquia lagoon container_registry s3; do if [ "${other_source}" != "${db_source}" ]; then assert_output_not_contains "Fetched from ${other_source}." fi @@ -335,7 +334,7 @@ EOF export VORTEX_PUSH_DB_PROCEED="1" export VORTEX_REFRESH_DB_SOURCE="lagoon" - for db_source in gdrive s3; do + for db_source in ftp s3; do export VORTEX_FETCH_DB_SOURCE="${db_source}" run .vortex/tooling/src/vortex-fetch-db @@ -349,62 +348,82 @@ EOF popd >/dev/null } +@test "fetch-db: Fetch an indexed dump store source from its refresh source with a push requested" { + pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 + + stub_fetch_scripts + + export VORTEX_DB_INDEX=3 + export VORTEX_FETCH_DB3_SOURCE="ftp" + export VORTEX_FETCH_DB3_PROCEED="1" + export VORTEX_FETCH_DB3_DIR="${BUILD_DIR}/db3" + export VORTEX_PUSH_DB_PROCEED="1" + export VORTEX_REFRESH_DB3_SOURCE="url" + export VORTEX_REFRESH_DB_SOURCE="lagoon" + + run .vortex/tooling/src/vortex-fetch-db + assert_success + assert_output_contains "Fetching from the url source in VORTEX_REFRESH_DB3_SOURCE as VORTEX_PUSH_DB_PROCEED is set to 1." + assert_output_contains "Fetched from url." + assert_output_not_contains "Fetched from ftp." + assert_output_not_contains "Fetched from lagoon." + + popd >/dev/null +} + @test "fetch-db: Fetch a dump store source from the store without a push requested" { pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 stub_fetch_scripts - export VORTEX_FETCH_DB_SOURCE="gdrive" + export VORTEX_FETCH_DB_SOURCE="ftp" export VORTEX_FETCH_DB_PROCEED="1" export VORTEX_REFRESH_DB_SOURCE="lagoon" unset VORTEX_PUSH_DB_PROCEED run .vortex/tooling/src/vortex-fetch-db assert_success - assert_output_contains "Fetched from gdrive." + assert_output_contains "Fetched from ftp." assert_output_not_contains "Fetched from lagoon." popd >/dev/null } -@test "fetch-db: Fetch an indexed dump store source from the store with a push requested" { +@test "fetch-db: Fetch an ftp source without a refresh source as an upstream source" { pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 stub_fetch_scripts - export VORTEX_DB_INDEX=3 - export VORTEX_FETCH_DB3_SOURCE="gdrive" - export VORTEX_FETCH_DB3_PROCEED="1" - export VORTEX_FETCH_DB3_DIR="${BUILD_DIR}/db3" + export VORTEX_FETCH_DB_SOURCE="ftp" + export VORTEX_FETCH_DB_PROCEED="1" export VORTEX_PUSH_DB_PROCEED="1" - export VORTEX_REFRESH_DB_SOURCE="lagoon" + export VORTEX_REFRESH_DB_SOURCE="" run .vortex/tooling/src/vortex-fetch-db assert_success - # Only the primary database is pushed to a dump store. - assert_output_contains "Fetched from gdrive." - assert_output_not_contains "Fetched from lagoon." + assert_output_not_contains "Fetching from the" + assert_output_contains "Fetched from ftp." popd >/dev/null } -@test "fetch-db: Fail without an upstream source for a dump store source with a push requested" { +@test "fetch-db: Fail with an invalid upstream source for a dump store source with a push requested" { pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 stub_fetch_scripts - export VORTEX_FETCH_DB_SOURCE="gdrive" + export VORTEX_FETCH_DB_SOURCE="ftp" export VORTEX_FETCH_DB_PROCEED="1" export VORTEX_PUSH_DB_PROCEED="1" export VORTEX_FETCH_DB_SEMAPHORE="${BUILD_DIR}/fetch-db-fresh" - for refresh_source in "" gdrive s3 container_registry none; do + for refresh_source in ftp container_registry none; do export VORTEX_REFRESH_DB_SOURCE="${refresh_source}" run .vortex/tooling/src/vortex-fetch-db assert_failure - assert_output_contains "[FAIL] The database dump file pushed to gdrive is fetched from an upstream source, but VORTEX_REFRESH_DB_SOURCE is \"${refresh_source}\". Set it to the source of the dump file: url, ftp, acquia or lagoon." - assert_output_not_contains "Fetched from gdrive." + assert_output_contains "[FAIL] The database dump file pushed to ftp is fetched from an upstream source, but VORTEX_REFRESH_DB_SOURCE is \"${refresh_source}\". Set it to another source of the dump file: url, ftp, acquia, lagoon or s3." + assert_output_not_contains "Fetched from ftp." assert_output_not_contains "Finished database fetch." assert_file_not_exists "${BUILD_DIR}/fetch-db-fresh" done @@ -444,7 +463,7 @@ EOF run .vortex/tooling/src/vortex-fetch-db assert_failure_status 1 assert_output_contains "Started database fetch." - assert_output_contains 'Unsupported database source "VORTEX_CONTAINER_REGISTRY" in VORTEX_FETCH_DB_SOURCE. Use one of: url, ftp, acquia, lagoon, container_registry, s3, gdrive, none.' + assert_output_contains 'Unsupported database source "VORTEX_CONTAINER_REGISTRY" in VORTEX_FETCH_DB_SOURCE. Use one of: url, ftp, acquia, lagoon, container_registry, s3, none.' assert_output_not_contains "Fetched from" assert_output_not_contains "Finished database fetch." assert_file_not_exists "${BUILD_DIR}/fetch-db-fresh" @@ -484,7 +503,7 @@ EOF run .vortex/tooling/src/vortex-fetch-db assert_failure_status 1 assert_output_contains "Started database 2 fetch." - assert_output_contains 'Unsupported database source "sftp" in VORTEX_FETCH_DB2_SOURCE. Use one of: url, ftp, acquia, lagoon, container_registry, s3, gdrive, none.' + assert_output_contains 'Unsupported database source "sftp" in VORTEX_FETCH_DB2_SOURCE. Use one of: url, ftp, acquia, lagoon, container_registry, s3, none.' assert_output_not_contains "Finished database 2 fetch." popd >/dev/null diff --git a/.vortex/tooling/tests/unit/push-db-ftp.bats b/.vortex/tooling/tests/unit/push-db-ftp.bats new file mode 100644 index 0000000000..b94fc182bf --- /dev/null +++ b/.vortex/tooling/tests/unit/push-db-ftp.bats @@ -0,0 +1,204 @@ +#!/usr/bin/env bats +## +# Unit tests for the 'vortex-push-db-ftp' script. +# +# shellcheck disable=SC2030,SC2031,SC2034 + +load ../_helper.bash + +# Exports the FTP values the database is fetched with. +setup_ftp_fetch_values() { + export VORTEX_FETCH_DB_FTP_USER="fetch-user" + export VORTEX_FETCH_DB_FTP_PASS="fetch-pass" + export VORTEX_FETCH_DB_FTP_HOST="ftp.example.com" + export VORTEX_FETCH_DB_FTP_PORT="21" + export VORTEX_FETCH_DB_FTP_FILE="db.sql" + unset VORTEX_PUSH_DB_FTP_USER VORTEX_PUSH_DB_FTP_PASS VORTEX_PUSH_DB_FTP_HOST VORTEX_PUSH_DB_FTP_PORT VORTEX_PUSH_DB_FTP_FILE VORTEX_PUSH_DB_FILE_PREFIX +} + +@test "push-db-ftp: Upload the dump and keep the dump it replaces" { + pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 + + mkdir -p .data + echo "CREATE TABLE test (id INT);" >.data/db.sql + + setup_ftp_fetch_values + + mock_curl=$(mock_command "curl") + mock_set_side_effect "${mock_curl}" "cat >${BUILD_DIR}/curl-config" 1 + + run .vortex/tooling/src/vortex-push-db-ftp + assert_success + assert_output_contains "[INFO] Started database dump push to FTP." + assert_output_contains "Local file: ./.data/db.sql" + assert_output_contains "Remote file: db.sql" + assert_output_contains "Previous file: db-previous.sql" + assert_output_contains "FTP host: ftp.example.com:21" + assert_output_contains "[ OK ] Uploaded the database dump file and kept the dump it replaced." + assert_output_contains "[ OK ] Finished database dump push to FTP." + assert_output_not_contains "fetch-pass" + + assert_equal "1" "$(mock_get_call_num "${mock_curl}")" + + args="$(mock_get_call_args "${mock_curl}" 1)" + assert_string_contains "${args}" "--config - --ftp-create-dirs --upload-file ./.data/db.sql" + # The upload takes a temporary name, and the rename into place comes last. + assert_string_contains "${args}" "--quote -*DELE db-previous.sql --quote -*RNFR db.sql --quote -*RNTO db-previous.sql --quote -RNFR db.sql.uploading --quote -RNTO db.sql" + assert_string_contains "${args}" "ftp://ftp.example.com:21/db.sql.uploading" + # The credentials reach curl on stdin, never as arguments. + assert_string_not_contains "${args}" "fetch-pass" + assert_file_contains "${BUILD_DIR}/curl-config" 'user = "fetch-user:fetch-pass"' + + rm -rf .data + + popd >/dev/null +} + +@test "push-db-ftp: Push to a remote directory with a file name prefix" { + pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 + + mkdir -p .data + echo "database content" >.data/db.sql + + setup_ftp_fetch_values + export VORTEX_PUSH_DB_FTP_HOST="push.example.com" + export VORTEX_PUSH_DB_FTP_PORT="2121" + export VORTEX_PUSH_DB_FTP_FILE="/backups/site/db.sql" + export VORTEX_PUSH_DB_FILE_PREFIX="gha-" + + mock_curl=$(mock_command "curl") + + run .vortex/tooling/src/vortex-push-db-ftp + assert_success + assert_output_contains "Remote file: backups/site/gha-db.sql" + assert_output_contains "Previous file: backups/site/gha-db-previous.sql" + assert_output_contains "FTP host: push.example.com:2121" + + args="$(mock_get_call_args "${mock_curl}" 1)" + # The rename commands run in the remote directory, so they use bare names. + assert_string_contains "${args}" "--quote -*DELE gha-db-previous.sql --quote -*RNFR gha-db.sql --quote -*RNTO gha-db-previous.sql --quote -RNFR gha-db.sql.uploading --quote -RNTO gha-db.sql" + assert_string_contains "${args}" "ftp://push.example.com:2121/backups/site/gha-db.sql.uploading" + + rm -rf .data + + popd >/dev/null +} + +@test "push-db-ftp: Push an indexed database with its own variables and file" { + pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 + + mkdir -p .data + echo "migration content" >.data/db2.sql + + setup_ftp_fetch_values + export VORTEX_DB_INDEX=2 + export VORTEX_FETCH_DB2_FTP_USER="db2-user" + export VORTEX_FETCH_DB2_FTP_PASS="db2-pass" + export VORTEX_FETCH_DB2_FTP_HOST="ftp2.example.com" + export VORTEX_FETCH_DB2_FTP_PORT="21" + export VORTEX_FETCH_DB2_FTP_FILE="db2.sql" + export VORTEX_FETCH_DB2_FILE="db2.sql" + + mock_curl=$(mock_command "curl") + mock_set_side_effect "${mock_curl}" "cat >${BUILD_DIR}/curl-config" 1 + + run .vortex/tooling/src/vortex-push-db-ftp + assert_success + assert_output_contains "[INFO] Started database 2 dump push to FTP." + assert_output_contains "Local file: ./.data/db2.sql" + assert_output_contains "Remote file: db2.sql" + assert_output_contains "Previous file: db2-previous.sql" + assert_output_contains "[ OK ] Finished database 2 dump push to FTP." + + assert_string_contains "$(mock_get_call_args "${mock_curl}" 1)" "ftp://ftp2.example.com:21/db2.sql.uploading" + assert_file_contains "${BUILD_DIR}/curl-config" 'user = "db2-user:db2-pass"' + + rm -rf .data + + popd >/dev/null +} + +@test "push-db-ftp: Escape quotes and backslashes in the credentials" { + pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 + + mkdir -p .data + echo "database content" >.data/db.sql + + setup_ftp_fetch_values + export VORTEX_FETCH_DB_FTP_PASS='pa"ss\word' + + mock_curl=$(mock_command "curl") + mock_set_side_effect "${mock_curl}" "cat >${BUILD_DIR}/curl-config" 1 + + run .vortex/tooling/src/vortex-push-db-ftp + assert_success + + assert_file_contains "${BUILD_DIR}/curl-config" 'user = "fetch-user:pa\"ss\\word"' + + rm -rf .data + + popd >/dev/null +} + +@test "push-db-ftp: Fail when a required value is missing" { + pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 + + mkdir -p .data + echo "database content" >.data/db.sql + + mock_curl=$(mock_command "curl") + + for name in USER PASS HOST FILE; do + setup_ftp_fetch_values + export "VORTEX_FETCH_DB_FTP_${name}=" + + run .vortex/tooling/src/vortex-push-db-ftp + assert_failure + assert_output_contains "[FAIL] Missing required value for VORTEX_PUSH_DB_FTP_${name} or VORTEX_FETCH_DB_FTP_${name}." + done + + assert_equal "0" "$(mock_get_call_num "${mock_curl}")" + + rm -rf .data + + popd >/dev/null +} + +@test "push-db-ftp: Fail when the database dump file does not exist" { + pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 + + rm -rf .data + + setup_ftp_fetch_values + + mock_curl=$(mock_command "curl") + + run .vortex/tooling/src/vortex-push-db-ftp + assert_failure + assert_output_contains "[FAIL] Database dump file ./.data/db.sql does not exist." + assert_output_not_contains "Started database dump push to FTP." + assert_equal "0" "$(mock_get_call_num "${mock_curl}")" + + popd >/dev/null +} + +@test "push-db-ftp: Fail when the upload or a rename fails" { + pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 + + mkdir -p .data + echo "database content" >.data/db.sql + + setup_ftp_fetch_values + + mock_curl=$(mock_command "curl") + mock_set_status "${mock_curl}" 21 1 + + run .vortex/tooling/src/vortex-push-db-ftp + assert_failure + assert_output_contains "[FAIL] Unable to push the database dump file to FTP." + assert_output_not_contains "Finished database dump push to FTP." + + rm -rf .data + + popd >/dev/null +} diff --git a/.vortex/tooling/tests/unit/push-db-gdrive.bats b/.vortex/tooling/tests/unit/push-db-gdrive.bats deleted file mode 100644 index 35773af729..0000000000 --- a/.vortex/tooling/tests/unit/push-db-gdrive.bats +++ /dev/null @@ -1,328 +0,0 @@ -#!/usr/bin/env bats -## -# Unit tests for the 'vortex-push-db-gdrive' script. -# -# shellcheck disable=SC2016,SC2030,SC2031,SC2034 - -load ../_helper.bash - -# Writes a service account key file with a newly generated private key. -fixture_gdrive_key() { - openssl genpkey -algorithm RSA -pkeyopt rsa_keygen_bits:2048 -out "${BUILD_DIR}/gdrive.pem" 2>/dev/null - php -r 'print json_encode(["type" => "service_account", "client_email" => "vortex@example.iam.gserviceaccount.com", "private_key" => file_get_contents($argv[1])]);' "${BUILD_DIR}/gdrive.pem" >"${BUILD_DIR}/gdrive.json" -} - -@test "push-db-gdrive: Create the database dump file in the folder" { - pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 - - mkdir -p .data - echo "CREATE TABLE test (id INT);" >.data/db.sql - fixture_gdrive_key - - unset VORTEX_GDRIVE_SERVICE_ACCOUNT_KEY - export VORTEX_PUSH_DB_GDRIVE_SERVICE_ACCOUNT_KEY="${BUILD_DIR}/gdrive.json" - export VORTEX_PUSH_DB_GDRIVE_FOLDER_ID="folder-123" - export VORTEX_PUSH_DB_GDRIVE_DB_DIR=".data" - export VORTEX_PUSH_DB_GDRIVE_DB_FILE="db.sql" - - mock_curl=$(mock_command "curl") - mock_set_output "${mock_curl}" '{"access_token":"test-token"}' 1 - mock_set_output "${mock_curl}" '{"files":[]}' 2 - mock_set_output "${mock_curl}" $'HTTP/2 200\r\nlocation: https://www.googleapis.com/upload/drive/v3/files?uploadType=resumable&upload_id=session-1\r\ncontent-length: 0\r\n' 3 - mock_set_output "${mock_curl}" $'{"id":"file-new"}\n200' 4 - - run .vortex/tooling/src/vortex-push-db-gdrive - assert_success - assert_output_contains "[INFO] Started database dump push to Google Drive." - assert_output_contains "Local file: .data/db.sql" - assert_output_contains "Remote file: db.sql" - assert_output_contains "Folder ID: folder-123" - assert_output_contains "[ OK ] Authenticated with the service account." - assert_output_contains "Creating a new file in the folder." - assert_output_contains "[ OK ] Started the upload session." - assert_output_contains "[ OK ] Uploaded the database dump file." - assert_output_contains "[ OK ] Finished database dump push to Google Drive." - assert_output_not_contains "test-token" - assert_output_not_contains "session-1" - - assert_equal "4" "$(mock_get_call_num "${mock_curl}")" - - # The token request carries a signed JWT assertion for the full Drive scope. - token_args="$(mock_get_call_args "${mock_curl}" 1)" - assertion="${token_args##*assertion=}" - assertion="${assertion%% *}" - IFS='.' read -r jwt_header jwt_claims jwt_signature <<<"${assertion}" - assert_not_empty "${jwt_signature}" - claims="$(php -r 'print base64_decode(strtr($argv[1], "-_", "+/"));' "${jwt_claims}")" - assert_string_contains "${claims}" '"scope":"https:\/\/www.googleapis.com\/auth\/drive"' - - assert_string_contains "$(mock_get_call_args "${mock_curl}" 2)" "q=name = 'db.sql' and 'folder-123' in parents and trashed = false" - - session_args="$(mock_get_call_args "${mock_curl}" 3)" - assert_string_contains "${session_args}" "-X POST" - assert_string_contains "${session_args}" "Authorization: Bearer test-token" - assert_string_contains "${session_args}" '--data {"name":"db.sql","parents":["folder-123"]}' - assert_string_contains "${session_args}" "https://www.googleapis.com/upload/drive/v3/files?uploadType=resumable&supportsAllDrives=true" - - upload_args="$(mock_get_call_args "${mock_curl}" 4)" - assert_string_contains "${upload_args}" "-X PUT --upload-file .data/db.sql" - assert_string_contains "${upload_args}" "https://www.googleapis.com/upload/drive/v3/files?uploadType=resumable&upload_id=session-1" - - rm -rf .data - - popd >/dev/null -} - -@test "push-db-gdrive: Replace the existing database dump file in place" { - pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 - - mkdir -p .data - echo "database content" >.data/db.sql - fixture_gdrive_key - - export VORTEX_PUSH_DB_GDRIVE_SERVICE_ACCOUNT_KEY="" - VORTEX_GDRIVE_SERVICE_ACCOUNT_KEY="$(cat "${BUILD_DIR}/gdrive.json")" - export VORTEX_GDRIVE_SERVICE_ACCOUNT_KEY - # The folder falls back to the folder the database is fetched from. - export VORTEX_PUSH_DB_GDRIVE_FOLDER_ID="" - export VORTEX_FETCH_DB_GDRIVE_FOLDER_ID="fetch-folder" - unset VORTEX_PUSH_DB_GDRIVE_DB_DIR VORTEX_PUSH_DB_GDRIVE_DB_FILE - - mock_curl=$(mock_command "curl") - mock_set_output "${mock_curl}" '{"access_token":"test-token"}' 1 - mock_set_output "${mock_curl}" '{"files":[{"id":"file-456"}]}' 2 - mock_set_output "${mock_curl}" $'HTTP/2 200\r\nLocation: https://www.googleapis.com/upload/drive/v3/files/file-456?uploadType=resumable&upload_id=session-2\r\n' 3 - mock_set_output "${mock_curl}" $'{"id":"file-456"}\n200' 4 - - run .vortex/tooling/src/vortex-push-db-gdrive - assert_success - assert_output_contains "Local file: ./.data/db.sql" - assert_output_contains "Folder ID: fetch-folder" - assert_output_contains "Replacing the existing file with ID file-456." - assert_output_contains "[ OK ] Finished database dump push to Google Drive." - - session_args="$(mock_get_call_args "${mock_curl}" 3)" - assert_string_contains "${session_args}" "-X PATCH" - assert_string_contains "${session_args}" "--data {}" - assert_string_contains "${session_args}" "https://www.googleapis.com/upload/drive/v3/files/file-456?uploadType=resumable&supportsAllDrives=true" - - assert_string_contains "$(mock_get_call_args "${mock_curl}" 4)" "upload_id=session-2" - - rm -rf .data - - popd >/dev/null -} - -@test "push-db-gdrive: Fail when the service account key is missing" { - pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 - - export VORTEX_PUSH_DB_GDRIVE_SERVICE_ACCOUNT_KEY="" - export VORTEX_GDRIVE_SERVICE_ACCOUNT_KEY="" - export VORTEX_PUSH_DB_GDRIVE_FOLDER_ID="folder-123" - - mock_curl=$(mock_command "curl") - - run .vortex/tooling/src/vortex-push-db-gdrive - assert_failure - assert_output_contains "[FAIL] Missing required value for VORTEX_PUSH_DB_GDRIVE_SERVICE_ACCOUNT_KEY or VORTEX_GDRIVE_SERVICE_ACCOUNT_KEY." - assert_equal "0" "$(mock_get_call_num "${mock_curl}")" - - popd >/dev/null -} - -@test "push-db-gdrive: Fail when the folder ID is missing" { - pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 - - export VORTEX_PUSH_DB_GDRIVE_SERVICE_ACCOUNT_KEY="{}" - export VORTEX_PUSH_DB_GDRIVE_FOLDER_ID="" - export VORTEX_FETCH_DB_GDRIVE_FOLDER_ID="" - - mock_curl=$(mock_command "curl") - - run .vortex/tooling/src/vortex-push-db-gdrive - assert_failure - assert_output_contains "[FAIL] Missing required value for VORTEX_PUSH_DB_GDRIVE_FOLDER_ID or VORTEX_FETCH_DB_GDRIVE_FOLDER_ID." - assert_equal "0" "$(mock_get_call_num "${mock_curl}")" - - popd >/dev/null -} - -@test "push-db-gdrive: Fail when the database dump file does not exist" { - pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 - - rm -rf .data - - export VORTEX_PUSH_DB_GDRIVE_SERVICE_ACCOUNT_KEY="{}" - export VORTEX_PUSH_DB_GDRIVE_FOLDER_ID="folder-123" - export VORTEX_PUSH_DB_GDRIVE_DB_DIR=".data" - export VORTEX_PUSH_DB_GDRIVE_DB_FILE="db.sql" - - mock_curl=$(mock_command "curl") - - run .vortex/tooling/src/vortex-push-db-gdrive - assert_failure - assert_output_contains "[FAIL] Database dump file .data/db.sql does not exist." - assert_output_not_contains "Started database dump push to Google Drive." - assert_equal "0" "$(mock_get_call_num "${mock_curl}")" - - popd >/dev/null -} - -@test "push-db-gdrive: Fail when the service account key is invalid" { - pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 - - mkdir -p .data - echo "database content" >.data/db.sql - - export VORTEX_PUSH_DB_GDRIVE_SERVICE_ACCOUNT_KEY="not json" - export VORTEX_PUSH_DB_GDRIVE_FOLDER_ID="folder-123" - export VORTEX_PUSH_DB_GDRIVE_DB_DIR=".data" - - mock_curl=$(mock_command "curl") - - run .vortex/tooling/src/vortex-push-db-gdrive - assert_failure - assert_output_contains "[FAIL] Unable to authenticate with the service account. Check that VORTEX_PUSH_DB_GDRIVE_SERVICE_ACCOUNT_KEY or VORTEX_GDRIVE_SERVICE_ACCOUNT_KEY is a valid service account JSON key." - assert_equal "0" "$(mock_get_call_num "${mock_curl}")" - - rm -rf .data - - popd >/dev/null -} - -@test "push-db-gdrive: Fail without creating a file when the folder cannot be listed" { - pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 - - mkdir -p .data - echo "database content" >.data/db.sql - fixture_gdrive_key - - export VORTEX_PUSH_DB_GDRIVE_SERVICE_ACCOUNT_KEY="${BUILD_DIR}/gdrive.json" - export VORTEX_PUSH_DB_GDRIVE_FOLDER_ID="folder-123" - export VORTEX_PUSH_DB_GDRIVE_DB_DIR=".data" - - mock_curl=$(mock_command "curl") - mock_set_output "${mock_curl}" '{"access_token":"test-token"}' 1 - mock_set_output "${mock_curl}" '{"error":{"code":403,"message":"Insufficient permissions."}}' 2 - - run .vortex/tooling/src/vortex-push-db-gdrive - assert_failure - assert_output_contains "[FAIL] Unable to list the folder-123 folder. Check that the folder ID is correct and that the folder is shared with the service account." - assert_output_not_contains "Creating a new file in the folder." - assert_equal "2" "$(mock_get_call_num "${mock_curl}")" - - rm -rf .data - - popd >/dev/null -} - -@test "push-db-gdrive: Fail when the upload session has no location" { - pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 - - mkdir -p .data - echo "database content" >.data/db.sql - fixture_gdrive_key - - export VORTEX_PUSH_DB_GDRIVE_SERVICE_ACCOUNT_KEY="${BUILD_DIR}/gdrive.json" - export VORTEX_PUSH_DB_GDRIVE_FOLDER_ID="folder-123" - export VORTEX_PUSH_DB_GDRIVE_DB_DIR=".data" - - mock_curl=$(mock_command "curl") - mock_set_output "${mock_curl}" '{"access_token":"test-token"}' 1 - mock_set_output "${mock_curl}" '{"files":[]}' 2 - mock_set_output "${mock_curl}" $'HTTP/2 403\r\ncontent-type: application/json\r\n' 3 - - run .vortex/tooling/src/vortex-push-db-gdrive - assert_failure - assert_output_contains "[FAIL] Unable to start the upload session: the response has no upload location. Check that the folder is in a shared drive and that the service account can edit it." - assert_equal "3" "$(mock_get_call_num "${mock_curl}")" - - rm -rf .data - - popd >/dev/null -} - -@test "push-db-gdrive: Fail when the upload returns a non-2xx status" { - pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 - - mkdir -p .data - echo "database content" >.data/db.sql - fixture_gdrive_key - - export VORTEX_PUSH_DB_GDRIVE_SERVICE_ACCOUNT_KEY="${BUILD_DIR}/gdrive.json" - export VORTEX_PUSH_DB_GDRIVE_FOLDER_ID="folder-123" - export VORTEX_PUSH_DB_GDRIVE_DB_DIR=".data" - - mock_curl=$(mock_command "curl") - mock_set_output "${mock_curl}" '{"access_token":"test-token"}' 1 - mock_set_output "${mock_curl}" '{"files":[]}' 2 - mock_set_output "${mock_curl}" $'HTTP/2 200\r\nlocation: https://www.googleapis.com/upload/drive/v3/files?upload_id=session-3\r\n' 3 - mock_set_output "${mock_curl}" $'{"error":{"code":403,"message":"The user does not have sufficient permissions."}}\n403' 4 - - run .vortex/tooling/src/vortex-push-db-gdrive - assert_failure - assert_output_contains "The user does not have sufficient permissions." - assert_output_contains "[FAIL] Unable to push to Google Drive: HTTP status 403." - assert_output_not_contains "Finished database dump push to Google Drive." - - rm -rf .data - - popd >/dev/null -} - -@test "push-db-gdrive: Fail when the upload request fails" { - pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 - - mkdir -p .data - echo "database content" >.data/db.sql - fixture_gdrive_key - - export VORTEX_PUSH_DB_GDRIVE_SERVICE_ACCOUNT_KEY="${BUILD_DIR}/gdrive.json" - export VORTEX_PUSH_DB_GDRIVE_FOLDER_ID="folder-123" - export VORTEX_PUSH_DB_GDRIVE_DB_DIR=".data" - - mock_curl=$(mock_command "curl") - mock_set_output "${mock_curl}" '{"access_token":"test-token"}' 1 - mock_set_output "${mock_curl}" '{"files":[]}' 2 - mock_set_output "${mock_curl}" $'HTTP/2 200\r\nlocation: https://www.googleapis.com/upload/drive/v3/files?upload_id=session-4\r\n' 3 - mock_set_status "${mock_curl}" 28 4 - - run .vortex/tooling/src/vortex-push-db-gdrive - assert_failure - assert_output_contains "[FAIL] Google Drive upload request failed with curl exit 28." - - rm -rf .data - - popd >/dev/null -} - -@test "push-db-gdrive: Push the database dump file the database is fetched with" { - pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 - - mkdir -p .data - echo "database content" >.data/db_fetched.sql - fixture_gdrive_key - - export VORTEX_PUSH_DB_GDRIVE_SERVICE_ACCOUNT_KEY="${BUILD_DIR}/gdrive.json" - export VORTEX_PUSH_DB_GDRIVE_FOLDER_ID="folder-123" - export VORTEX_FETCH_DB_FILE="db_fetched.sql" - unset VORTEX_PUSH_DB_GDRIVE_DB_DIR VORTEX_PUSH_DB_GDRIVE_DB_FILE - - mock_curl=$(mock_command "curl") - mock_set_output "${mock_curl}" '{"access_token":"test-token"}' 1 - mock_set_output "${mock_curl}" '{"files":[]}' 2 - mock_set_output "${mock_curl}" $'HTTP/2 200\r\nlocation: https://www.googleapis.com/upload/drive/v3/files?upload_id=session-5\r\n' 3 - mock_set_output "${mock_curl}" $'{"id":"file-new"}\n200' 4 - - run .vortex/tooling/src/vortex-push-db-gdrive - assert_success - assert_output_contains "Local file: ./.data/db_fetched.sql" - assert_output_contains "Remote file: db_fetched.sql" - - assert_string_contains "$(mock_get_call_args "${mock_curl}" 2)" "q=name = 'db_fetched.sql' and" - assert_string_contains "$(mock_get_call_args "${mock_curl}" 4)" "--upload-file ./.data/db_fetched.sql" - - rm -rf .data - - popd >/dev/null -} diff --git a/.vortex/tooling/tests/unit/push-db-s3.bats b/.vortex/tooling/tests/unit/push-db-s3.bats index c467452983..72793df610 100644 --- a/.vortex/tooling/tests/unit/push-db-s3.bats +++ b/.vortex/tooling/tests/unit/push-db-s3.bats @@ -21,6 +21,7 @@ load ../_helper.bash "S3 region: ap-southeast-2" "Storage class: STANDARD" + "@curl * # 0 # \n200" "@curl * # 0 # 200" "[ OK ] Finished database dump push to S3." @@ -61,6 +62,7 @@ load ../_helper.bash "Local file: ./.data/db.sql" "Remote file: db.sql" + "@curl * # 0 # \n200" "@curl * # 0 # 200" "[ OK ] Finished database dump push to S3." @@ -100,6 +102,7 @@ load ../_helper.bash "S3 bucket: shortcut-bucket" "S3 region: us-east-1" + "@curl * # 0 # \n200" "@curl * # 0 # 200" "[ OK ] Finished database dump push to S3." @@ -143,6 +146,7 @@ load ../_helper.bash "Remote file: backup/db_latest.sql" + "@curl * # 0 # \n200" "@curl * # 0 # 200" "[ OK ] Finished database dump push to S3." @@ -183,6 +187,7 @@ load ../_helper.bash "Remote file: backups/daily/db.sql" "S3 prefix: backups/daily/" + "@curl * # 0 # \n200" "@curl * # 0 # 200" "[ OK ] Finished database dump push to S3." @@ -223,6 +228,7 @@ load ../_helper.bash "Remote file: backups/daily/db.sql" "S3 prefix: backups/daily/" + "@curl * # 0 # \n200" "@curl * # 0 # 200" "[ OK ] Finished database dump push to S3." @@ -261,6 +267,7 @@ load ../_helper.bash declare -a STEPS=( "[INFO] Started database dump push to S3." + "@curl * # 0 # \n200" "@curl * # 0 # AccessDenied\n403" "[FAIL] Unable to push to S3: HTTP status 403." @@ -439,6 +446,7 @@ load ../_helper.bash "S3 region: eu-west-1" "S3 prefix: store/" + "@curl * # 0 # \n200" "@curl * # 0 # 200" "[ OK ] Finished database dump push to S3." @@ -484,6 +492,7 @@ load ../_helper.bash "Local file: ./.data/db_fetched.sql" "Remote file: db_fetched.sql" + "@curl * # 0 # \n200" "@curl * # 0 # 200" "[ OK ] Finished database dump push to S3." @@ -509,3 +518,139 @@ load ../_helper.bash popd >/dev/null } + +# Exports the variables of a push to the 'test-bucket' bucket with the +# 'backups/' prefix. +setup_s3_push() { + mkdir -p .data + echo "database content" >.data/db.sql + + unset S3_ACCESS_KEY S3_SECRET_KEY S3_BUCKET S3_REGION S3_PREFIX + export VORTEX_FETCH_DB_S3_ACCESS_KEY="" VORTEX_FETCH_DB_S3_SECRET_KEY="" VORTEX_FETCH_DB_S3_BUCKET="" VORTEX_FETCH_DB_S3_REGION="" VORTEX_FETCH_DB_S3_PREFIX="" + + export VORTEX_PUSH_DB_S3_ACCESS_KEY="testaccesskey" + export VORTEX_PUSH_DB_S3_SECRET_KEY="testsecretkey" + export VORTEX_PUSH_DB_S3_BUCKET="test-bucket" + export VORTEX_PUSH_DB_S3_REGION="ap-southeast-2" + export VORTEX_PUSH_DB_S3_PREFIX="backups" + export VORTEX_PUSH_DB_S3_DB_DIR=".data" + export VORTEX_PUSH_DB_S3_DB_FILE="db.sql" + unset VORTEX_PUSH_DB_S3_REMOTE_FILE VORTEX_PUSH_DB_FILE_PREFIX +} + +@test "push-db-s3: Keep the current dump as the previous one before the upload" { + pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 + + setup_s3_push + + mock_curl=$(mock_command "curl") + mock_set_output "${mock_curl}" $'\n200' 1 + mock_set_output "${mock_curl}" "200" 2 + + run .vortex/tooling/src/vortex-push-db-s3 + assert_success + assert_output_contains "Remote file: backups/db.sql" + assert_output_contains "Previous file: backups/db-previous.sql" + assert_output_contains "[ OK ] Kept the current database dump as the previous one." + assert_output_contains "[ OK ] Uploaded the database dump file." + assert_output_contains "[ OK ] Finished database dump push to S3." + + assert_equal "2" "$(mock_get_call_num "${mock_curl}")" + + # The copy names the current dump as its source and sends no body. + copy_args="$(mock_get_call_args "${mock_curl}" 1)" + assert_string_contains "${copy_args}" "--request PUT" + assert_string_contains "${copy_args}" "--header x-amz-copy-source: /test-bucket/backups/db.sql" + assert_string_not_contains "${copy_args}" "--upload-file" + assert_string_contains "${copy_args}" "https://test-bucket.s3.ap-southeast-2.amazonaws.com/backups/db-previous.sql" + + upload_args="$(mock_get_call_args "${mock_curl}" 2)" + assert_string_contains "${upload_args}" "--upload-file .data/db.sql" + assert_string_contains "${upload_args}" "https://test-bucket.s3.ap-southeast-2.amazonaws.com/backups/db.sql" + + rm -rf .data + + popd >/dev/null +} + +@test "push-db-s3: Upload the first dump when there is no current one" { + pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 + + setup_s3_push + + mock_curl=$(mock_command "curl") + mock_set_output "${mock_curl}" $'NoSuchKey\n404' 1 + mock_set_output "${mock_curl}" "200" 2 + + run .vortex/tooling/src/vortex-push-db-s3 + assert_success + assert_output_contains "[ OK ] Skipped keeping the previous database dump as there is no current one." + assert_output_contains "[ OK ] Finished database dump push to S3." + assert_equal "2" "$(mock_get_call_num "${mock_curl}")" + + rm -rf .data + + popd >/dev/null +} + +@test "push-db-s3: Fail without uploading when the current dump cannot be kept" { + pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 + + setup_s3_push + + # A copy error can come with a 200 status. Each run makes 1 request. + mock_curl=$(mock_command "curl") + mock_set_output "${mock_curl}" $'InternalError\n200' 1 + mock_set_output "${mock_curl}" $'AccessDenied\n403' 2 + + for status in 200 403; do + run .vortex/tooling/src/vortex-push-db-s3 + assert_failure + assert_output_contains "Response: " + assert_output_contains "[FAIL] Unable to keep the current database dump as the previous one: HTTP status ${status}." + assert_output_not_contains "Uploaded the database dump file." + done + + assert_equal "2" "$(mock_get_call_num "${mock_curl}")" + + rm -rf .data + + popd >/dev/null +} + +@test "push-db-s3: Push an indexed database with a file name prefix" { + pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 + + mkdir -p .data + echo "migration content" >.data/db2.sql + + unset S3_ACCESS_KEY S3_SECRET_KEY S3_BUCKET S3_REGION S3_PREFIX + export VORTEX_DB_INDEX=2 + export VORTEX_FETCH_DB2_S3_ACCESS_KEY="db2-access-key" + export VORTEX_FETCH_DB2_S3_SECRET_KEY="db2-secret-key" + export VORTEX_FETCH_DB2_S3_BUCKET="db2-bucket" + export VORTEX_FETCH_DB2_S3_REGION="eu-west-1" + export VORTEX_FETCH_DB2_S3_PREFIX="" + export VORTEX_FETCH_DB2_FILE="db2.sql" + export VORTEX_PUSH_DB_FILE_PREFIX="circleci-" + + mock_curl=$(mock_command "curl") + mock_set_output "${mock_curl}" $'\n200' 1 + mock_set_output "${mock_curl}" "200" 2 + + run .vortex/tooling/src/vortex-push-db-s3 + assert_success + assert_output_contains "[INFO] Started database 2 dump push to S3." + assert_output_contains "Local file: ./.data/db2.sql" + assert_output_contains "Remote file: circleci-db2.sql" + assert_output_contains "Previous file: circleci-db2-previous.sql" + assert_output_contains "S3 bucket: db2-bucket" + assert_output_contains "[ OK ] Finished database 2 dump push to S3." + + assert_string_contains "$(mock_get_call_args "${mock_curl}" 1)" "--header x-amz-copy-source: /db2-bucket/circleci-db2.sql" + assert_string_contains "$(mock_get_call_args "${mock_curl}" 2)" "https://db2-bucket.s3.eu-west-1.amazonaws.com/circleci-db2.sql" + + rm -rf .data + + popd >/dev/null +} diff --git a/.vortex/tooling/tests/unit/push-db.bats b/.vortex/tooling/tests/unit/push-db.bats index d2dee534f0..dbc3d256f4 100644 --- a/.vortex/tooling/tests/unit/push-db.bats +++ b/.vortex/tooling/tests/unit/push-db.bats @@ -2,16 +2,21 @@ ## # Unit tests for the 'vortex-push-db' script. # -# shellcheck disable=SC2030,SC2031 +# The stub markers use single quotes on purpose: the `${...}` must reach the +# generated stub literally and expand when the stub runs - so SC2016 is +# suppressed here. +# +# shellcheck disable=SC2016,SC2030,SC2031 load ../_helper.bash @test "push-db: Skip when the push is not requested" { pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 - stub_sibling "vortex-push-db-gdrive" "Pushed to gdrive." + stub_sibling "vortex-push-db-ftp" "Pushed to ftp." - export VORTEX_FETCH_DB_SOURCE="gdrive" + export VORTEX_FETCH_DB_SOURCE="ftp" + export VORTEX_REFRESH_DB_SOURCE="lagoon" for proceed in "" 0; do export VORTEX_PUSH_DB_PROCEED="${proceed}" @@ -20,7 +25,7 @@ load ../_helper.bash assert_success assert_output_contains "[INFO] Started database push." assert_output_contains "[ OK ] Skipped database push as VORTEX_PUSH_DB_PROCEED is not set to 1." - assert_output_not_contains "Pushed to gdrive." + assert_output_not_contains "Pushed to ftp." assert_output_not_contains "Finished database push." done @@ -30,42 +35,66 @@ load ../_helper.bash @test "push-db: Route each dump store source to its push script" { pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 - stub_sibling "vortex-push-db-gdrive" "Pushed to gdrive." - stub_sibling "vortex-push-db-s3" "Pushed to s3." + stub_sibling "vortex-push-db-ftp" 'Pushed to ftp with index [${VORTEX_DB_INDEX}].' + stub_sibling "vortex-push-db-s3" 'Pushed to s3 with index [${VORTEX_DB_INDEX}].' export VORTEX_PUSH_DB_PROCEED=1 + export VORTEX_REFRESH_DB_SOURCE="lagoon" - for db_source in gdrive s3; do + for db_source in ftp s3; do export VORTEX_FETCH_DB_SOURCE="${db_source}" run .vortex/tooling/src/vortex-push-db assert_success - assert_output_contains "Pushed to ${db_source}." + assert_output_contains "Pushed to ${db_source} with index []." assert_output_contains "[ OK ] Finished database push." - for other_source in gdrive s3; do + for other_source in ftp s3; do [ "${other_source}" = "${db_source}" ] && continue - assert_output_not_contains "Pushed to ${other_source}." + assert_output_not_contains "Pushed to ${other_source}" done done popd >/dev/null } +@test "push-db: Push an indexed database with its own variables" { + pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 + + stub_sibling "vortex-push-db-ftp" 'Pushed to ftp with index [${VORTEX_DB_INDEX}].' + + export VORTEX_DB_INDEX=2 + export VORTEX_PUSH_DB_PROCEED=1 + export VORTEX_FETCH_DB2_SOURCE="ftp" + export VORTEX_REFRESH_DB2_SOURCE="url" + # The indexed database is pushed as fetched, so the sanitization setting + # does not apply to it. + export VORTEX_PROVISION_SANITIZE_DB_SKIP=1 + + run .vortex/tooling/src/vortex-push-db + assert_success + assert_output_contains "[INFO] Started database 2 push." + assert_output_contains "Pushed to ftp with index [2]." + assert_output_contains "[ OK ] Finished database 2 push." + + popd >/dev/null +} + @test "push-db: Skip for a source that does not accept a push" { pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 - stub_sibling "vortex-push-db-gdrive" "Pushed to gdrive." + stub_sibling "vortex-push-db-ftp" "Pushed to ftp." stub_sibling "vortex-push-db-s3" "Pushed to s3." export VORTEX_PUSH_DB_PROCEED=1 + export VORTEX_REFRESH_DB_SOURCE="lagoon" - for db_source in url ftp acquia lagoon container_registry none; do + for db_source in url acquia lagoon container_registry none; do export VORTEX_FETCH_DB_SOURCE="${db_source}" run .vortex/tooling/src/vortex-push-db assert_success - assert_output_contains "[ OK ] Skipped database push as the \"${db_source}\" database source in VORTEX_FETCH_DB_SOURCE does not accept a push. Use one of: gdrive, s3." + assert_output_contains "[ OK ] Skipped database push as the \"${db_source}\" database source in VORTEX_FETCH_DB_SOURCE does not accept a push. Use one of: ftp, s3." assert_output_not_contains "Pushed to" assert_output_not_contains "Finished database push." done @@ -73,18 +102,35 @@ load ../_helper.bash popd >/dev/null } +@test "push-db: Skip for a dump store source without an upstream source" { + pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 + + stub_sibling "vortex-push-db-ftp" "Pushed to ftp." + + export VORTEX_PUSH_DB_PROCEED=1 + export VORTEX_FETCH_DB_SOURCE="ftp" + export VORTEX_REFRESH_DB_SOURCE="" + + run .vortex/tooling/src/vortex-push-db + assert_success + assert_output_contains "[ OK ] Skipped database push as VORTEX_REFRESH_DB_SOURCE is not set, so the ftp source is not a dump store." + assert_output_not_contains "Pushed to ftp." + + popd >/dev/null +} + @test "push-db: Use the source from .env by default" { pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 - stub_sibling "vortex-push-db-gdrive" "Pushed to gdrive." + stub_sibling "vortex-push-db-ftp" "Pushed to ftp." unset VORTEX_FETCH_DB_SOURCE export VORTEX_PUSH_DB_PROCEED=1 run .vortex/tooling/src/vortex-push-db assert_success - assert_output_contains '[ OK ] Skipped database push as the "url" database source in VORTEX_FETCH_DB_SOURCE does not accept a push. Use one of: gdrive, s3.' - assert_output_not_contains "Pushed to gdrive." + assert_output_contains '[ OK ] Skipped database push as the "url" database source in VORTEX_FETCH_DB_SOURCE does not accept a push. Use one of: ftp, s3.' + assert_output_not_contains "Pushed to ftp." popd >/dev/null } @@ -92,16 +138,17 @@ load ../_helper.bash @test "push-db: Refuse to push a database that is not sanitized" { pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 - stub_sibling "vortex-push-db-gdrive" "Pushed to gdrive." + stub_sibling "vortex-push-db-ftp" "Pushed to ftp." - export VORTEX_FETCH_DB_SOURCE="gdrive" + export VORTEX_FETCH_DB_SOURCE="ftp" + export VORTEX_REFRESH_DB_SOURCE="lagoon" export VORTEX_PUSH_DB_PROCEED=1 export VORTEX_PROVISION_SANITIZE_DB_SKIP=1 run .vortex/tooling/src/vortex-push-db assert_failure assert_output_contains "[FAIL] Refused to push the database as VORTEX_PROVISION_SANITIZE_DB_SKIP is set to 1 and the database dump is not sanitized." - assert_output_not_contains "Pushed to gdrive." + assert_output_not_contains "Pushed to ftp." assert_output_not_contains "Finished database push." popd >/dev/null @@ -114,6 +161,7 @@ load ../_helper.bash chmod +x .vortex/tooling/src/vortex-push-db-s3 export VORTEX_FETCH_DB_SOURCE="s3" + export VORTEX_REFRESH_DB_SOURCE="lagoon" export VORTEX_PUSH_DB_PROCEED=1 run .vortex/tooling/src/vortex-push-db diff --git a/.vortex/tooling/tests/unit/refresh-db.bats b/.vortex/tooling/tests/unit/refresh-db.bats index 785087de4a..9160756ad6 100644 --- a/.vortex/tooling/tests/unit/refresh-db.bats +++ b/.vortex/tooling/tests/unit/refresh-db.bats @@ -236,7 +236,7 @@ load ../_helper.bash run .vortex/tooling/src/vortex-refresh-db assert_failure - assert_output_contains "[FAIL] The myorg/db image is built from a database dump file, but VORTEX_REFRESH_DB_SOURCE is \"${refresh_source}\". Set it to the source of the dump file: url, ftp, acquia, lagoon, s3 or gdrive." + assert_output_contains "[FAIL] The myorg/db image is built from a database dump file, but VORTEX_REFRESH_DB_SOURCE is \"${refresh_source}\". Set it to the source of the dump file: url, ftp, acquia, lagoon or s3." assert_output_not_contains "Logged in to the registry." assert_output_not_contains "Fetched the dump file." done @@ -259,7 +259,8 @@ load ../_helper.bash stub_sibling "vortex-fetch-db" "Fetched the dump file." stub_sibling "vortex-export-db" 'Exported the database to ${1}.' - export VORTEX_FETCH_DB_SOURCE="gdrive" + export VORTEX_FETCH_DB_SOURCE="ftp" + export VORTEX_REFRESH_DB_SOURCE="lagoon" mock_command "sleep" >/dev/null mock_docker=$(mock_command "docker") @@ -296,7 +297,8 @@ load ../_helper.bash echo "store" >.data/db.sql unset VORTEX_DB_IMAGE VORTEX_DB_IMAGE_BASE VORTEX_PUSH_DB_PROCEED - export VORTEX_FETCH_DB_SOURCE="gdrive" + export VORTEX_FETCH_DB_SOURCE="ftp" + export VORTEX_REFRESH_DB_SOURCE="lagoon" stub_sibling "vortex-login-container-registry" "Logged in to the registry." stub_sibling "vortex-export-db" 'Exported the database to ${1}.' @@ -316,11 +318,38 @@ load ../_helper.bash popd >/dev/null } +@test "refresh-db: Exports an ftp source database unsanitized without a refresh source" { + pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 + + mkdir -p .data + echo "upstream" >.data/db.sql + + unset VORTEX_DB_IMAGE VORTEX_DB_IMAGE_BASE VORTEX_REFRESH_DB_SOURCE + export VORTEX_FETCH_DB_SOURCE="ftp" + export VORTEX_PUSH_DB_PROCEED=1 + + stub_sibling "vortex-login-container-registry" "Logged in to the registry." + stub_sibling "vortex-export-db" 'Exported the database to ${1}.' + + mock_command "sleep" >/dev/null + mock_docker=$(mock_command "docker") + + run .vortex/tooling/src/vortex-refresh-db + assert_success + assert_output_contains "[ OK ] Finished database refresh." + + # Without a refresh source, the ftp source is not a dump store. + assert_equal "5" "$(mock_get_call_num "${mock_docker}")" + + popd >/dev/null +} + @test "refresh-db: Fails when the sanitization is skipped for a dump store source with a push requested" { pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 unset VORTEX_DB_IMAGE VORTEX_DB_IMAGE_BASE - export VORTEX_FETCH_DB_SOURCE="gdrive" + export VORTEX_FETCH_DB_SOURCE="ftp" + export VORTEX_REFRESH_DB_SOURCE="lagoon" export VORTEX_PUSH_DB_PROCEED=1 export VORTEX_PROVISION_SANITIZE_DB_SKIP=1 @@ -330,7 +359,7 @@ load ../_helper.bash run .vortex/tooling/src/vortex-refresh-db assert_failure - assert_output_contains "[FAIL] The database dump file pushed to gdrive must be sanitized, but VORTEX_PROVISION_SANITIZE_DB_SKIP is set to 1." + assert_output_contains "[FAIL] The database dump file pushed to ftp must be sanitized, but VORTEX_PROVISION_SANITIZE_DB_SKIP is set to 1." assert_output_not_contains "Logged in to the registry." assert_equal "0" "$(mock_get_call_num "${mock_docker}")" @@ -345,6 +374,7 @@ load ../_helper.bash unset VORTEX_DB_IMAGE VORTEX_DB_IMAGE_BASE export VORTEX_FETCH_DB_SOURCE="s3" + export VORTEX_REFRESH_DB_SOURCE="lagoon" export VORTEX_PUSH_DB_PROCEED=1 stub_sibling "vortex-login-container-registry" "Logged in to the registry." From c48d3f03845bde2575fe5cfa364098877c45d8bc Mon Sep 17 00:00:00 2001 From: Alex Skrypnyk Date: Wed, 11 Nov 2026 14:37:51 +1100 Subject: [PATCH 12/21] Renamed the S3 push test loop variable that clashed with the BATS status variable. --- .vortex/tooling/tests/unit/push-db-s3.bats | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.vortex/tooling/tests/unit/push-db-s3.bats b/.vortex/tooling/tests/unit/push-db-s3.bats index 72793df610..b29fd798a1 100644 --- a/.vortex/tooling/tests/unit/push-db-s3.bats +++ b/.vortex/tooling/tests/unit/push-db-s3.bats @@ -603,11 +603,11 @@ setup_s3_push() { mock_set_output "${mock_curl}" $'InternalError\n200' 1 mock_set_output "${mock_curl}" $'AccessDenied\n403' 2 - for status in 200 403; do + for code in 200 403; do run .vortex/tooling/src/vortex-push-db-s3 assert_failure assert_output_contains "Response: " - assert_output_contains "[FAIL] Unable to keep the current database dump as the previous one: HTTP status ${status}." + assert_output_contains "[FAIL] Unable to keep the current database dump as the previous one: HTTP status ${code}." assert_output_not_contains "Uploaded the database dump file." done From ee66ae6d8b41124f8d624941beb054af0c5b4e00 Mon Sep 17 00:00:00 2001 From: Alex Skrypnyk Date: Wed, 11 Nov 2026 14:44:00 +1100 Subject: [PATCH 13/21] Wired the FTP and S3 dump store push into CI and the installer, and documented database storage with diagrams. --- .circleci/config.yml | 22 +- .circleci/vortex-test-common.yml | 14 ++ .env | 47 ++-- .env.local.example | 8 - .github/workflows/build-test-deploy.yml | 42 +++- .github/workflows/vortex-test-didi.yml | 84 ++++++- .../content/continuous-integration/README.mdx | 56 +---- .../content/development/database-storage.mdx | 219 ++++++++++++++++++ .../docs/content/development/provision.mdx | 7 +- .../docs/content/development/variables.mdx | 21 +- .vortex/docs/content/features.mdx | 10 +- .vortex/docs/cspell.json | 4 +- .../Prompts/Handlers/DatabaseFetchSource.php | 10 +- .../Prompts/Handlers/MigrationFetchSource.php | 10 +- .../DatabaseFetchSourceHandlerProcessTest.php | 3 - ...MigrationFetchSourceHandlerProcessTest.php | 17 +- 16 files changed, 416 insertions(+), 158 deletions(-) create mode 100644 .vortex/docs/content/development/database-storage.mdx diff --git a/.circleci/config.yml b/.circleci/config.yml index 025c05723d..8e9a014630 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -372,9 +372,23 @@ jobs: command: | [ ! -f /tmp/fetch-db-fresh ] && echo "==> No freshly fetched database. DB push will not proceed." && exit 0 ./vendor/bin/vortex-push-db + #;< VORTEX_DEV + if [ -n "${VORTEX_PUSH_DB_FTP_HOST:-}" ]; then VORTEX_FETCH_DB_SOURCE=ftp VORTEX_REFRESH_DB_SOURCE=url ./vendor/bin/vortex-push-db; fi + #;> VORTEX_DEV no_output_timeout: 30m #;> DB_PUSH + #;< MIGRATION_DB_PUSH + # Push the freshly-fetched migration database dump to its fetch source, + # next to the primary dump. No-op unless the push was requested. + - run: + name: Push migration DB + command: | + [ ! -f /tmp/fetch-db-fresh ] && echo "==> No freshly fetched database. DB push will not proceed." && exit 0 + VORTEX_DB_INDEX=2 ./vendor/bin/vortex-push-db + no_output_timeout: 30m + #;> MIGRATION_DB_PUSH + - save_cache: # Save cache per default branch and the timestamp. # The cache will not be saved if it already exists. @@ -399,10 +413,12 @@ jobs: # Deploy container image. VORTEX_EXPORT_DB_CONTAINER_REGISTRY_PUSH_PROCEED: 1 #;> DB_IN_IMAGE - #;< DB_PUSH - # Refresh the database dump in the database fetch source. + # Refresh the database dumps in the fetch sources that are dump stores. VORTEX_PUSH_DB_PROCEED: 1 - #;> DB_PUSH + #;< VORTEX_DEV + # Tell this provider's demo database dumps apart on the test FTP server. + VORTEX_PUSH_DB_FILE_PREFIX: circleci- + #;> VORTEX_DEV # Do not build the Drupal front-end. VORTEX_FRONTEND_BUILD_SKIP: 1 #;> !PROVISION_TYPE_PROFILE diff --git a/.circleci/vortex-test-common.yml b/.circleci/vortex-test-common.yml index 7433dfe36e..a901d33adc 100644 --- a/.circleci/vortex-test-common.yml +++ b/.circleci/vortex-test-common.yml @@ -192,9 +192,23 @@ jobs: command: | [ ! -f /tmp/fetch-db-fresh ] && echo "==> No freshly fetched database. DB push will not proceed." && exit 0 ./vendor/bin/vortex-push-db + #;< VORTEX_DEV + if [ -n "${VORTEX_PUSH_DB_FTP_HOST:-}" ]; then VORTEX_FETCH_DB_SOURCE=ftp VORTEX_REFRESH_DB_SOURCE=url ./vendor/bin/vortex-push-db; fi + #;> VORTEX_DEV no_output_timeout: 30m #;> DB_PUSH + #;< MIGRATION_DB_PUSH + # Push the freshly-fetched migration database dump to its fetch source, + # next to the primary dump. No-op unless the push was requested. + - run: + name: Push migration DB + command: | + [ ! -f /tmp/fetch-db-fresh ] && echo "==> No freshly fetched database. DB push will not proceed." && exit 0 + VORTEX_DB_INDEX=2 ./vendor/bin/vortex-push-db + no_output_timeout: 30m + #;> MIGRATION_DB_PUSH + - save_cache: # Save cache per default branch and the timestamp. # The cache will not be saved if it already exists. diff --git a/.env b/.env index eaa76a2e98..41d5604266 100644 --- a/.env +++ b/.env @@ -222,9 +222,9 @@ VORTEX_FETCH_DB_SOURCE=url # VORTEX_DB_IMAGE=your_org/your_site:latest # Source of the database dump file the nightly CI job imports into a clean -# base image to build the database container image: url, ftp, acquia, lagoon, -# s3 or gdrive. The fetch script of that source reads its own variables, such -# as VORTEX_FETCH_DB_URL for url. Add them, and keep secrets in CI. +# base image to build the database container image: url, ftp, acquia, lagoon +# or s3. The fetch script of that source reads its own variables, such as +# VORTEX_FETCH_DB_URL for url. Add them, and keep secrets in CI. # VORTEX_REFRESH_DB_SOURCE=url #;> DB_FETCH_SOURCE_CONTAINER_REGISTRY @@ -258,21 +258,14 @@ VORTEX_FETCH_DB_S3_BUCKET= VORTEX_FETCH_DB_S3_REGION=ap-southeast-2 #;> DB_FETCH_SOURCE_S3 -#;< DB_FETCH_SOURCE_GDRIVE -# Database dump file sourced from Google Drive. - -# Google Drive folder ID with the database dump file. -# -# The ID is the last segment of the folder URL. The folder must be in a shared -# drive and shared with the service account. -VORTEX_FETCH_DB_GDRIVE_FOLDER_ID= -#;> DB_FETCH_SOURCE_GDRIVE - #;< DB_PUSH -# Source of the database dump file the nightly CI job sanitizes and pushes to -# the database fetch source: url, ftp, acquia or lagoon. The fetch script of -# that source reads its own variables, such as VORTEX_FETCH_DB_URL for url. -# Add them, and keep secrets in CI. +# Upstream source of the database dump file, such as url, acquia or lagoon. +# +# Uncomment to make the database fetch source a dump store: the nightly CI +# job fetches the dump from this source, sanitizes it and pushes it to the +# fetch source, keeping the dump it replaces as the previous one. The fetch +# script of this source reads its own variables, such as VORTEX_FETCH_DB_URL +# for url. Add them, and keep secrets in CI. # VORTEX_REFRESH_DB_SOURCE=url #;> DB_PUSH @@ -315,9 +308,9 @@ VORTEX_FETCH_DB2_SOURCE=url # Source of the second database dump file the nightly CI job imports into a # clean base image to rebuild the migration database container image: url, -# ftp, acquia, lagoon, s3 or gdrive. The fetch script of that source reads its -# own variables, such as VORTEX_FETCH_DB2_URL for url. Add them, and keep -# secrets in CI. +# ftp, acquia, lagoon or s3. The fetch script of that source reads its own +# variables, such as VORTEX_FETCH_DB2_URL for url. Add them, and keep secrets +# in CI. # VORTEX_REFRESH_DB2_SOURCE=url #;> MIGRATION_DB_FETCH_SOURCE_CONTAINER_REGISTRY @@ -351,12 +344,14 @@ VORTEX_FETCH_DB2_S3_BUCKET= VORTEX_FETCH_DB2_S3_REGION=ap-southeast-2 #;> MIGRATION_DB_FETCH_SOURCE_S3 -#;< MIGRATION_DB_FETCH_SOURCE_GDRIVE -# Second database dump file sourced from Google Drive. - -# Google Drive folder ID with the second database dump file. -VORTEX_FETCH_DB2_GDRIVE_FOLDER_ID= -#;> MIGRATION_DB_FETCH_SOURCE_GDRIVE +#;< MIGRATION_DB_PUSH +# Upstream source of the second database dump file, such as url or lagoon. +# +# Uncomment to make the second database fetch source a dump store: the +# nightly CI job fetches the dump from this source and pushes it, as fetched, +# to the fetch source, next to the primary dump. +# VORTEX_REFRESH_DB2_SOURCE=url +#;> MIGRATION_DB_PUSH # Environment to fetch the second database from. # diff --git a/.env.local.example b/.env.local.example index 4f87f00489..b7a2eb4d0c 100644 --- a/.env.local.example +++ b/.env.local.example @@ -69,14 +69,6 @@ VORTEX_FETCH_DB_S3_ACCESS_KEY= VORTEX_FETCH_DB_S3_SECRET_KEY= #;> DB_FETCH_SOURCE_S3 -#;< DB_FETCH_SOURCE_GDRIVE -# Database dump file sourced from Google Drive. -# Service account key: Google Cloud console -> IAM & Admin -> Service accounts -> Keys -> Add key - -# Path to the Google Cloud service account JSON key file. Keep the file outside the project directory. -VORTEX_GDRIVE_SERVICE_ACCOUNT_KEY= -#;> DB_FETCH_SOURCE_GDRIVE - #;< DB_FETCH_SOURCE_CONTAINER_REGISTRY # Database image sourced from a container registry. diff --git a/.github/workflows/build-test-deploy.yml b/.github/workflows/build-test-deploy.yml index f876f1867b..b8f27a0700 100644 --- a/.github/workflows/build-test-deploy.yml +++ b/.github/workflows/build-test-deploy.yml @@ -225,13 +225,32 @@ jobs: VORTEX_CI_DB_CACHE_FALLBACK: "yes" # Which branch to use as a source of DB caches. VORTEX_CI_DB_CACHE_BRANCH: "develop" + #;< DB_FETCH_SOURCE_FTP + VORTEX_FETCH_DB_FTP_USER: ${{ secrets.VORTEX_FETCH_DB_FTP_USER }} + VORTEX_FETCH_DB_FTP_PASS: ${{ secrets.VORTEX_FETCH_DB_FTP_PASS }} + #;> DB_FETCH_SOURCE_FTP #;< DB_FETCH_SOURCE_S3 VORTEX_FETCH_DB_S3_ACCESS_KEY: ${{ secrets.VORTEX_FETCH_DB_S3_ACCESS_KEY }} VORTEX_FETCH_DB_S3_SECRET_KEY: ${{ secrets.VORTEX_FETCH_DB_S3_SECRET_KEY }} #;> DB_FETCH_SOURCE_S3 - #;< DB_FETCH_SOURCE_GDRIVE - VORTEX_GDRIVE_SERVICE_ACCOUNT_KEY: ${{ secrets.VORTEX_GDRIVE_SERVICE_ACCOUNT_KEY }} - #;> DB_FETCH_SOURCE_GDRIVE + #;< MIGRATION_DB_FETCH_SOURCE_FTP + VORTEX_FETCH_DB2_FTP_USER: ${{ secrets.VORTEX_FETCH_DB2_FTP_USER }} + VORTEX_FETCH_DB2_FTP_PASS: ${{ secrets.VORTEX_FETCH_DB2_FTP_PASS }} + #;> MIGRATION_DB_FETCH_SOURCE_FTP + #;< MIGRATION_DB_FETCH_SOURCE_S3 + VORTEX_FETCH_DB2_S3_ACCESS_KEY: ${{ secrets.VORTEX_FETCH_DB2_S3_ACCESS_KEY }} + VORTEX_FETCH_DB2_S3_SECRET_KEY: ${{ secrets.VORTEX_FETCH_DB2_S3_SECRET_KEY }} + #;> MIGRATION_DB_FETCH_SOURCE_S3 + #;< VORTEX_DEV + # FTP server that receives the demo database on the nightly run, under + # a prefix that tells this provider's dumps apart. + VORTEX_PUSH_DB_FTP_HOST: ${{ secrets.VORTEX_PUSH_DB_FTP_HOST }} + VORTEX_PUSH_DB_FTP_PORT: ${{ secrets.VORTEX_PUSH_DB_FTP_PORT }} + VORTEX_PUSH_DB_FTP_USER: ${{ secrets.VORTEX_PUSH_DB_FTP_USER }} + VORTEX_PUSH_DB_FTP_PASS: ${{ secrets.VORTEX_PUSH_DB_FTP_PASS }} + VORTEX_PUSH_DB_FTP_FILE: ${{ vars.VORTEX_PUSH_DB_FTP_FILE || 'db.sql' }} + VORTEX_PUSH_DB_FILE_PREFIX: gha- + #;> VORTEX_DEV steps: # Frees disk space by removing preinstalled toolchains unused by this project. Steps run in a container, so the host is reached via the Docker socket. @@ -288,10 +307,8 @@ jobs: # Deploy container image. echo "VORTEX_EXPORT_DB_CONTAINER_REGISTRY_PUSH_PROCEED=1" #;> DB_IN_IMAGE - #;< DB_PUSH - # Refresh the database dump in the database fetch source. + # Refresh the database dumps in the fetch sources that are dump stores. echo "VORTEX_PUSH_DB_PROCEED=1" - #;> DB_PUSH } >> "$GITHUB_ENV" - name: Create cache keys files for database caching @@ -362,9 +379,22 @@ jobs: run: | if [ ! -f /tmp/fetch-db-fresh ]; then echo "==> No freshly fetched database. DB push will not proceed."; exit 0; fi ./vendor/bin/vortex-push-db + #;< VORTEX_DEV + if [ -n "${VORTEX_PUSH_DB_FTP_HOST:-}" ]; then VORTEX_FETCH_DB_SOURCE=ftp VORTEX_REFRESH_DB_SOURCE=url ./vendor/bin/vortex-push-db; fi + #;> VORTEX_DEV timeout-minutes: 30 #;> DB_PUSH + #;< MIGRATION_DB_PUSH + # Push the freshly-fetched migration database dump to its fetch source, + # next to the primary dump. No-op unless the push was requested. + - name: Push migration DB + run: | + if [ ! -f /tmp/fetch-db-fresh ]; then echo "==> No freshly fetched database. DB push will not proceed."; exit 0; fi + VORTEX_DB_INDEX=2 ./vendor/bin/vortex-push-db + timeout-minutes: 30 + #;> MIGRATION_DB_PUSH + # Save cache per default branch and the timestamp. # The cache will not be saved if it already exists. # Note that the cache fallback flag is enabled for this case in order diff --git a/.github/workflows/vortex-test-didi.yml b/.github/workflows/vortex-test-didi.yml index 4a8857b8ab..0741270dd3 100644 --- a/.github/workflows/vortex-test-didi.yml +++ b/.github/workflows/vortex-test-didi.yml @@ -64,13 +64,32 @@ jobs: VORTEX_CI_DB_CACHE_FALLBACK: "yes" # Which branch to use as a source of DB caches. VORTEX_CI_DB_CACHE_BRANCH: vortex-dev-didi-fi + #;< DB_FETCH_SOURCE_FTP + VORTEX_FETCH_DB_FTP_USER: ${{ secrets.VORTEX_FETCH_DB_FTP_USER }} + VORTEX_FETCH_DB_FTP_PASS: ${{ secrets.VORTEX_FETCH_DB_FTP_PASS }} + #;> DB_FETCH_SOURCE_FTP #;< DB_FETCH_SOURCE_S3 VORTEX_FETCH_DB_S3_ACCESS_KEY: ${{ secrets.VORTEX_FETCH_DB_S3_ACCESS_KEY }} VORTEX_FETCH_DB_S3_SECRET_KEY: ${{ secrets.VORTEX_FETCH_DB_S3_SECRET_KEY }} #;> DB_FETCH_SOURCE_S3 - #;< DB_FETCH_SOURCE_GDRIVE - VORTEX_GDRIVE_SERVICE_ACCOUNT_KEY: ${{ secrets.VORTEX_GDRIVE_SERVICE_ACCOUNT_KEY }} - #;> DB_FETCH_SOURCE_GDRIVE + #;< MIGRATION_DB_FETCH_SOURCE_FTP + VORTEX_FETCH_DB2_FTP_USER: ${{ secrets.VORTEX_FETCH_DB2_FTP_USER }} + VORTEX_FETCH_DB2_FTP_PASS: ${{ secrets.VORTEX_FETCH_DB2_FTP_PASS }} + #;> MIGRATION_DB_FETCH_SOURCE_FTP + #;< MIGRATION_DB_FETCH_SOURCE_S3 + VORTEX_FETCH_DB2_S3_ACCESS_KEY: ${{ secrets.VORTEX_FETCH_DB2_S3_ACCESS_KEY }} + VORTEX_FETCH_DB2_S3_SECRET_KEY: ${{ secrets.VORTEX_FETCH_DB2_S3_SECRET_KEY }} + #;> MIGRATION_DB_FETCH_SOURCE_S3 + #;< VORTEX_DEV + # FTP server that receives the demo database on the nightly run, under + # a prefix that tells this provider's dumps apart. + VORTEX_PUSH_DB_FTP_HOST: ${{ secrets.VORTEX_PUSH_DB_FTP_HOST }} + VORTEX_PUSH_DB_FTP_PORT: ${{ secrets.VORTEX_PUSH_DB_FTP_PORT }} + VORTEX_PUSH_DB_FTP_USER: ${{ secrets.VORTEX_PUSH_DB_FTP_USER }} + VORTEX_PUSH_DB_FTP_PASS: ${{ secrets.VORTEX_PUSH_DB_FTP_PASS }} + VORTEX_PUSH_DB_FTP_FILE: ${{ vars.VORTEX_PUSH_DB_FTP_FILE || 'db.sql' }} + VORTEX_PUSH_DB_FILE_PREFIX: gha- + #;> VORTEX_DEV VORTEX_FETCH_DB_SOURCE: url VORTEX_FETCH_DB_FORCE: 1 VORTEX_DB_IMAGE: drevops/vortex-dev-mariadb-drupal-data-demo-destination-11.x @@ -136,10 +155,8 @@ jobs: # Deploy container image. echo "VORTEX_EXPORT_DB_CONTAINER_REGISTRY_PUSH_PROCEED=1" #;> DB_IN_IMAGE - #;< DB_PUSH - # Refresh the database dump in the database fetch source. + # Refresh the database dumps in the fetch sources that are dump stores. echo "VORTEX_PUSH_DB_PROCEED=1" - #;> DB_PUSH } >> "$GITHUB_ENV" - name: Create cache keys files for database caching @@ -210,9 +227,22 @@ jobs: run: | if [ ! -f /tmp/fetch-db-fresh ]; then echo "==> No freshly fetched database. DB push will not proceed."; exit 0; fi ./vendor/bin/vortex-push-db + #;< VORTEX_DEV + if [ -n "${VORTEX_PUSH_DB_FTP_HOST:-}" ]; then VORTEX_FETCH_DB_SOURCE=ftp VORTEX_REFRESH_DB_SOURCE=url ./vendor/bin/vortex-push-db; fi + #;> VORTEX_DEV timeout-minutes: 30 #;> DB_PUSH + #;< MIGRATION_DB_PUSH + # Push the freshly-fetched migration database dump to its fetch source, + # next to the primary dump. No-op unless the push was requested. + - name: Push migration DB + run: | + if [ ! -f /tmp/fetch-db-fresh ]; then echo "==> No freshly fetched database. DB push will not proceed."; exit 0; fi + VORTEX_DB_INDEX=2 ./vendor/bin/vortex-push-db + timeout-minutes: 30 + #;> MIGRATION_DB_PUSH + # Save cache per default branch and the timestamp. # The cache will not be saved if it already exists. # Note that the cache fallback flag is enabled for this case in order @@ -592,13 +622,32 @@ jobs: VORTEX_CI_DB_CACHE_FALLBACK: "yes" # Which branch to use as a source of DB caches. VORTEX_CI_DB_CACHE_BRANCH: vortex-dev-didi-ii + #;< DB_FETCH_SOURCE_FTP + VORTEX_FETCH_DB_FTP_USER: ${{ secrets.VORTEX_FETCH_DB_FTP_USER }} + VORTEX_FETCH_DB_FTP_PASS: ${{ secrets.VORTEX_FETCH_DB_FTP_PASS }} + #;> DB_FETCH_SOURCE_FTP #;< DB_FETCH_SOURCE_S3 VORTEX_FETCH_DB_S3_ACCESS_KEY: ${{ secrets.VORTEX_FETCH_DB_S3_ACCESS_KEY }} VORTEX_FETCH_DB_S3_SECRET_KEY: ${{ secrets.VORTEX_FETCH_DB_S3_SECRET_KEY }} #;> DB_FETCH_SOURCE_S3 - #;< DB_FETCH_SOURCE_GDRIVE - VORTEX_GDRIVE_SERVICE_ACCOUNT_KEY: ${{ secrets.VORTEX_GDRIVE_SERVICE_ACCOUNT_KEY }} - #;> DB_FETCH_SOURCE_GDRIVE + #;< MIGRATION_DB_FETCH_SOURCE_FTP + VORTEX_FETCH_DB2_FTP_USER: ${{ secrets.VORTEX_FETCH_DB2_FTP_USER }} + VORTEX_FETCH_DB2_FTP_PASS: ${{ secrets.VORTEX_FETCH_DB2_FTP_PASS }} + #;> MIGRATION_DB_FETCH_SOURCE_FTP + #;< MIGRATION_DB_FETCH_SOURCE_S3 + VORTEX_FETCH_DB2_S3_ACCESS_KEY: ${{ secrets.VORTEX_FETCH_DB2_S3_ACCESS_KEY }} + VORTEX_FETCH_DB2_S3_SECRET_KEY: ${{ secrets.VORTEX_FETCH_DB2_S3_SECRET_KEY }} + #;> MIGRATION_DB_FETCH_SOURCE_S3 + #;< VORTEX_DEV + # FTP server that receives the demo database on the nightly run, under + # a prefix that tells this provider's dumps apart. + VORTEX_PUSH_DB_FTP_HOST: ${{ secrets.VORTEX_PUSH_DB_FTP_HOST }} + VORTEX_PUSH_DB_FTP_PORT: ${{ secrets.VORTEX_PUSH_DB_FTP_PORT }} + VORTEX_PUSH_DB_FTP_USER: ${{ secrets.VORTEX_PUSH_DB_FTP_USER }} + VORTEX_PUSH_DB_FTP_PASS: ${{ secrets.VORTEX_PUSH_DB_FTP_PASS }} + VORTEX_PUSH_DB_FTP_FILE: ${{ vars.VORTEX_PUSH_DB_FTP_FILE || 'db.sql' }} + VORTEX_PUSH_DB_FILE_PREFIX: gha- + #;> VORTEX_DEV VORTEX_FETCH_DB_SOURCE: container_registry VORTEX_FETCH_DB_FORCE: 1 VORTEX_DB_IMAGE: drevops/vortex-dev-mariadb-drupal-data-demo-destination-11.x @@ -663,10 +712,8 @@ jobs: # Deploy container image. echo "VORTEX_EXPORT_DB_CONTAINER_REGISTRY_PUSH_PROCEED=1" #;> DB_IN_IMAGE - #;< DB_PUSH - # Refresh the database dump in the database fetch source. + # Refresh the database dumps in the fetch sources that are dump stores. echo "VORTEX_PUSH_DB_PROCEED=1" - #;> DB_PUSH } >> "$GITHUB_ENV" - name: Create cache keys files for database caching @@ -756,9 +803,22 @@ jobs: run: | if [ ! -f /tmp/fetch-db-fresh ]; then echo "==> No freshly fetched database. DB push will not proceed."; exit 0; fi ./vendor/bin/vortex-push-db + #;< VORTEX_DEV + if [ -n "${VORTEX_PUSH_DB_FTP_HOST:-}" ]; then VORTEX_FETCH_DB_SOURCE=ftp VORTEX_REFRESH_DB_SOURCE=url ./vendor/bin/vortex-push-db; fi + #;> VORTEX_DEV timeout-minutes: 30 #;> DB_PUSH + #;< MIGRATION_DB_PUSH + # Push the freshly-fetched migration database dump to its fetch source, + # next to the primary dump. No-op unless the push was requested. + - name: Push migration DB + run: | + if [ ! -f /tmp/fetch-db-fresh ]; then echo "==> No freshly fetched database. DB push will not proceed."; exit 0; fi + VORTEX_DB_INDEX=2 ./vendor/bin/vortex-push-db + timeout-minutes: 30 + #;> MIGRATION_DB_PUSH + # Save cache per default branch and the timestamp. # The cache will not be saved if it already exists. # Note that the cache fallback flag is enabled for this case in order diff --git a/.vortex/docs/content/continuous-integration/README.mdx b/.vortex/docs/content/continuous-integration/README.mdx index 2f47a89386..3668dbaabd 100644 --- a/.vortex/docs/content/continuous-integration/README.mdx +++ b/.vortex/docs/content/continuous-integration/README.mdx @@ -79,9 +79,9 @@ site from: - Caches the dump, so the `build` job and later runs skip the download - On the nightly run of a project that stores its database in a container image, pushes the refreshed image to the registry -- On the nightly run of a project with the `s3` or `gdrive` fetch source, - pushes a sanitized dump to that source (see [Push the sanitized database - to a dump store](#push-the-sanitized-database-to-a-dump-store)) +- On the nightly run of a project with a dump store, pushes the sanitized + dump to the FTP server or S3 bucket (see [Database + storage](../development/database-storage.mdx#fetch-from-a-dump-store)) The job exists only when the site provisions from a database dump. With the `profile` provision type, the pipeline has no `database` job and no nightly @@ -389,7 +389,7 @@ future **Vortex** update ships. The scheduled run builds a database container image from a dump file. With the `container_registry` fetch source, set `VORTEX_REFRESH_DB_SOURCE` to the -source of that file: `url`, `ftp`, `acquia`, `lagoon`, `s3` or `gdrive`. +source of that file: `url`, `ftp`, `acquia`, `lagoon` or `s3`. `VORTEX_REFRESH_DB_SOURCE` selects a fetch script, and each fetch script reads its own variables. The installer adds to `.env` the variables of the @@ -412,54 +412,6 @@ For the migration database, set `VORTEX_REFRESH_DB2_SOURCE` and the variables apply to both databases. See [Refresh the migration database image](../development/migrations.mdx#refresh-the-migration-database-image). -### Push the sanitized database to a dump store - -A dump store is an `s3` or `gdrive` fetch source that holds a sanitized -copy of your hosting database. Developers and branch builds fetch that -copy and need no hosting access. On the nightly run, the `database` job -fetches a dump from `VORTEX_REFRESH_DB_SOURCE` and sanitizes it. Its -**Push DB** step then uploads the dump to the fetch source. - -To refresh a dump store every night: - -1. In `.env`, set `VORTEX_FETCH_DB_SOURCE` to `s3` or `gdrive`. The - installer sets it and adds the variables of that source. -2. In `.env`, set `VORTEX_REFRESH_DB_SOURCE` to the hosting source: `url`, - `ftp`, `acquia` or `lagoon`. Add the variables of that source from the - table in [Dump file source for a database - image](#dump-file-source-for-a-database-image). -3. Add the credentials of the dump store as CI secrets: - - | Source | Secrets | - |---|---| - | `s3` | `VORTEX_FETCH_DB_S3_ACCESS_KEY` and `VORTEX_FETCH_DB_S3_SECRET_KEY` | - | `gdrive` | `VORTEX_GDRIVE_SERVICE_ACCOUNT_KEY`, with the content of the JSON key file on 1 line, such as the output of `jq -c . key.json` | - - GitHub Actions passes these secrets to the `database` job. Add the - secrets of the hosting source to the `env` block of that job. - -4. After the next nightly run, confirm that its **Push DB** step ends with - `Finished database push.` - -The `s3` push writes to the bucket, region and prefix that the fetch -reads. The `gdrive` push replaces a file with the same name in place, so -the file keeps its ID and sharing. Both pushes stop with an error when -`VORTEX_PROVISION_SANITIZE_DB_SKIP` is `1`. - -The `gdrive` source signs in with a Google Cloud service account key. -Service accounts have no storage of their own, so the folder must be in a -shared drive. To set up the folder: - -1. In the Google Cloud console, enable the Google Drive API and create a - service account with a JSON key. -2. In a shared drive, create a folder and add the service account email - as a **Content manager**. -3. Copy the folder ID, the last segment of the folder URL, into - `VORTEX_FETCH_DB_GDRIVE_FOLDER_ID` in `.env`. -4. Give developers the key of a second service account with the - **Viewer** role. They set `VORTEX_GDRIVE_SERVICE_ACCOUNT_KEY` in - `.env.local` to the path of the key file. - ## Trigger conditions Both providers build on branch pushes, pull requests and tag pushes, plus diff --git a/.vortex/docs/content/development/database-storage.mdx b/.vortex/docs/content/development/database-storage.mdx new file mode 100644 index 0000000000..e0a7dad597 --- /dev/null +++ b/.vortex/docs/content/development/database-storage.mdx @@ -0,0 +1,219 @@ +--- +sidebar_label: Database storage +sidebar_position: 6 +--- + +# Database storage + +Your site's database reaches a developer's machine and CI in 1 of 3 ways. +The fetch source in `VORTEX_FETCH_DB_SOURCE` picks the way. This page +shows how fetching, refreshing and pushing the database work in each case. +It also covers setting up an FTP server or an S3 bucket as a dump store, +so developers get a sanitized dump without access to your hosting. + +The page covers these commands: + +- `ahoy fetch-db` (`vortex-fetch-db`) downloads a dump into `.data` +- `vortex-refresh-db` provisions a site from the fetched dump in CI and + exports it again +- `vortex-push-db` uploads the exported dump to a dump store on the + nightly CI run + +## Fetch from the source + +With the `url`, `ftp`, `acquia`, `lagoon` or `s3` source, every +environment downloads the dump from that source. Developers need +credentials for it. + +```text + Developer CI (every run with no cached dump) + ───────── ────────────────────────────────── + ahoy fetch-db Fetch DB + │ │ + ▼ ▼ + .data/db.sql ◀── source ──▶ .data/db.sql + │ │ + ▼ ▼ + ahoy provision Export DB (vortex-refresh-db) + │ provisions the dump and exports + │ it as fetched + ▼ + cache ──▶ build job +``` + +## Fetch a database image + +With the `container_registry` source, the database lives in a container +image. Developers and CI pull the image. The nightly CI run rebuilds the +image from a dump file and pushes it back. See [Dump file source for a +database +image](../continuous-integration/README.mdx#dump-file-source-for-a-database-image). + +```text + Nightly CI + ────────── + Fetch DB ── VORTEX_REFRESH_DB_SOURCE (for example, lagoon) + │ + ▼ + Export DB (vortex-refresh-db) + │ imports the dump into a clean base image + ▼ + Deploy DB image (vortex-push-db-image) ──▶ container registry +``` + +## Fetch from a dump store + +A dump store is an `ftp` or `s3` source with an upstream source set in +`VORTEX_REFRESH_DB_SOURCE`, such as `lagoon`. Developers and branch builds +fetch from the store and need no hosting access. The nightly CI run +refreshes the store from the upstream source: + +```text + Nightly CI (VORTEX_PUSH_DB_PROCEED=1) + ───────────────────────────────────── + Fetch DB + │ fetches from VORTEX_REFRESH_DB_SOURCE in place of the store + ▼ + Export DB (vortex-refresh-db) + │ provisions the dump, sanitizes it and exports it + ▼ + Push DB (vortex-push-db) + │ 1. keeps the current dump as db-previous.sql + │ 2. uploads the new dump as db.sql + ▼ + FTP server or S3 bucket + ├── db.sql ◀── ahoy fetch-db, branch builds + └── db-previous.sql +``` + +The store always holds 2 copies: the current dump and the dump it +replaced. Fetching reads the current dump. To fetch the previous dump from +an FTP store, set `VORTEX_FETCH_DB_FTP_FILE` to its path in `.env.local`, +for example `your_site/db-previous.sql`. + +Every other CI run fetches from the store, so only the nightly run needs +the credentials of the upstream source. + +The pushed dump leaves your hosting and CI, so it's always sanitized. The +nightly run stops with an error when `VORTEX_PROVISION_SANITIZE_DB_SKIP` +is `1`. + +### Multiple databases + +The migration database pushes its own dump file to its own fetch source. +Its files sit next to the primary dump in the same store: + +```text + FTP server or S3 bucket + ├── db.sql primary database (VORTEX_FETCH_DB_FTP_FILE) + ├── db-previous.sql + ├── db2.sql migration database (VORTEX_FETCH_DB2_FTP_FILE) + └── db2-previous.sql +``` + +To make the migration source a dump store, set +`VORTEX_REFRESH_DB2_SOURCE` to its upstream source. The +**Push migration DB** step pushes the migration dump as fetched, because +**Vortex** sanitizes only the Drupal database. + +## Set up an FTP dump store + +Before you begin, create an FTP account that can write to a directory on +the server. The push uploads the dump under a temporary name, then renames +the files into place. The server must allow the `DELE`, `RNFR` and `RNTO` +commands, which standard FTP servers do. + +1. In `.env`, set the fetch source and the server: + + ```shell + VORTEX_FETCH_DB_SOURCE=ftp + VORTEX_FETCH_DB_FTP_HOST=ftp.example.com + VORTEX_FETCH_DB_FTP_PORT=21 + VORTEX_FETCH_DB_FTP_FILE=your_site/db.sql + ``` + +2. In `.env`, uncomment `VORTEX_REFRESH_DB_SOURCE` and set it to your + upstream source, for example `lagoon`. Add the variables and secrets of + that source from the table in [Dump file source for a database + image](../continuous-integration/README.mdx#dump-file-source-for-a-database-image). +3. Add `VORTEX_FETCH_DB_FTP_USER` and `VORTEX_FETCH_DB_FTP_PASS` as CI + secrets. GitHub Actions passes them to the `database` job. CircleCI + reads them from the project environment variables. +4. Ask each developer to set `VORTEX_FETCH_DB_FTP_USER` and + `VORTEX_FETCH_DB_FTP_PASS` in `.env.local`. +5. After the next nightly run, confirm that the **Push DB** step ends with + `Finished database push.` and that the server holds `db.sql`. + +To give CI an account with write access and developers a read-only one, +set the CI account in `VORTEX_PUSH_DB_FTP_USER` and +`VORTEX_PUSH_DB_FTP_PASS`. On GitHub Actions, map those 2 secrets in the +`env` block of the `database` job. + +FTP sends the credentials and the dump unencrypted. Use a server on a +network you trust, or the S3 store. + +## Set up an S3 dump store + +Before you begin, create a private S3 bucket with public access blocked. + +1. Create an IAM user for CI with this policy. Replace `your-bucket` with + your bucket name: + + ```json + { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": ["s3:GetObject", "s3:PutObject"], + "Resource": "arn:aws:s3:::your-bucket/*" + }, + { + "Effect": "Allow", + "Action": ["s3:ListBucket"], + "Resource": "arn:aws:s3:::your-bucket" + } + ] + } + ``` + + The push copies the current dump before the upload. Without + `s3:ListBucket`, S3 answers a missing dump with an access error, and + the first push fails. + +2. In `.env`, set the fetch source and the bucket: + + ```shell + VORTEX_FETCH_DB_SOURCE=s3 + VORTEX_FETCH_DB_S3_BUCKET=your-bucket + VORTEX_FETCH_DB_S3_REGION=ap-southeast-2 + VORTEX_FETCH_DB_S3_PREFIX=your_site + ``` + +3. In `.env`, uncomment `VORTEX_REFRESH_DB_SOURCE` and set it to your + upstream source, and add the variables and secrets of that source. +4. Add the access key of the IAM user as the + `VORTEX_FETCH_DB_S3_ACCESS_KEY` and `VORTEX_FETCH_DB_S3_SECRET_KEY` CI + secrets. +5. Create a second IAM user for developers with `s3:GetObject` only. Ask + each developer to set its keys in `.env.local`. +6. After the next nightly run, confirm that the **Push DB** step ends with + `Finished database push.` and that the bucket holds `db.sql`. + +S3 copies a dump of up to 5 GB in 1 request, so the push fails for a +larger dump. + +## Tell pushes apart + +`VORTEX_PUSH_DB_FILE_PREFIX` adds a prefix to the names of the pushed +files, such as `gha-db.sql` and `gha-db-previous.sql`. Use a different +prefix in each place that pushes to the same store. A fetch reads the +prefixed files only when its remote filename includes the prefix. + +## What's next + +- [Variables reference](variables.mdx) lists every `VORTEX_PUSH_DB_` + variable +- [Provisioning](provision.mdx) covers importing the fetched dump +- [Continuous integration](../continuous-integration/README.mdx) covers + the `database` job and its caching diff --git a/.vortex/docs/content/development/provision.mdx b/.vortex/docs/content/development/provision.mdx index eecfbc5c06..4117778be9 100644 --- a/.vortex/docs/content/development/provision.mdx +++ b/.vortex/docs/content/development/provision.mdx @@ -375,14 +375,13 @@ following values: - `container_registry`: a database container image, pulled instead of a file - `s3`: a file in an S3 bucket -- `gdrive`: a file in a Google Drive folder - `none`: no source; you put the dump into `.data` yourself Any other value stops the command with an error that lists these values. -The nightly CI run can refresh an `s3` or `gdrive` source with a sanitized -dump from your hosting. See [Push the sanitized database to a dump -store](../continuous-integration/README.mdx#push-the-sanitized-database-to-a-dump-store). +An `ftp` or `s3` source can also be a dump store that the nightly CI run +refreshes with a sanitized dump. See [Database +storage](database-storage.mdx). Each source reads its own settings, which the [Variables reference](variables.mdx) lists. The hosting sources copy the diff --git a/.vortex/docs/content/development/variables.mdx b/.vortex/docs/content/development/variables.mdx index 34081e0e34..b36750e223 100644 --- a/.vortex/docs/content/development/variables.mdx +++ b/.vortex/docs/content/development/variables.mdx @@ -321,7 +321,6 @@ The list below is automatically generated with [Shellvar](https://github.com/ale | `VORTEX_FETCH_DB2_FTP_FILE` | Second database dump FTP file name. | `db2.sql` | `.env` | | `VORTEX_FETCH_DB2_FTP_HOST` | Second database dump FTP host. | `UNDEFINED` | `.env` | | `VORTEX_FETCH_DB2_FTP_PORT` | Second database dump FTP port. | `21` | `.env` | -| `VORTEX_FETCH_DB2_GDRIVE_FOLDER_ID` | Google Drive folder ID with the second database dump file. | `UNDEFINED` | `.env` | | `VORTEX_FETCH_DB2_S3_BUCKET` | AWS S3 bucket name for second database fetch. | `UNDEFINED` | `.env` | | `VORTEX_FETCH_DB2_S3_REGION` | AWS S3 region. | `ap-southeast-2` | `.env` | | `VORTEX_FETCH_DB2_SOURCE` | Second database fetch source. | `url` | `.env` | @@ -351,10 +350,6 @@ The list below is automatically generated with [Shellvar](https://github.com/ale | `VORTEX_FETCH_DB_FTP_PASS` | Database dump FTP password. | `UNDEFINED` | `.env.local.example`, `.vortex/tooling/src/vortex-fetch-db-ftp` | | `VORTEX_FETCH_DB_FTP_PORT` | Database dump FTP port. | `21` | `.env`, `.vortex/tooling/src/vortex-fetch-db-ftp` | | `VORTEX_FETCH_DB_FTP_USER` | Database dump FTP user. | `UNDEFINED` | `.env.local.example`, `.vortex/tooling/src/vortex-fetch-db-ftp` | -| `VORTEX_FETCH_DB_GDRIVE_DB_DIR` | Directory with database dump file. | `./.data` | `.vortex/tooling/src/vortex-fetch-db-gdrive` | -| `VORTEX_FETCH_DB_GDRIVE_DB_FILE` | Database dump file name.

The same name is used for the file in the Google Drive folder. | `db.sql` | `.vortex/tooling/src/vortex-fetch-db-gdrive` | -| `VORTEX_FETCH_DB_GDRIVE_FOLDER_ID` | Google Drive folder ID with the database dump file.

The ID is the last segment of the folder URL. The folder must be in a shared drive and shared with the service account. | `UNDEFINED` | `.env`, `.vortex/tooling/src/vortex-fetch-db-gdrive` | -| `VORTEX_FETCH_DB_GDRIVE_SERVICE_ACCOUNT_KEY` | Google Cloud service account key: the JSON content or a path to the JSON key file. | `${VORTEX_GDRIVE_SERVICE_ACCOUNT_KEY}` | `.vortex/tooling/src/vortex-fetch-db-gdrive` | | `VORTEX_FETCH_DB_LAGOON_DB_DIR` | Directory where DB dumps are stored on the host. | `./.data` | `.vortex/tooling/src/vortex-fetch-db-lagoon` | | `VORTEX_FETCH_DB_LAGOON_DB_FILE` | Database dump file name on the host. | `db.sql` | `.vortex/tooling/src/vortex-fetch-db-lagoon` | | `VORTEX_FETCH_DB_LAGOON_PROJECT` | Lagoon project name. | `${LAGOON_PROJECT}` | `.vortex/tooling/src/vortex-fetch-db-lagoon` | @@ -397,7 +392,6 @@ The list below is automatically generated with [Shellvar](https://github.com/ale | `VORTEX_FETCH_FILES_SSH_FINGERPRINT` | SSH key fingerprint used to connect to a remote. | `UNDEFINED` | `.vortex/tooling/src/vortex-fetch-files-acquia`, `.vortex/tooling/src/vortex-fetch-files-lagoon` | | `VORTEX_FETCH_FILES_URL` | Public files archive sourced from a URL.

A tar archive, optionally compressed, with the content of the public files directory at its root. HTTP Basic Authentication credentials should be embedded into the value. | `UNDEFINED` | `.env`, `.vortex/tooling/src/vortex-fetch-files-url` | | `VORTEX_FRONTEND_BUILD_SKIP` | Skip building of the front-end assets inside the container image.

Set to `1` to build theme assets on the host or as part of deployment instead of during the container image build. | `UNDEFINED` | `.env` | -| `VORTEX_GDRIVE_SERVICE_ACCOUNT_KEY` | Path to the Google Cloud service account JSON key file. Keep the file outside the project directory. | `UNDEFINED` | `.env.local.example` | | `VORTEX_IMPORT_DB_FILE` | Database dump file name. | `db.sql` | `.vortex/tooling/src/vortex-import-db-file` | | `VORTEX_IMPORT_DB_FILE_DIR` | Directory with database dump file. | `./.data` | `.vortex/tooling/src/vortex-import-db-file` | | `VORTEX_INSTALLER_INTERACTIVE` | Run installer in interactive mode. | `0` | `.vortex/tooling/src/vortex-update` | @@ -542,10 +536,14 @@ The list below is automatically generated with [Shellvar](https://github.com/ale | `VORTEX_PUSH_CONTAINER_REGISTRY_MAP` | Comma-separated map of container services and images to push in format "service1=org/image1,service2=org/image2". | `UNDEFINED` | `.vortex/tooling/src/vortex-push-container-registry`, `.vortex/tooling/src/vortex-push-db-image` | | `VORTEX_PUSH_CONTAINER_REGISTRY_PASS` | The password to login into the container registry. | `${VORTEX_CONTAINER_REGISTRY_PASS}` | `.vortex/tooling/src/vortex-push-container-registry` | | `VORTEX_PUSH_CONTAINER_REGISTRY_USER` | The username to login into the container registry. | `${VORTEX_CONTAINER_REGISTRY_USER}` | `.vortex/tooling/src/vortex-push-container-registry` | -| `VORTEX_PUSH_DB_GDRIVE_DB_DIR` | Directory with database dump file. | `./.data` | `.vortex/tooling/src/vortex-push-db-gdrive` | -| `VORTEX_PUSH_DB_GDRIVE_DB_FILE` | Database dump file name.

Defaults to the name the database is fetched and refreshed with. The same name is used for the file in the Google Drive folder. | `db.sql` | `.vortex/tooling/src/vortex-push-db-gdrive` | -| `VORTEX_PUSH_DB_GDRIVE_FOLDER_ID` | Google Drive folder ID to push the database dump file to.

Defaults to the folder the database is fetched from. | `${VORTEX_FETCH_DB_GDRIVE_FOLDER_ID}` | `.vortex/tooling/src/vortex-push-db-gdrive` | -| `VORTEX_PUSH_DB_GDRIVE_SERVICE_ACCOUNT_KEY` | Google Cloud service account key: the JSON content or a path to the JSON key file. | `${VORTEX_GDRIVE_SERVICE_ACCOUNT_KEY}` | `.vortex/tooling/src/vortex-push-db-gdrive` | +| `VORTEX_PUSH_DB_FILE_PREFIX` | Prefix for the names of the pushed database dump files.

Use a different prefix in each place that pushes to the same storage, such as 'gha-' and 'circleci-', to tell their dumps apart. A fetch reads the prefixed name only when its remote file name includes the prefix. | `UNDEFINED` | `.vortex/tooling/src/vortex-push-db-ftp`, `.vortex/tooling/src/vortex-push-db-s3` | +| `VORTEX_PUSH_DB_FTP_DB_DIR` | Directory with database dump file. | `./.data` | `.vortex/tooling/src/vortex-push-db-ftp` | +| `VORTEX_PUSH_DB_FTP_DB_FILE` | Database dump file name.

Defaults to the name the database is fetched and refreshed with. | `db.sql` | `.vortex/tooling/src/vortex-push-db-ftp` | +| `VORTEX_PUSH_DB_FTP_FILE` | The remote file name, including any directories. | `UNDEFINED` | `.vortex/tooling/src/vortex-push-db-ftp` | +| `VORTEX_PUSH_DB_FTP_HOST` | The FTP host. | `UNDEFINED` | `.vortex/tooling/src/vortex-push-db-ftp` | +| `VORTEX_PUSH_DB_FTP_PASS` | The FTP password. | `UNDEFINED` | `.vortex/tooling/src/vortex-push-db-ftp` | +| `VORTEX_PUSH_DB_FTP_PORT` | The FTP port. | `21` | `.vortex/tooling/src/vortex-push-db-ftp` | +| `VORTEX_PUSH_DB_FTP_USER` | The FTP user. | `UNDEFINED` | `.vortex/tooling/src/vortex-push-db-ftp` | | `VORTEX_PUSH_DB_PROCEED` | Proceed with push.

Set to `1` in the nightly CI job only, so the storage is refreshed once a day and never from a feature branch build. | `UNDEFINED` | `.vortex/tooling/src/vortex-push-db` | | `VORTEX_PUSH_DB_S3_ACCESS_KEY` | AWS access key. | `${S3_ACCESS_KEY}` | `.vortex/tooling/src/vortex-push-db-s3` | | `VORTEX_PUSH_DB_S3_BUCKET` | S3 bucket name. | `${S3_BUCKET}` | `.vortex/tooling/src/vortex-push-db-s3` | @@ -553,10 +551,11 @@ The list below is automatically generated with [Shellvar](https://github.com/ale | `VORTEX_PUSH_DB_S3_DB_FILE` | Database dump file name.

Defaults to the name the database is fetched and refreshed with. | `db.sql` | `.vortex/tooling/src/vortex-push-db-s3` | | `VORTEX_PUSH_DB_S3_PREFIX` | S3 prefix (path within the bucket). | `${S3_PREFIX}` | `.vortex/tooling/src/vortex-push-db-s3` | | `VORTEX_PUSH_DB_S3_REGION` | S3 region. | `${S3_REGION}` | `.vortex/tooling/src/vortex-push-db-s3` | -| `VORTEX_PUSH_DB_S3_REMOTE_FILE` | Remote database dump file name.

Defaults to the database dump file name, which is the name the database is fetched with. | `${VORTEX_PUSH_DB_S3_DB_FILE}` | `.vortex/tooling/src/vortex-push-db-s3` | +| `VORTEX_PUSH_DB_S3_REMOTE_FILE` | Remote database dump file name.

Defaults to the remote file name the database is fetched from, then to the database dump file name. | `${VORTEX_PUSH_DB_S3_DB_FILE}` | `.vortex/tooling/src/vortex-push-db-s3` | | `VORTEX_PUSH_DB_S3_SECRET_KEY` | AWS secret key. | `${S3_SECRET_KEY}` | `.vortex/tooling/src/vortex-push-db-s3` | | `VORTEX_PUSH_DB_S3_STORAGE_CLASS` | S3 storage class. | `STANDARD` | `.vortex/tooling/src/vortex-push-db-s3` | | `VORTEX_REFRESH_DB_PROCEED` | Proceed with the refresh of an indexed database.

The primary database is always refreshed. An indexed database, such as the migration database with VORTEX_DB_INDEX=`2`, is refreshed only when VORTEX_REFRESH_DB2_PROCEED is set to `1`. | `UNDEFINED` | `.vortex/tooling/src/vortex-refresh-db` | +| `VORTEX_REFRESH_DB_SOURCE` | Upstream source of the pushed database dump file. | `UNDEFINED` | `.vortex/tooling/src/vortex-push-db`, `.vortex/tooling/src/vortex-refresh-db` | | `VORTEX_RELEASE_VERSION_SCHEME` | Versioning scheme used for releases.

Can be one of: calver, semver, other @see https://www.vortextemplate.com/docs/releasing | `calver` | `.env` | | `VORTEX_SHOW_LOGIN` | Show one-time login link. | `UNDEFINED` | `.vortex/tooling/src/vortex-info` | | `VORTEX_SSH_DISABLE_STRICT_HOST_KEY_CHECKING` | Disable strict host key checking in SSH. | `0` | `.vortex/tooling/src/vortex-setup-ssh` | diff --git a/.vortex/docs/content/features.mdx b/.vortex/docs/content/features.mdx index da09bc16c2..860fafd525 100644 --- a/.vortex/docs/content/features.mdx +++ b/.vortex/docs/content/features.mdx @@ -169,8 +169,8 @@ import { - _Lint_ the code without provisioning a site - _Fetch the database_ from the hosting provider and cache it for the - day, with a nightly refresh of the database container image or of a - sanitized dump in S3 or Google Drive + day, with a nightly refresh of the database image or of a sanitized + dump on FTP or S3 - _Build_ and test a fully provisioned site, with [Behat](https://docs.behat.org/en/latest/) tests split across parallel containers @@ -245,8 +245,7 @@ import { ### Other hosting providers - Webhook deployments for any provider that accepts a deployment URL - - Database fetch from a URL, FTP, S3, Google Drive or a container - registry + - Database fetch from a URL, FTP, S3 or a container registry :::note @@ -375,8 +374,7 @@ import { [`drevops/vortex-tooling`](https://packagist.org/packages/drevops/vortex-tooling) Composer package, run the same way in every environment - Customization through environment variables - - Database sourced from the hosting provider, a URL, FTP, S3, Google - Drive or a + - Database sourced from the hosting provider, a URL, FTP, S3 or a container image seeded with [Database data images](https://github.com/drevops/database-data-images) diff --git a/.vortex/docs/cspell.json b/.vortex/docs/cspell.json index 1e33bea1dd..681610a5d9 100644 --- a/.vortex/docs/cspell.json +++ b/.vortex/docs/cspell.json @@ -13,12 +13,13 @@ "Buildx", "Diffy", "Dockerfiles", - "GDRIVE", "GHSA", "Incapsula", "MAILHUB", "NRAK", "Olivero", + "RNFR", + "RNTO", "REDIS", "Runsheet", "Runsheets", @@ -54,7 +55,6 @@ "drush", "ergebnis", "fieldsets", - "gdrive", "gherkinlint", "gitleaks", "hadolint", diff --git a/.vortex/installer/src/Prompts/Handlers/DatabaseFetchSource.php b/.vortex/installer/src/Prompts/Handlers/DatabaseFetchSource.php index f8e78d3d35..a1d628e4ad 100644 --- a/.vortex/installer/src/Prompts/Handlers/DatabaseFetchSource.php +++ b/.vortex/installer/src/Prompts/Handlers/DatabaseFetchSource.php @@ -21,8 +21,6 @@ class DatabaseFetchSource extends AbstractHandler { const S3 = 's3'; - const GDRIVE = 'gdrive'; - const NONE = 'none'; /** @@ -57,7 +55,6 @@ public function options(array $responses): ?array { self::LAGOON => 'Lagoon environment', self::CONTAINER_REGISTRY => 'Container registry', self::S3 => 'S3 bucket', - self::GDRIVE => 'Google Drive', self::NONE => 'None', ]; @@ -132,7 +129,6 @@ public function process(): void { self::LAGOON, self::CONTAINER_REGISTRY, self::S3, - self::GDRIVE, ]; foreach ($types as $type) { @@ -152,9 +148,9 @@ public function process(): void { File::removeTokenAsync('DB_IN_IMAGE'); } - // DB_PUSH wraps the nightly push of the sanitized dump, which only a - // dump store source accepts. - if (!in_array($v, [self::S3, self::GDRIVE], TRUE)) { + // DB_PUSH wraps the nightly push of the sanitized dump, which only an + // 'ftp' or 's3' source accepts. + if (!in_array($v, [self::FTP, self::S3], TRUE)) { File::removeTokenAsync('DB_PUSH'); } } diff --git a/.vortex/installer/src/Prompts/Handlers/MigrationFetchSource.php b/.vortex/installer/src/Prompts/Handlers/MigrationFetchSource.php index 636cdc11ce..e0be98f19d 100644 --- a/.vortex/installer/src/Prompts/Handlers/MigrationFetchSource.php +++ b/.vortex/installer/src/Prompts/Handlers/MigrationFetchSource.php @@ -21,8 +21,6 @@ class MigrationFetchSource extends AbstractHandler { const S3 = 's3'; - const GDRIVE = 'gdrive'; - /** * {@inheritdoc} */ @@ -55,7 +53,6 @@ public function options(array $responses): ?array { self::LAGOON => 'Lagoon environment', self::CONTAINER_REGISTRY => 'Container registry', self::S3 => 'S3 bucket', - self::GDRIVE => 'Google Drive', ]; if (isset($responses[HostingProvider::id()])) { @@ -133,7 +130,6 @@ public function process(): void { self::LAGOON, self::CONTAINER_REGISTRY, self::S3, - self::GDRIVE, ]; foreach ($types as $type) { @@ -145,6 +141,12 @@ public function process(): void { File::removeTokenAsync($token); } } + + // MIGRATION_DB_PUSH wraps the nightly push of the migration dump, which + // only an 'ftp' or 's3' source accepts. + if (!in_array($v, [self::FTP, self::S3], TRUE)) { + File::removeTokenAsync('MIGRATION_DB_PUSH'); + } } } diff --git a/.vortex/installer/tests/Functional/Prompts/Handlers/DatabaseFetchSourceHandlerProcessTest.php b/.vortex/installer/tests/Functional/Prompts/Handlers/DatabaseFetchSourceHandlerProcessTest.php index 0b9cd9d62b..9ca89d1241 100644 --- a/.vortex/installer/tests/Functional/Prompts/Handlers/DatabaseFetchSourceHandlerProcessTest.php +++ b/.vortex/installer/tests/Functional/Prompts/Handlers/DatabaseFetchSourceHandlerProcessTest.php @@ -36,9 +36,6 @@ public static function dataProviderHandlerProcess(): \Iterator { yield 'db_fetch_source_s3' => [ static::cw(fn(AbstractHandlerProcessTestCase $test): string => $test->prompts[DatabaseFetchSource::id()] = DatabaseFetchSource::S3), ]; - yield 'db_fetch_source_gdrive' => [ - static::cw(fn(AbstractHandlerProcessTestCase $test): string => $test->prompts[DatabaseFetchSource::id()] = DatabaseFetchSource::GDRIVE), - ]; } } diff --git a/.vortex/installer/tests/Functional/Prompts/Handlers/MigrationFetchSourceHandlerProcessTest.php b/.vortex/installer/tests/Functional/Prompts/Handlers/MigrationFetchSourceHandlerProcessTest.php index e410c67712..9c128f4a8a 100644 --- a/.vortex/installer/tests/Functional/Prompts/Handlers/MigrationFetchSourceHandlerProcessTest.php +++ b/.vortex/installer/tests/Functional/Prompts/Handlers/MigrationFetchSourceHandlerProcessTest.php @@ -27,6 +27,7 @@ public static function dataProviderHandlerProcess(): \Iterator { $test->assertFileNotContainsString(static::$sut . '/.github/workflows/build-test-deploy.yml', 'VORTEX_DB_INDEX=2 ./vendor/bin/vortex-refresh-db'); $test->assertFileNotContainsString(static::$sut . '/.github/workflows/build-test-deploy.yml', 'VORTEX_DB_IMAGE_BASE'); $test->assertFileNotContainsString(static::$sut . '/.github/workflows/build-test-deploy.yml', 'Deploy DB image'); + $test->assertFileNotContainsString(static::$sut . '/.github/workflows/build-test-deploy.yml', 'VORTEX_DB_INDEX=2 ./vendor/bin/vortex-push-db'); }), ]; yield 'migration_fetch_source_ftp' => [ @@ -39,6 +40,7 @@ public static function dataProviderHandlerProcess(): \Iterator { $test->assertFileContainsString(static::$sut . '/.env', 'VORTEX_FETCH_DB2_FTP_HOST'); $test->assertFileNotContainsString(static::$sut . '/.env', 'VORTEX_FETCH_DB2_URL='); $test->assertFileNotContainsString(static::$sut . '/.env', 'VORTEX_FETCH_DB2_S3_BUCKET'); + $test->assertFileContainsString(static::$sut . '/.github/workflows/build-test-deploy.yml', 'VORTEX_DB_INDEX=2 ./vendor/bin/vortex-push-db'); }), ]; yield 'migration_fetch_source_acquia' => [ @@ -75,20 +77,7 @@ public static function dataProviderHandlerProcess(): \Iterator { $test->assertFileContainsString(static::$sut . '/.env', 'VORTEX_FETCH_DB2_S3_BUCKET'); $test->assertFileNotContainsString(static::$sut . '/.env', 'VORTEX_FETCH_DB2_URL='); $test->assertFileNotContainsString(static::$sut . '/.env', 'VORTEX_FETCH_DB2_FTP_HOST'); - }), - ]; - yield 'migration_fetch_source_gdrive' => [ - static::cw(function (AbstractHandlerProcessTestCase $test): void { - $test->prompts[Migration::id()] = TRUE; - $test->prompts[MigrationFetchSource::id()] = MigrationFetchSource::GDRIVE; - }), - static::cw(function (AbstractHandlerProcessTestCase $test): void { - $test->assertFileContainsString(static::$sut . '/.env', 'VORTEX_FETCH_DB2_SOURCE=gdrive'); - $test->assertFileContainsString(static::$sut . '/.env', 'VORTEX_FETCH_DB2_GDRIVE_FOLDER_ID='); - $test->assertFileNotContainsString(static::$sut . '/.env', 'VORTEX_FETCH_DB2_URL='); - $test->assertFileNotContainsString(static::$sut . '/.env', 'VORTEX_FETCH_DB2_S3_BUCKET'); - // The migration database is fetched from the store but never pushed. - $test->assertFileNotContainsString(static::$sut . '/.github/workflows/build-test-deploy.yml', 'vortex-push-db'); + $test->assertFileContainsString(static::$sut . '/.github/workflows/build-test-deploy.yml', 'VORTEX_DB_INDEX=2 ./vendor/bin/vortex-push-db'); }), ]; yield 'migration_fetch_source_container_registry' => [ From bf6df027314704e264ddb5e6db5d8acaa239e7f5 Mon Sep 17 00:00:00 2001 From: Alex Skrypnyk Date: Wed, 11 Nov 2026 14:44:07 +1100 Subject: [PATCH 14/21] Updated snapshots. --- .../.github/workflows/build-test-deploy.yml | 2 ++ .../ciprovider_circleci/.circleci/config.yml | 2 ++ .../.github/workflows/build-test-deploy.yml | 2 +- .../.circleci/config.yml | 2 ++ .../.github/workflows/build-test-deploy.yml | 2 +- .../db_fetch_source_container_registry/.env | 6 ++-- .../.github/workflows/build-test-deploy.yml | 6 ++-- .../handler_process/db_fetch_source_ftp/.env | 21 +++++++++----- .../.github/workflows/build-test-deploy.yml | 25 +++++++++++++++++ .../handler_process/db_fetch_source_s3/.env | 25 +++++++++-------- .../.github/workflows/build-test-deploy.yml | 11 +------- .../.circleci/config.yml | 2 ++ .../.circleci/config.yml | 2 ++ .../.github/workflows/build-test-deploy.yml | 2 +- .../.circleci/config.yml | 2 ++ .../.github/workflows/build-test-deploy.yml | 2 +- .../.github/workflows/build-test-deploy.yml | 2 +- .../.circleci/config.yml | 2 ++ .../.github/workflows/build-test-deploy.yml | 4 +-- .../.circleci/config.yml | 2 ++ .../.github/workflows/build-test-deploy.yml | 4 +-- .../.github/workflows/build-test-deploy.yml | 4 +-- .../.env | 6 ++-- .../.github/workflows/build-test-deploy.yml | 8 +++--- .../migration_fetch_source_ftp/.env | 9 +++++- .../.github/workflows/build-test-deploy.yml | 28 +++++++++++++++++-- .../.github/workflows/build-test-deploy.yml | 4 +-- .../migration_fetch_source_s3/.env | 9 +++++- .../.github/workflows/build-test-deploy.yml | 28 +++++++++++++++++-- .../.github/workflows/build-test-deploy.yml | 4 +-- .../.github/workflows/build-test-deploy.yml | 4 +-- .../.github/workflows/build-test-deploy.yml | 4 +-- .../.github/workflows/build-test-deploy.yml | 4 +-- .../.github/workflows/build-test-deploy.yml | 4 +-- .../.github/workflows/build-test-deploy.yml | 4 +-- .../.github/workflows/build-test-deploy.yml | 4 +-- .../.github/workflows/build-test-deploy.yml | 4 +-- .../.github/workflows/build-test-deploy.yml | 10 ++++--- .../.github/workflows/build-test-deploy.yml | 4 +-- .../.github/workflows/build-test-deploy.yml | 4 +-- .../.github/workflows/build-test-deploy.yml | 4 +-- .../timezone_circleci/.circleci/config.yml | 2 ++ .../.circleci/config.yml | 2 ++ .../.github/workflows/build-test-deploy.yml | 8 +++--- .../.circleci/config.yml | 2 ++ .../.github/workflows/build-test-deploy.yml | 6 ++-- .../.circleci/config.yml | 2 ++ .../.github/workflows/build-test-deploy.yml | 8 +++--- .../.circleci/config.yml | 2 ++ .../.github/workflows/build-test-deploy.yml | 4 +-- .../.circleci/config.yml | 2 ++ .../.circleci/config.yml | 2 ++ .../.github/workflows/build-test-deploy.yml | 4 +-- .../.circleci/config.yml | 2 ++ .../.circleci/config.yml | 2 ++ .../.github/workflows/build-test-deploy.yml | 4 +-- .../.circleci/config.yml | 2 ++ .../.github/workflows/build-test-deploy.yml | 6 ++-- .../.circleci/config.yml | 2 ++ .../.circleci/config.yml | 2 ++ .../.circleci/config.yml | 2 ++ .../.github/workflows/build-test-deploy.yml | 4 +-- .../.circleci/config.yml | 2 ++ .../.circleci/config.yml | 2 ++ .../.circleci/config.yml | 2 ++ .../.github/workflows/build-test-deploy.yml | 4 +-- .../.circleci/config.yml | 2 ++ .../.github/workflows/build-test-deploy.yml | 10 +++---- 68 files changed, 254 insertions(+), 116 deletions(-) create mode 100644 .vortex/installer/tests/Fixtures/handler_process/db_fetch_source_ftp/.github/workflows/build-test-deploy.yml diff --git a/.vortex/installer/tests/Fixtures/handler_process/_baseline/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/_baseline/.github/workflows/build-test-deploy.yml index b7059b0f25..3580a1e345 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/_baseline/.github/workflows/build-test-deploy.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/_baseline/.github/workflows/build-test-deploy.yml @@ -241,6 +241,8 @@ jobs: { echo "VORTEX_CI_DB_CACHE_FALLBACK=no" echo "VORTEX_FRONTEND_BUILD_SKIP=1" + # Refresh the database dumps in the fetch sources that are dump stores. + echo "VORTEX_PUSH_DB_PROCEED=1" } >> "$GITHUB_ENV" - name: Create cache keys files for database caching diff --git a/.vortex/installer/tests/Fixtures/handler_process/ciprovider_circleci/.circleci/config.yml b/.vortex/installer/tests/Fixtures/handler_process/ciprovider_circleci/.circleci/config.yml index f56bdf626c..b59dde79d2 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/ciprovider_circleci/.circleci/config.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/ciprovider_circleci/.circleci/config.yml @@ -307,6 +307,8 @@ jobs: VORTEX_DEPLOY_SSH_FINGERPRINT: *deploy_ssh_fingerprint # Enforce fresh DB build (do not rely on fallback caches). VORTEX_CI_DB_CACHE_FALLBACK: 'no' + # Refresh the database dumps in the fetch sources that are dump stores. + VORTEX_PUSH_DB_PROCEED: 1 # Do not build the Drupal front-end. VORTEX_FRONTEND_BUILD_SKIP: 1 diff --git a/.vortex/installer/tests/Fixtures/handler_process/code_coverage_provider_codecov/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/code_coverage_provider_codecov/.github/workflows/build-test-deploy.yml index 65a0cc3f96..613d58ed05 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/code_coverage_provider_codecov/.github/workflows/build-test-deploy.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/code_coverage_provider_codecov/.github/workflows/build-test-deploy.yml @@ -1,4 +1,4 @@ -@@ -490,6 +490,17 @@ +@@ -492,6 +492,17 @@ hide_and_recreate: true diff --git a/.vortex/installer/tests/Fixtures/handler_process/code_coverage_provider_codecov_circleci/.circleci/config.yml b/.vortex/installer/tests/Fixtures/handler_process/code_coverage_provider_codecov_circleci/.circleci/config.yml index 095f6fc249..ea65c0eaad 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/code_coverage_provider_codecov_circleci/.circleci/config.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/code_coverage_provider_codecov_circleci/.circleci/config.yml @@ -307,6 +307,8 @@ jobs: VORTEX_DEPLOY_SSH_FINGERPRINT: *deploy_ssh_fingerprint # Enforce fresh DB build (do not rely on fallback caches). VORTEX_CI_DB_CACHE_FALLBACK: 'no' + # Refresh the database dumps in the fetch sources that are dump stores. + VORTEX_PUSH_DB_PROCEED: 1 # Do not build the Drupal front-end. VORTEX_FRONTEND_BUILD_SKIP: 1 diff --git a/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_acquia/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_acquia/.github/workflows/build-test-deploy.yml index c4971de6bd..a4f3384e34 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_acquia/.github/workflows/build-test-deploy.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_acquia/.github/workflows/build-test-deploy.yml @@ -1,4 +1,4 @@ -@@ -269,6 +269,9 @@ +@@ -271,6 +271,9 @@ VORTEX_FETCH_DB_SEMAPHORE=/tmp/fetch-db-fresh ./vendor/bin/vortex-fetch-db echo "db_hash=${{ hashFiles('.data') }}" >> "$GITHUB_ENV" timeout-minutes: 30 diff --git a/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_container_registry/.env b/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_container_registry/.env index e6357bf6c3..f98032044d 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_container_registry/.env +++ b/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_container_registry/.env @@ -14,9 +14,9 @@ +VORTEX_DB_IMAGE=the_empire/star_wars:latest +# Source of the database dump file the nightly CI job imports into a clean -+# base image to build the database container image: url, ftp, acquia, lagoon, -+# s3 or gdrive. The fetch script of that source reads its own variables, such -+# as VORTEX_FETCH_DB_URL for url. Add them, and keep secrets in CI. ++# base image to build the database container image: url, ftp, acquia, lagoon ++# or s3. The fetch script of that source reads its own variables, such as ++# VORTEX_FETCH_DB_URL for url. Add them, and keep secrets in CI. +# VORTEX_REFRESH_DB_SOURCE=url + # Environment to fetch the database from. diff --git a/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_container_registry/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_container_registry/.github/workflows/build-test-deploy.yml index 5a2acf73b9..3086561937 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_container_registry/.github/workflows/build-test-deploy.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_container_registry/.github/workflows/build-test-deploy.yml @@ -6,10 +6,10 @@ + echo "VORTEX_DB_IMAGE_BASE=drevops/db-data-mysql-8.4:__VERSION__" + # Deploy container image. + echo "VORTEX_EXPORT_DB_CONTAINER_REGISTRY_PUSH_PROCEED=1" + # Refresh the database dumps in the fetch sources that are dump stores. + echo "VORTEX_PUSH_DB_PROCEED=1" } >> "$GITHUB_ENV" - - - name: Create cache keys files for database caching -@@ -274,6 +278,15 @@ +@@ -276,6 +280,15 @@ run: | if [ ! -f /tmp/fetch-db-fresh ]; then echo "==> Database fetch semaphore file is missing. DB export will not proceed."; exit 0; fi ./vendor/bin/vortex-refresh-db diff --git a/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_ftp/.env b/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_ftp/.env index 74fe89268f..a6c57df4f1 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_ftp/.env +++ b/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_ftp/.env @@ -1,4 +1,4 @@ -@@ -180,13 +180,19 @@ +@@ -180,12 +180,27 @@ VORTEX_DB_FILE=db.sql # Database fetch source. @@ -6,11 +6,8 @@ +VORTEX_FETCH_DB_SOURCE=ftp -# Database dump file sourced from a URL. --# --# HTTP Basic Authentication credentials should be embedded into the value. --VORTEX_FETCH_DB_URL= +# Database dump file sourced from FTP. - ++ +# Database dump FTP host. +VORTEX_FETCH_DB_FTP_HOST= + @@ -20,10 +17,20 @@ +# Database dump FTP file name. +VORTEX_FETCH_DB_FTP_FILE=db.sql + ++# Upstream source of the database dump file, such as url, acquia or lagoon. + # +-# HTTP Basic Authentication credentials should be embedded into the value. +-VORTEX_FETCH_DB_URL= ++# Uncomment to make the database fetch source a dump store: the nightly CI ++# job fetches the dump from this source, sanitizes it and pushes it to the ++# fetch source, keeping the dump it replaces as the previous one. The fetch ++# script of this source reads its own variables, such as VORTEX_FETCH_DB_URL ++# for url. Add them, and keep secrets in CI. ++# VORTEX_REFRESH_DB_SOURCE=url + # Environment to fetch the database from. # - # Applies to hosting environments. -@@ -254,17 +260,3 @@ +@@ -254,17 +269,3 @@ # with optional names in the format "email|name". # Example: "to1@example.com|Jane Doe, to2@example.com|John Doe" VORTEX_NOTIFY_EMAIL_RECIPIENTS="webmaster@star-wars.com|Webmaster" diff --git a/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_ftp/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_ftp/.github/workflows/build-test-deploy.yml new file mode 100644 index 0000000000..99ebadce06 --- /dev/null +++ b/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_ftp/.github/workflows/build-test-deploy.yml @@ -0,0 +1,25 @@ +@@ -191,6 +191,8 @@ + VORTEX_CI_DB_CACHE_FALLBACK: "yes" + # Which branch to use as a source of DB caches. + VORTEX_CI_DB_CACHE_BRANCH: "develop" ++ VORTEX_FETCH_DB_FTP_USER: ${{ secrets.VORTEX_FETCH_DB_FTP_USER }} ++ VORTEX_FETCH_DB_FTP_PASS: ${{ secrets.VORTEX_FETCH_DB_FTP_PASS }} + + steps: + # Frees disk space by removing preinstalled toolchains unused by this project. Steps run in a container, so the host is reached via the Docker socket. +@@ -276,6 +278,15 @@ + run: | + if [ ! -f /tmp/fetch-db-fresh ]; then echo "==> Database fetch semaphore file is missing. DB export will not proceed."; exit 0; fi + ./vendor/bin/vortex-refresh-db ++ timeout-minutes: 30 ++ ++ # Push the freshly-exported sanitized database dump to the database fetch ++ # source. No-op unless a fresh DB was fetched (the fetch semaphore) and the ++ # push was requested (VORTEX_PUSH_DB_PROCEED, in the script). ++ - name: Push DB ++ run: | ++ if [ ! -f /tmp/fetch-db-fresh ]; then echo "==> No freshly fetched database. DB push will not proceed."; exit 0; fi ++ ./vendor/bin/vortex-push-db + timeout-minutes: 30 + + # Save cache per default branch and the timestamp. diff --git a/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_s3/.env b/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_s3/.env index b26eb7cb6a..63867d4b6e 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_s3/.env +++ b/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_s3/.env @@ -1,4 +1,4 @@ -@@ -180,13 +180,22 @@ +@@ -180,12 +180,24 @@ VORTEX_DB_FILE=db.sql # Database fetch source. @@ -6,27 +6,28 @@ +VORTEX_FETCH_DB_SOURCE=s3 -# Database dump file sourced from a URL. --# --# HTTP Basic Authentication credentials should be embedded into the value. --VORTEX_FETCH_DB_URL= +# Database dump file sourced from S3. - ++ +# AWS S3 bucket name for database fetch. +VORTEX_FETCH_DB_S3_BUCKET= + +# AWS S3 region. +VORTEX_FETCH_DB_S3_REGION=ap-southeast-2 + -+# Source of the database dump file the nightly CI job sanitizes and pushes to -+# the database fetch source: url, ftp, acquia or lagoon. The fetch script of -+# that source reads its own variables, such as VORTEX_FETCH_DB_URL for url. -+# Add them, and keep secrets in CI. ++# Upstream source of the database dump file, such as url, acquia or lagoon. + # +-# HTTP Basic Authentication credentials should be embedded into the value. +-VORTEX_FETCH_DB_URL= ++# Uncomment to make the database fetch source a dump store: the nightly CI ++# job fetches the dump from this source, sanitizes it and pushes it to the ++# fetch source, keeping the dump it replaces as the previous one. The fetch ++# script of this source reads its own variables, such as VORTEX_FETCH_DB_URL ++# for url. Add them, and keep secrets in CI. +# VORTEX_REFRESH_DB_SOURCE=url -+ + # Environment to fetch the database from. # - # Applies to hosting environments. -@@ -254,17 +263,3 @@ +@@ -254,17 +266,3 @@ # with optional names in the format "email|name". # Example: "to1@example.com|Jane Doe, to2@example.com|John Doe" VORTEX_NOTIFY_EMAIL_RECIPIENTS="webmaster@star-wars.com|Webmaster" diff --git a/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_s3/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_s3/.github/workflows/build-test-deploy.yml index d8ec3cb540..b95ee1f3c8 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_s3/.github/workflows/build-test-deploy.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_s3/.github/workflows/build-test-deploy.yml @@ -7,16 +7,7 @@ steps: # Frees disk space by removing preinstalled toolchains unused by this project. Steps run in a container, so the host is reached via the Docker socket. -@@ -241,6 +243,8 @@ - { - echo "VORTEX_CI_DB_CACHE_FALLBACK=no" - echo "VORTEX_FRONTEND_BUILD_SKIP=1" -+ # Refresh the database dump in the database fetch source. -+ echo "VORTEX_PUSH_DB_PROCEED=1" - } >> "$GITHUB_ENV" - - - name: Create cache keys files for database caching -@@ -274,6 +278,15 @@ +@@ -276,6 +278,15 @@ run: | if [ ! -f /tmp/fetch-db-fresh ]; then echo "==> Database fetch semaphore file is missing. DB export will not proceed."; exit 0; fi ./vendor/bin/vortex-refresh-db diff --git a/.vortex/installer/tests/Fixtures/handler_process/deploy_types_all_circleci/.circleci/config.yml b/.vortex/installer/tests/Fixtures/handler_process/deploy_types_all_circleci/.circleci/config.yml index f56bdf626c..b59dde79d2 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/deploy_types_all_circleci/.circleci/config.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/deploy_types_all_circleci/.circleci/config.yml @@ -307,6 +307,8 @@ jobs: VORTEX_DEPLOY_SSH_FINGERPRINT: *deploy_ssh_fingerprint # Enforce fresh DB build (do not rely on fallback caches). VORTEX_CI_DB_CACHE_FALLBACK: 'no' + # Refresh the database dumps in the fetch sources that are dump stores. + VORTEX_PUSH_DB_PROCEED: 1 # Do not build the Drupal front-end. VORTEX_FRONTEND_BUILD_SKIP: 1 diff --git a/.vortex/installer/tests/Fixtures/handler_process/deploy_types_none_circleci/.circleci/config.yml b/.vortex/installer/tests/Fixtures/handler_process/deploy_types_none_circleci/.circleci/config.yml index 70ccf44837..6b2526e489 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/deploy_types_none_circleci/.circleci/config.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/deploy_types_none_circleci/.circleci/config.yml @@ -307,6 +307,8 @@ jobs: VORTEX_DEPLOY_SSH_FINGERPRINT: *deploy_ssh_fingerprint # Enforce fresh DB build (do not rely on fallback caches). VORTEX_CI_DB_CACHE_FALLBACK: 'no' + # Refresh the database dumps in the fetch sources that are dump stores. + VORTEX_PUSH_DB_PROCEED: 1 # Do not build the Drupal front-end. VORTEX_FRONTEND_BUILD_SKIP: 1 diff --git a/.vortex/installer/tests/Fixtures/handler_process/deploy_types_none_gha/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/deploy_types_none_gha/.github/workflows/build-test-deploy.yml index 24407e1e7d..d0b2146bf0 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/deploy_types_none_gha/.github/workflows/build-test-deploy.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/deploy_types_none_gha/.github/workflows/build-test-deploy.yml @@ -1,4 +1,4 @@ -@@ -573,104 +573,3 @@ +@@ -575,104 +575,3 @@ timeout-minutes: 120 # Cancel the action after 120 minutes, regardless of whether a connection has been established. with: detached: true diff --git a/.vortex/installer/tests/Fixtures/handler_process/deps_updates_provider_ci_circleci/.circleci/config.yml b/.vortex/installer/tests/Fixtures/handler_process/deps_updates_provider_ci_circleci/.circleci/config.yml index f56bdf626c..b59dde79d2 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/deps_updates_provider_ci_circleci/.circleci/config.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/deps_updates_provider_ci_circleci/.circleci/config.yml @@ -307,6 +307,8 @@ jobs: VORTEX_DEPLOY_SSH_FINGERPRINT: *deploy_ssh_fingerprint # Enforce fresh DB build (do not rely on fallback caches). VORTEX_CI_DB_CACHE_FALLBACK: 'no' + # Refresh the database dumps in the fetch sources that are dump stores. + VORTEX_PUSH_DB_PROCEED: 1 # Do not build the Drupal front-end. VORTEX_FRONTEND_BUILD_SKIP: 1 diff --git a/.vortex/installer/tests/Fixtures/handler_process/hosting_acquia/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/hosting_acquia/.github/workflows/build-test-deploy.yml index c4971de6bd..a4f3384e34 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/hosting_acquia/.github/workflows/build-test-deploy.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/hosting_acquia/.github/workflows/build-test-deploy.yml @@ -1,4 +1,4 @@ -@@ -269,6 +269,9 @@ +@@ -271,6 +271,9 @@ VORTEX_FETCH_DB_SEMAPHORE=/tmp/fetch-db-fresh ./vendor/bin/vortex-fetch-db echo "db_hash=${{ hashFiles('.data') }}" >> "$GITHUB_ENV" timeout-minutes: 30 diff --git a/.vortex/installer/tests/Fixtures/handler_process/hosting_project_name___acquia/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/hosting_project_name___acquia/.github/workflows/build-test-deploy.yml index c4971de6bd..a4f3384e34 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/hosting_project_name___acquia/.github/workflows/build-test-deploy.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/hosting_project_name___acquia/.github/workflows/build-test-deploy.yml @@ -1,4 +1,4 @@ -@@ -269,6 +269,9 @@ +@@ -271,6 +271,9 @@ VORTEX_FETCH_DB_SEMAPHORE=/tmp/fetch-db-fresh ./vendor/bin/vortex-fetch-db echo "db_hash=${{ hashFiles('.data') }}" >> "$GITHUB_ENV" timeout-minutes: 30 diff --git a/.vortex/installer/tests/Fixtures/handler_process/migration_disabled_circleci/.circleci/config.yml b/.vortex/installer/tests/Fixtures/handler_process/migration_disabled_circleci/.circleci/config.yml index f56bdf626c..b59dde79d2 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/migration_disabled_circleci/.circleci/config.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/migration_disabled_circleci/.circleci/config.yml @@ -307,6 +307,8 @@ jobs: VORTEX_DEPLOY_SSH_FINGERPRINT: *deploy_ssh_fingerprint # Enforce fresh DB build (do not rely on fallback caches). VORTEX_CI_DB_CACHE_FALLBACK: 'no' + # Refresh the database dumps in the fetch sources that are dump stores. + VORTEX_PUSH_DB_PROCEED: 1 # Do not build the Drupal front-end. VORTEX_FRONTEND_BUILD_SKIP: 1 diff --git a/.vortex/installer/tests/Fixtures/handler_process/migration_enabled/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/migration_enabled/.github/workflows/build-test-deploy.yml index 4292b4f549..460b6bc35f 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/migration_enabled/.github/workflows/build-test-deploy.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/migration_enabled/.github/workflows/build-test-deploy.yml @@ -1,4 +1,4 @@ -@@ -270,6 +270,9 @@ +@@ -272,6 +272,9 @@ echo "db_hash=${{ hashFiles('.data') }}" >> "$GITHUB_ENV" timeout-minutes: 30 @@ -8,7 +8,7 @@ - name: Export DB run: | if [ ! -f /tmp/fetch-db-fresh ]; then echo "==> Database fetch semaphore file is missing. DB export will not proceed."; exit 0; fi -@@ -431,6 +434,11 @@ +@@ -433,6 +436,11 @@ docker compose exec cli mkdir -p .data docker compose --progress quiet cp -L .data/db.sql cli:/app/.data/db.sql rm -f .data/db.sql diff --git a/.vortex/installer/tests/Fixtures/handler_process/migration_enabled_circleci/.circleci/config.yml b/.vortex/installer/tests/Fixtures/handler_process/migration_enabled_circleci/.circleci/config.yml index 69a8a20830..657e41c70f 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/migration_enabled_circleci/.circleci/config.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/migration_enabled_circleci/.circleci/config.yml @@ -312,6 +312,8 @@ jobs: VORTEX_DEPLOY_SSH_FINGERPRINT: *deploy_ssh_fingerprint # Enforce fresh DB build (do not rely on fallback caches). VORTEX_CI_DB_CACHE_FALLBACK: 'no' + # Refresh the database dumps in the fetch sources that are dump stores. + VORTEX_PUSH_DB_PROCEED: 1 # Do not build the Drupal front-end. VORTEX_FRONTEND_BUILD_SKIP: 1 diff --git a/.vortex/installer/tests/Fixtures/handler_process/migration_enabled_lagoon/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/migration_enabled_lagoon/.github/workflows/build-test-deploy.yml index 4292b4f549..460b6bc35f 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/migration_enabled_lagoon/.github/workflows/build-test-deploy.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/migration_enabled_lagoon/.github/workflows/build-test-deploy.yml @@ -1,4 +1,4 @@ -@@ -270,6 +270,9 @@ +@@ -272,6 +272,9 @@ echo "db_hash=${{ hashFiles('.data') }}" >> "$GITHUB_ENV" timeout-minutes: 30 @@ -8,7 +8,7 @@ - name: Export DB run: | if [ ! -f /tmp/fetch-db-fresh ]; then echo "==> Database fetch semaphore file is missing. DB export will not proceed."; exit 0; fi -@@ -431,6 +434,11 @@ +@@ -433,6 +436,11 @@ docker compose exec cli mkdir -p .data docker compose --progress quiet cp -L .data/db.sql cli:/app/.data/db.sql rm -f .data/db.sql diff --git a/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_acquia/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_acquia/.github/workflows/build-test-deploy.yml index 4292b4f549..460b6bc35f 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_acquia/.github/workflows/build-test-deploy.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_acquia/.github/workflows/build-test-deploy.yml @@ -1,4 +1,4 @@ -@@ -270,6 +270,9 @@ +@@ -272,6 +272,9 @@ echo "db_hash=${{ hashFiles('.data') }}" >> "$GITHUB_ENV" timeout-minutes: 30 @@ -8,7 +8,7 @@ - name: Export DB run: | if [ ! -f /tmp/fetch-db-fresh ]; then echo "==> Database fetch semaphore file is missing. DB export will not proceed."; exit 0; fi -@@ -431,6 +434,11 @@ +@@ -433,6 +436,11 @@ docker compose exec cli mkdir -p .data docker compose --progress quiet cp -L .data/db.sql cli:/app/.data/db.sql rm -f .data/db.sql diff --git a/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_container_registry/.env b/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_container_registry/.env index fc0db2960c..cdfd843852 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_container_registry/.env +++ b/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_container_registry/.env @@ -24,9 +24,9 @@ + +# Source of the second database dump file the nightly CI job imports into a +# clean base image to rebuild the migration database container image: url, -+# ftp, acquia, lagoon, s3 or gdrive. The fetch script of that source reads its -+# own variables, such as VORTEX_FETCH_DB2_URL for url. Add them, and keep -+# secrets in CI. ++# ftp, acquia, lagoon or s3. The fetch script of that source reads its own ++# variables, such as VORTEX_FETCH_DB2_URL for url. Add them, and keep secrets ++# in CI. +# VORTEX_REFRESH_DB2_SOURCE=url + +# Environment to fetch the second database from. diff --git a/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_container_registry/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_container_registry/.github/workflows/build-test-deploy.yml index 975da88adb..e819850e04 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_container_registry/.github/workflows/build-test-deploy.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_container_registry/.github/workflows/build-test-deploy.yml @@ -6,10 +6,10 @@ + echo "VORTEX_DB_IMAGE_BASE=drevops/db-data-mysql-8.4:__VERSION__" + # Deploy container image. + echo "VORTEX_EXPORT_DB_CONTAINER_REGISTRY_PUSH_PROCEED=1" + # Refresh the database dumps in the fetch sources that are dump stores. + echo "VORTEX_PUSH_DB_PROCEED=1" } >> "$GITHUB_ENV" - - - name: Create cache keys files for database caching -@@ -270,9 +274,13 @@ +@@ -272,9 +276,13 @@ echo "db_hash=${{ hashFiles('.data') }}" >> "$GITHUB_ENV" timeout-minutes: 30 @@ -23,7 +23,7 @@ ./vendor/bin/vortex-refresh-db timeout-minutes: 30 -@@ -431,6 +439,11 @@ +@@ -433,6 +441,11 @@ docker compose exec cli mkdir -p .data docker compose --progress quiet cp -L .data/db.sql cli:/app/.data/db.sql rm -f .data/db.sql diff --git a/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_ftp/.env b/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_ftp/.env index 7f2e72048c..ae7d4bb2fe 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_ftp/.env +++ b/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_ftp/.env @@ -1,4 +1,4 @@ -@@ -195,6 +195,35 @@ +@@ -195,6 +195,42 @@ VORTEX_FETCH_DB_ENVIRONMENT=prod ################################################################################ @@ -25,6 +25,13 @@ +# Second database dump FTP file name. +VORTEX_FETCH_DB2_FTP_FILE=db2.sql + ++# Upstream source of the second database dump file, such as url or lagoon. ++# ++# Uncomment to make the second database fetch source a dump store: the ++# nightly CI job fetches the dump from this source and pushes it, as fetched, ++# to the fetch source, next to the primary dump. ++# VORTEX_REFRESH_DB2_SOURCE=url ++ +# Environment to fetch the second database from. +# +# Applies to hosting environments. diff --git a/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_ftp/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_ftp/.github/workflows/build-test-deploy.yml index 4292b4f549..bf29fc7cbc 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_ftp/.github/workflows/build-test-deploy.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_ftp/.github/workflows/build-test-deploy.yml @@ -1,4 +1,13 @@ -@@ -270,6 +270,9 @@ +@@ -191,6 +191,8 @@ + VORTEX_CI_DB_CACHE_FALLBACK: "yes" + # Which branch to use as a source of DB caches. + VORTEX_CI_DB_CACHE_BRANCH: "develop" ++ VORTEX_FETCH_DB2_FTP_USER: ${{ secrets.VORTEX_FETCH_DB2_FTP_USER }} ++ VORTEX_FETCH_DB2_FTP_PASS: ${{ secrets.VORTEX_FETCH_DB2_FTP_PASS }} + + steps: + # Frees disk space by removing preinstalled toolchains unused by this project. Steps run in a container, so the host is reached via the Docker socket. +@@ -272,6 +274,9 @@ echo "db_hash=${{ hashFiles('.data') }}" >> "$GITHUB_ENV" timeout-minutes: 30 @@ -8,7 +17,22 @@ - name: Export DB run: | if [ ! -f /tmp/fetch-db-fresh ]; then echo "==> Database fetch semaphore file is missing. DB export will not proceed."; exit 0; fi -@@ -431,6 +434,11 @@ +@@ -278,6 +283,14 @@ + ./vendor/bin/vortex-refresh-db + timeout-minutes: 30 + ++ # Push the freshly-fetched migration database dump to its fetch source, ++ # next to the primary dump. No-op unless the push was requested. ++ - name: Push migration DB ++ run: | ++ if [ ! -f /tmp/fetch-db-fresh ]; then echo "==> No freshly fetched database. DB push will not proceed."; exit 0; fi ++ VORTEX_DB_INDEX=2 ./vendor/bin/vortex-push-db ++ timeout-minutes: 30 ++ + # Save cache per default branch and the timestamp. + # The cache will not be saved if it already exists. + # Note that the cache fallback flag is enabled for this case in order +@@ -433,6 +446,11 @@ docker compose exec cli mkdir -p .data docker compose --progress quiet cp -L .data/db.sql cli:/app/.data/db.sql rm -f .data/db.sql diff --git a/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_lagoon/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_lagoon/.github/workflows/build-test-deploy.yml index 4292b4f549..460b6bc35f 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_lagoon/.github/workflows/build-test-deploy.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_lagoon/.github/workflows/build-test-deploy.yml @@ -1,4 +1,4 @@ -@@ -270,6 +270,9 @@ +@@ -272,6 +272,9 @@ echo "db_hash=${{ hashFiles('.data') }}" >> "$GITHUB_ENV" timeout-minutes: 30 @@ -8,7 +8,7 @@ - name: Export DB run: | if [ ! -f /tmp/fetch-db-fresh ]; then echo "==> Database fetch semaphore file is missing. DB export will not proceed."; exit 0; fi -@@ -431,6 +434,11 @@ +@@ -433,6 +436,11 @@ docker compose exec cli mkdir -p .data docker compose --progress quiet cp -L .data/db.sql cli:/app/.data/db.sql rm -f .data/db.sql diff --git a/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_s3/.env b/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_s3/.env index cddcb5610a..b1aec4e0f5 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_s3/.env +++ b/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_s3/.env @@ -1,4 +1,4 @@ -@@ -195,6 +195,32 @@ +@@ -195,6 +195,39 @@ VORTEX_FETCH_DB_ENVIRONMENT=prod ################################################################################ @@ -22,6 +22,13 @@ +# AWS S3 region. +VORTEX_FETCH_DB2_S3_REGION=ap-southeast-2 + ++# Upstream source of the second database dump file, such as url or lagoon. ++# ++# Uncomment to make the second database fetch source a dump store: the ++# nightly CI job fetches the dump from this source and pushes it, as fetched, ++# to the fetch source, next to the primary dump. ++# VORTEX_REFRESH_DB2_SOURCE=url ++ +# Environment to fetch the second database from. +# +# Applies to hosting environments. diff --git a/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_s3/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_s3/.github/workflows/build-test-deploy.yml index 4292b4f549..283b4f1f72 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_s3/.github/workflows/build-test-deploy.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_s3/.github/workflows/build-test-deploy.yml @@ -1,4 +1,13 @@ -@@ -270,6 +270,9 @@ +@@ -191,6 +191,8 @@ + VORTEX_CI_DB_CACHE_FALLBACK: "yes" + # Which branch to use as a source of DB caches. + VORTEX_CI_DB_CACHE_BRANCH: "develop" ++ VORTEX_FETCH_DB2_S3_ACCESS_KEY: ${{ secrets.VORTEX_FETCH_DB2_S3_ACCESS_KEY }} ++ VORTEX_FETCH_DB2_S3_SECRET_KEY: ${{ secrets.VORTEX_FETCH_DB2_S3_SECRET_KEY }} + + steps: + # Frees disk space by removing preinstalled toolchains unused by this project. Steps run in a container, so the host is reached via the Docker socket. +@@ -272,6 +274,9 @@ echo "db_hash=${{ hashFiles('.data') }}" >> "$GITHUB_ENV" timeout-minutes: 30 @@ -8,7 +17,22 @@ - name: Export DB run: | if [ ! -f /tmp/fetch-db-fresh ]; then echo "==> Database fetch semaphore file is missing. DB export will not proceed."; exit 0; fi -@@ -431,6 +434,11 @@ +@@ -278,6 +283,14 @@ + ./vendor/bin/vortex-refresh-db + timeout-minutes: 30 + ++ # Push the freshly-fetched migration database dump to its fetch source, ++ # next to the primary dump. No-op unless the push was requested. ++ - name: Push migration DB ++ run: | ++ if [ ! -f /tmp/fetch-db-fresh ]; then echo "==> No freshly fetched database. DB push will not proceed."; exit 0; fi ++ VORTEX_DB_INDEX=2 ./vendor/bin/vortex-push-db ++ timeout-minutes: 30 ++ + # Save cache per default branch and the timestamp. + # The cache will not be saved if it already exists. + # Note that the cache fallback flag is enabled for this case in order +@@ -433,6 +446,11 @@ docker compose exec cli mkdir -p .data docker compose --progress quiet cp -L .data/db.sql cli:/app/.data/db.sql rm -f .data/db.sql diff --git a/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_url/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_url/.github/workflows/build-test-deploy.yml index 4292b4f549..460b6bc35f 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_url/.github/workflows/build-test-deploy.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_url/.github/workflows/build-test-deploy.yml @@ -1,4 +1,4 @@ -@@ -270,6 +270,9 @@ +@@ -272,6 +272,9 @@ echo "db_hash=${{ hashFiles('.data') }}" >> "$GITHUB_ENV" timeout-minutes: 30 @@ -8,7 +8,7 @@ - name: Export DB run: | if [ ! -f /tmp/fetch-db-fresh ]; then echo "==> Database fetch semaphore file is missing. DB export will not proceed."; exit 0; fi -@@ -431,6 +434,11 @@ +@@ -433,6 +436,11 @@ docker compose exec cli mkdir -p .data docker compose --progress quiet cp -L .data/db.sql cli:/app/.data/db.sql rm -f .data/db.sql diff --git a/.vortex/installer/tests/Fixtures/handler_process/modules_no_dev_modules_storybook/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/modules_no_dev_modules_storybook/.github/workflows/build-test-deploy.yml index c0b7f62c2c..ab7610ac1b 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/modules_no_dev_modules_storybook/.github/workflows/build-test-deploy.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/modules_no_dev_modules_storybook/.github/workflows/build-test-deploy.yml @@ -1,4 +1,4 @@ -@@ -312,7 +312,6 @@ +@@ -314,7 +314,6 @@ VORTEX_CI_RUNNER_TOTAL: ${{ strategy.job-total }} VORTEX_CI_IS_JEST_RUNNER: ${{ matrix.instance == 0 || strategy.job-total == 1 }} VORTEX_CI_IS_PHPUNIT_RUNNER: ${{ matrix.instance == 0 || strategy.job-total == 1 }} @@ -6,7 +6,7 @@ # Runs on every instance, using the `p` profile. VORTEX_CI_IS_BEHAT_RUNNER: true # Subtracted from the instance index to derive the profile number. Raise -@@ -499,18 +498,6 @@ +@@ -501,18 +500,6 @@ fi env: VORTEX_CI_CODE_COVERAGE_THRESHOLD: ${{ vars.VORTEX_CI_CODE_COVERAGE_THRESHOLD || '90' }} diff --git a/.vortex/installer/tests/Fixtures/handler_process/modules_no_devel_sdc_devel/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/modules_no_devel_sdc_devel/.github/workflows/build-test-deploy.yml index c0b7f62c2c..ab7610ac1b 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/modules_no_devel_sdc_devel/.github/workflows/build-test-deploy.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/modules_no_devel_sdc_devel/.github/workflows/build-test-deploy.yml @@ -1,4 +1,4 @@ -@@ -312,7 +312,6 @@ +@@ -314,7 +314,6 @@ VORTEX_CI_RUNNER_TOTAL: ${{ strategy.job-total }} VORTEX_CI_IS_JEST_RUNNER: ${{ matrix.instance == 0 || strategy.job-total == 1 }} VORTEX_CI_IS_PHPUNIT_RUNNER: ${{ matrix.instance == 0 || strategy.job-total == 1 }} @@ -6,7 +6,7 @@ # Runs on every instance, using the `p` profile. VORTEX_CI_IS_BEHAT_RUNNER: true # Subtracted from the instance index to derive the profile number. Raise -@@ -499,18 +498,6 @@ +@@ -501,18 +500,6 @@ fi env: VORTEX_CI_CODE_COVERAGE_THRESHOLD: ${{ vars.VORTEX_CI_CODE_COVERAGE_THRESHOLD || '90' }} diff --git a/.vortex/installer/tests/Fixtures/handler_process/modules_no_devel_sdc_devel_generated_content/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/modules_no_devel_sdc_devel_generated_content/.github/workflows/build-test-deploy.yml index c0b7f62c2c..ab7610ac1b 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/modules_no_devel_sdc_devel_generated_content/.github/workflows/build-test-deploy.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/modules_no_devel_sdc_devel_generated_content/.github/workflows/build-test-deploy.yml @@ -1,4 +1,4 @@ -@@ -312,7 +312,6 @@ +@@ -314,7 +314,6 @@ VORTEX_CI_RUNNER_TOTAL: ${{ strategy.job-total }} VORTEX_CI_IS_JEST_RUNNER: ${{ matrix.instance == 0 || strategy.job-total == 1 }} VORTEX_CI_IS_PHPUNIT_RUNNER: ${{ matrix.instance == 0 || strategy.job-total == 1 }} @@ -6,7 +6,7 @@ # Runs on every instance, using the `p` profile. VORTEX_CI_IS_BEHAT_RUNNER: true # Subtracted from the instance index to derive the profile number. Raise -@@ -499,18 +498,6 @@ +@@ -501,18 +500,6 @@ fi env: VORTEX_CI_CODE_COVERAGE_THRESHOLD: ${{ vars.VORTEX_CI_CODE_COVERAGE_THRESHOLD || '90' }} diff --git a/.vortex/installer/tests/Fixtures/handler_process/modules_no_devel_sdc_devel_generated_content_testmode/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/modules_no_devel_sdc_devel_generated_content_testmode/.github/workflows/build-test-deploy.yml index c0b7f62c2c..ab7610ac1b 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/modules_no_devel_sdc_devel_generated_content_testmode/.github/workflows/build-test-deploy.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/modules_no_devel_sdc_devel_generated_content_testmode/.github/workflows/build-test-deploy.yml @@ -1,4 +1,4 @@ -@@ -312,7 +312,6 @@ +@@ -314,7 +314,6 @@ VORTEX_CI_RUNNER_TOTAL: ${{ strategy.job-total }} VORTEX_CI_IS_JEST_RUNNER: ${{ matrix.instance == 0 || strategy.job-total == 1 }} VORTEX_CI_IS_PHPUNIT_RUNNER: ${{ matrix.instance == 0 || strategy.job-total == 1 }} @@ -6,7 +6,7 @@ # Runs on every instance, using the `p` profile. VORTEX_CI_IS_BEHAT_RUNNER: true # Subtracted from the instance index to derive the profile number. Raise -@@ -499,18 +498,6 @@ +@@ -501,18 +500,6 @@ fi env: VORTEX_CI_CODE_COVERAGE_THRESHOLD: ${{ vars.VORTEX_CI_CODE_COVERAGE_THRESHOLD || '90' }} diff --git a/.vortex/installer/tests/Fixtures/handler_process/modules_no_devel_sdc_devel_generated_content_testmode_reroute_email/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/modules_no_devel_sdc_devel_generated_content_testmode_reroute_email/.github/workflows/build-test-deploy.yml index c0b7f62c2c..ab7610ac1b 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/modules_no_devel_sdc_devel_generated_content_testmode_reroute_email/.github/workflows/build-test-deploy.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/modules_no_devel_sdc_devel_generated_content_testmode_reroute_email/.github/workflows/build-test-deploy.yml @@ -1,4 +1,4 @@ -@@ -312,7 +312,6 @@ +@@ -314,7 +314,6 @@ VORTEX_CI_RUNNER_TOTAL: ${{ strategy.job-total }} VORTEX_CI_IS_JEST_RUNNER: ${{ matrix.instance == 0 || strategy.job-total == 1 }} VORTEX_CI_IS_PHPUNIT_RUNNER: ${{ matrix.instance == 0 || strategy.job-total == 1 }} @@ -6,7 +6,7 @@ # Runs on every instance, using the `p` profile. VORTEX_CI_IS_BEHAT_RUNNER: true # Subtracted from the instance index to derive the profile number. Raise -@@ -499,18 +498,6 @@ +@@ -501,18 +500,6 @@ fi env: VORTEX_CI_CODE_COVERAGE_THRESHOLD: ${{ vars.VORTEX_CI_CODE_COVERAGE_THRESHOLD || '90' }} diff --git a/.vortex/installer/tests/Fixtures/handler_process/modules_no_sdc_devel/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/modules_no_sdc_devel/.github/workflows/build-test-deploy.yml index c0b7f62c2c..ab7610ac1b 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/modules_no_sdc_devel/.github/workflows/build-test-deploy.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/modules_no_sdc_devel/.github/workflows/build-test-deploy.yml @@ -1,4 +1,4 @@ -@@ -312,7 +312,6 @@ +@@ -314,7 +314,6 @@ VORTEX_CI_RUNNER_TOTAL: ${{ strategy.job-total }} VORTEX_CI_IS_JEST_RUNNER: ${{ matrix.instance == 0 || strategy.job-total == 1 }} VORTEX_CI_IS_PHPUNIT_RUNNER: ${{ matrix.instance == 0 || strategy.job-total == 1 }} @@ -6,7 +6,7 @@ # Runs on every instance, using the `p` profile. VORTEX_CI_IS_BEHAT_RUNNER: true # Subtracted from the instance index to derive the profile number. Raise -@@ -499,18 +498,6 @@ +@@ -501,18 +500,6 @@ fi env: VORTEX_CI_CODE_COVERAGE_THRESHOLD: ${{ vars.VORTEX_CI_CODE_COVERAGE_THRESHOLD || '90' }} diff --git a/.vortex/installer/tests/Fixtures/handler_process/modules_none/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/modules_none/.github/workflows/build-test-deploy.yml index c0b7f62c2c..ab7610ac1b 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/modules_none/.github/workflows/build-test-deploy.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/modules_none/.github/workflows/build-test-deploy.yml @@ -1,4 +1,4 @@ -@@ -312,7 +312,6 @@ +@@ -314,7 +314,6 @@ VORTEX_CI_RUNNER_TOTAL: ${{ strategy.job-total }} VORTEX_CI_IS_JEST_RUNNER: ${{ matrix.instance == 0 || strategy.job-total == 1 }} VORTEX_CI_IS_PHPUNIT_RUNNER: ${{ matrix.instance == 0 || strategy.job-total == 1 }} @@ -6,7 +6,7 @@ # Runs on every instance, using the `p` profile. VORTEX_CI_IS_BEHAT_RUNNER: true # Subtracted from the instance index to derive the profile number. Raise -@@ -499,18 +498,6 @@ +@@ -501,18 +500,6 @@ fi env: VORTEX_CI_CODE_COVERAGE_THRESHOLD: ${{ vars.VORTEX_CI_CODE_COVERAGE_THRESHOLD || '90' }} diff --git a/.vortex/installer/tests/Fixtures/handler_process/provision_profile/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/provision_profile/.github/workflows/build-test-deploy.yml index 042d1c6b55..0dfba85dcf 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/provision_profile/.github/workflows/build-test-deploy.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/provision_profile/.github/workflows/build-test-deploy.yml @@ -15,7 +15,7 @@ container: # https://hub.docker.com/r/drevops/ci-runner -@@ -167,130 +164,8 @@ +@@ -167,132 +164,8 @@ run: docker compose exec -T cli bash -c "npm --prefix=\${WEBROOT}/themes/custom/\${DRUPAL_THEME} run lint" continue-on-error: ${{ vars.VORTEX_CI_NODEJS_LINT_IGNORE_FAILURE == '1' }} @@ -93,6 +93,8 @@ - { - echo "VORTEX_CI_DB_CACHE_FALLBACK=no" - echo "VORTEX_FRONTEND_BUILD_SKIP=1" +- # Refresh the database dumps in the fetch sources that are dump stores. +- echo "VORTEX_PUSH_DB_PROCEED=1" - } >> "$GITHUB_ENV" - - - name: Create cache keys files for database caching @@ -146,7 +148,7 @@ permissions: contents: read -@@ -333,14 +208,6 @@ +@@ -335,14 +208,6 @@ VORTEX_SSH_DISABLE_STRICT_HOST_KEY_CHECKING: "1" VORTEX_SSH_REMOVE_ALL_KEYS: "1" VORTEX_DEBUG: ${{ vars.VORTEX_DEBUG }} @@ -161,7 +163,7 @@ steps: # Frees disk space by removing preinstalled toolchains unused by this project. Steps run in a container, so the host is reached via the Docker socket. -@@ -379,29 +246,6 @@ +@@ -381,29 +246,6 @@ - name: Install Vortex tooling run: ./scripts/vortex-tooling.sh @@ -191,7 +193,7 @@ - name: Login to container registry run: ./vendor/bin/vortex-login-container-registry -@@ -577,7 +421,6 @@ +@@ -579,7 +421,6 @@ deploy: runs-on: ubuntu-latest needs: [build, lint] diff --git a/.vortex/installer/tests/Fixtures/handler_process/theme_claro/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/theme_claro/.github/workflows/build-test-deploy.yml index da2e5188f3..0cd1802313 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/theme_claro/.github/workflows/build-test-deploy.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/theme_claro/.github/workflows/build-test-deploy.yml @@ -10,7 +10,7 @@ database: runs-on: ubuntu-latest if: ${{ !inputs.deploy_target && (github.event_name == 'push' || !startsWith(github.head_ref, 'project/')) }} -@@ -312,7 +307,6 @@ +@@ -314,7 +309,6 @@ VORTEX_CI_RUNNER_TOTAL: ${{ strategy.job-total }} VORTEX_CI_IS_JEST_RUNNER: ${{ matrix.instance == 0 || strategy.job-total == 1 }} VORTEX_CI_IS_PHPUNIT_RUNNER: ${{ matrix.instance == 0 || strategy.job-total == 1 }} @@ -18,7 +18,7 @@ # Runs on every instance, using the `p` profile. VORTEX_CI_IS_BEHAT_RUNNER: true # Subtracted from the instance index to derive the profile number. Raise -@@ -499,18 +493,6 @@ +@@ -501,18 +495,6 @@ fi env: VORTEX_CI_CODE_COVERAGE_THRESHOLD: ${{ vars.VORTEX_CI_CODE_COVERAGE_THRESHOLD || '90' }} diff --git a/.vortex/installer/tests/Fixtures/handler_process/theme_olivero/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/theme_olivero/.github/workflows/build-test-deploy.yml index da2e5188f3..0cd1802313 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/theme_olivero/.github/workflows/build-test-deploy.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/theme_olivero/.github/workflows/build-test-deploy.yml @@ -10,7 +10,7 @@ database: runs-on: ubuntu-latest if: ${{ !inputs.deploy_target && (github.event_name == 'push' || !startsWith(github.head_ref, 'project/')) }} -@@ -312,7 +307,6 @@ +@@ -314,7 +309,6 @@ VORTEX_CI_RUNNER_TOTAL: ${{ strategy.job-total }} VORTEX_CI_IS_JEST_RUNNER: ${{ matrix.instance == 0 || strategy.job-total == 1 }} VORTEX_CI_IS_PHPUNIT_RUNNER: ${{ matrix.instance == 0 || strategy.job-total == 1 }} @@ -18,7 +18,7 @@ # Runs on every instance, using the `p` profile. VORTEX_CI_IS_BEHAT_RUNNER: true # Subtracted from the instance index to derive the profile number. Raise -@@ -499,18 +493,6 @@ +@@ -501,18 +495,6 @@ fi env: VORTEX_CI_CODE_COVERAGE_THRESHOLD: ${{ vars.VORTEX_CI_CODE_COVERAGE_THRESHOLD || '90' }} diff --git a/.vortex/installer/tests/Fixtures/handler_process/theme_stark/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/theme_stark/.github/workflows/build-test-deploy.yml index da2e5188f3..0cd1802313 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/theme_stark/.github/workflows/build-test-deploy.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/theme_stark/.github/workflows/build-test-deploy.yml @@ -10,7 +10,7 @@ database: runs-on: ubuntu-latest if: ${{ !inputs.deploy_target && (github.event_name == 'push' || !startsWith(github.head_ref, 'project/')) }} -@@ -312,7 +307,6 @@ +@@ -314,7 +309,6 @@ VORTEX_CI_RUNNER_TOTAL: ${{ strategy.job-total }} VORTEX_CI_IS_JEST_RUNNER: ${{ matrix.instance == 0 || strategy.job-total == 1 }} VORTEX_CI_IS_PHPUNIT_RUNNER: ${{ matrix.instance == 0 || strategy.job-total == 1 }} @@ -18,7 +18,7 @@ # Runs on every instance, using the `p` profile. VORTEX_CI_IS_BEHAT_RUNNER: true # Subtracted from the instance index to derive the profile number. Raise -@@ -499,18 +493,6 @@ +@@ -501,18 +495,6 @@ fi env: VORTEX_CI_CODE_COVERAGE_THRESHOLD: ${{ vars.VORTEX_CI_CODE_COVERAGE_THRESHOLD || '90' }} diff --git a/.vortex/installer/tests/Fixtures/handler_process/timezone_circleci/.circleci/config.yml b/.vortex/installer/tests/Fixtures/handler_process/timezone_circleci/.circleci/config.yml index f56bdf626c..b59dde79d2 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/timezone_circleci/.circleci/config.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/timezone_circleci/.circleci/config.yml @@ -307,6 +307,8 @@ jobs: VORTEX_DEPLOY_SSH_FINGERPRINT: *deploy_ssh_fingerprint # Enforce fresh DB build (do not rely on fallback caches). VORTEX_CI_DB_CACHE_FALLBACK: 'no' + # Refresh the database dumps in the fetch sources that are dump stores. + VORTEX_PUSH_DB_PROCEED: 1 # Do not build the Drupal front-end. VORTEX_FRONTEND_BUILD_SKIP: 1 diff --git a/.vortex/installer/tests/Fixtures/handler_process/tools_groups_no_be_lint_circleci/.circleci/config.yml b/.vortex/installer/tests/Fixtures/handler_process/tools_groups_no_be_lint_circleci/.circleci/config.yml index 962162638f..56e766a652 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/tools_groups_no_be_lint_circleci/.circleci/config.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/tools_groups_no_be_lint_circleci/.circleci/config.yml @@ -295,6 +295,8 @@ jobs: VORTEX_DEPLOY_SSH_FINGERPRINT: *deploy_ssh_fingerprint # Enforce fresh DB build (do not rely on fallback caches). VORTEX_CI_DB_CACHE_FALLBACK: 'no' + # Refresh the database dumps in the fetch sources that are dump stores. + VORTEX_PUSH_DB_PROCEED: 1 # Do not build the Drupal front-end. VORTEX_FRONTEND_BUILD_SKIP: 1 diff --git a/.vortex/installer/tests/Fixtures/handler_process/tools_groups_no_be_tests/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/tools_groups_no_be_tests/.github/workflows/build-test-deploy.yml index 2f4b0324b6..e5104a12f6 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/tools_groups_no_be_tests/.github/workflows/build-test-deploy.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/tools_groups_no_be_tests/.github/workflows/build-test-deploy.yml @@ -9,7 +9,7 @@ - name: Lint module code with NodeJS linters run: docker compose exec -T cli bash -c "npm run lint" continue-on-error: ${{ vars.VORTEX_CI_NODEJS_LINT_IGNORE_FAILURE == '1' }} -@@ -311,14 +307,7 @@ +@@ -313,14 +309,7 @@ VORTEX_CI_RUNNER_INDEX: ${{ strategy.job-index }} VORTEX_CI_RUNNER_TOTAL: ${{ strategy.job-total }} VORTEX_CI_IS_JEST_RUNNER: ${{ matrix.instance == 0 || strategy.job-total == 1 }} @@ -24,7 +24,7 @@ container: # https://hub.docker.com/r/drevops/ci-runner -@@ -440,66 +429,6 @@ +@@ -442,66 +431,6 @@ run: docker compose exec -T cli bash -c "npm test" continue-on-error: ${{ vars.VORTEX_CI_JEST_IGNORE_FAILURE == '1' }} @@ -91,7 +91,7 @@ - name: Validate Single Directory Components if: ${{ env.VORTEX_CI_IS_SDC_DEVEL_RUNNER == 'true' }} run: | -@@ -512,19 +441,6 @@ +@@ -514,19 +443,6 @@ fi continue-on-error: ${{ vars.VORTEX_CI_SDC_DEVEL_IGNORE_FAILURE == '1' }} @@ -111,7 +111,7 @@ - name: Process test logs and artifacts if: always() run: | -@@ -541,16 +457,6 @@ +@@ -543,16 +459,6 @@ path: .logs include-hidden-files: true if-no-files-found: error diff --git a/.vortex/installer/tests/Fixtures/handler_process/tools_groups_no_be_tests_circleci/.circleci/config.yml b/.vortex/installer/tests/Fixtures/handler_process/tools_groups_no_be_tests_circleci/.circleci/config.yml index 0b83288c75..4125037c62 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/tools_groups_no_be_tests_circleci/.circleci/config.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/tools_groups_no_be_tests_circleci/.circleci/config.yml @@ -303,6 +303,8 @@ jobs: VORTEX_DEPLOY_SSH_FINGERPRINT: *deploy_ssh_fingerprint # Enforce fresh DB build (do not rely on fallback caches). VORTEX_CI_DB_CACHE_FALLBACK: 'no' + # Refresh the database dumps in the fetch sources that are dump stores. + VORTEX_PUSH_DB_PROCEED: 1 # Do not build the Drupal front-end. VORTEX_FRONTEND_BUILD_SKIP: 1 diff --git a/.vortex/installer/tests/Fixtures/handler_process/tools_groups_no_fe_lint/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/tools_groups_no_fe_lint/.github/workflows/build-test-deploy.yml index 452179b6d7..4d1ca474b4 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/tools_groups_no_fe_lint/.github/workflows/build-test-deploy.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/tools_groups_no_fe_lint/.github/workflows/build-test-deploy.yml @@ -22,7 +22,7 @@ database: runs-on: ubuntu-latest if: ${{ !inputs.deploy_target && (github.event_name == 'push' || !startsWith(github.head_ref, 'project/')) }} -@@ -310,7 +300,6 @@ +@@ -312,7 +302,6 @@ # https://www.vortextemplate.com/docs/continuous-integration#test-parallelism VORTEX_CI_RUNNER_INDEX: ${{ strategy.job-index }} VORTEX_CI_RUNNER_TOTAL: ${{ strategy.job-total }} @@ -30,7 +30,7 @@ VORTEX_CI_IS_PHPUNIT_RUNNER: ${{ matrix.instance == 0 || strategy.job-total == 1 }} VORTEX_CI_IS_SDC_DEVEL_RUNNER: ${{ matrix.instance == 0 || strategy.job-total == 1 }} # Runs on every instance, using the `p` profile. -@@ -421,7 +410,6 @@ +@@ -423,7 +412,6 @@ docker compose exec $(env | cut -f1 -d= | sed 's/^/-e /') -T cli bash -c " \ if [ -n \"\${PACKAGE_TOKEN:-}\" ]; then composer config --global --auth github-oauth.github.com \"\${PACKAGE_TOKEN}\"; fi && \ COMPOSER_MEMORY_LIMIT=-1 composer --ansi install --prefer-dist" @@ -38,7 +38,7 @@ - name: Provision site run: | -@@ -434,11 +422,6 @@ +@@ -436,11 +424,6 @@ fi docker compose exec $(env | cut -f1 -d= | sed 's/^/-e /') -T cli ./vendor/bin/vortex-provision timeout-minutes: 30 diff --git a/.vortex/installer/tests/Fixtures/handler_process/tools_groups_no_fe_lint_circleci/.circleci/config.yml b/.vortex/installer/tests/Fixtures/handler_process/tools_groups_no_fe_lint_circleci/.circleci/config.yml index 0d5dd2cae9..d21bef58a2 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/tools_groups_no_fe_lint_circleci/.circleci/config.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/tools_groups_no_fe_lint_circleci/.circleci/config.yml @@ -286,6 +286,8 @@ jobs: VORTEX_DEPLOY_SSH_FINGERPRINT: *deploy_ssh_fingerprint # Enforce fresh DB build (do not rely on fallback caches). VORTEX_CI_DB_CACHE_FALLBACK: 'no' + # Refresh the database dumps in the fetch sources that are dump stores. + VORTEX_PUSH_DB_PROCEED: 1 # Do not build the Drupal front-end. VORTEX_FRONTEND_BUILD_SKIP: 1 diff --git a/.vortex/installer/tests/Fixtures/handler_process/tools_groups_no_fe_lint_no_theme/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/tools_groups_no_fe_lint_no_theme/.github/workflows/build-test-deploy.yml index 8ffe668f17..3245313f2f 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/tools_groups_no_fe_lint_no_theme/.github/workflows/build-test-deploy.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/tools_groups_no_fe_lint_no_theme/.github/workflows/build-test-deploy.yml @@ -22,7 +22,7 @@ database: runs-on: ubuntu-latest if: ${{ !inputs.deploy_target && (github.event_name == 'push' || !startsWith(github.head_ref, 'project/')) }} -@@ -310,9 +300,7 @@ +@@ -312,9 +302,7 @@ # https://www.vortextemplate.com/docs/continuous-integration#test-parallelism VORTEX_CI_RUNNER_INDEX: ${{ strategy.job-index }} VORTEX_CI_RUNNER_TOTAL: ${{ strategy.job-total }} @@ -32,7 +32,7 @@ # Runs on every instance, using the `p` profile. VORTEX_CI_IS_BEHAT_RUNNER: true # Subtracted from the instance index to derive the profile number. Raise -@@ -421,7 +409,6 @@ +@@ -423,7 +411,6 @@ docker compose exec $(env | cut -f1 -d= | sed 's/^/-e /') -T cli bash -c " \ if [ -n \"\${PACKAGE_TOKEN:-}\" ]; then composer config --global --auth github-oauth.github.com \"\${PACKAGE_TOKEN}\"; fi && \ COMPOSER_MEMORY_LIMIT=-1 composer --ansi install --prefer-dist" @@ -40,7 +40,7 @@ - name: Provision site run: | -@@ -435,11 +422,6 @@ +@@ -437,11 +424,6 @@ docker compose exec $(env | cut -f1 -d= | sed 's/^/-e /') -T cli ./vendor/bin/vortex-provision timeout-minutes: 30 @@ -52,7 +52,7 @@ - name: Test with PHPUnit if: ${{ env.VORTEX_CI_IS_PHPUNIT_RUNNER == 'true' }} run: docker compose exec -T cli vendor/bin/phpunit -@@ -499,18 +481,6 @@ +@@ -501,18 +483,6 @@ fi env: VORTEX_CI_CODE_COVERAGE_THRESHOLD: ${{ vars.VORTEX_CI_CODE_COVERAGE_THRESHOLD || '90' }} diff --git a/.vortex/installer/tests/Fixtures/handler_process/tools_groups_no_fe_lint_no_theme_circleci/.circleci/config.yml b/.vortex/installer/tests/Fixtures/handler_process/tools_groups_no_fe_lint_no_theme_circleci/.circleci/config.yml index 87d8397465..8e015f51d7 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/tools_groups_no_fe_lint_no_theme_circleci/.circleci/config.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/tools_groups_no_fe_lint_no_theme_circleci/.circleci/config.yml @@ -277,6 +277,8 @@ jobs: VORTEX_DEPLOY_SSH_FINGERPRINT: *deploy_ssh_fingerprint # Enforce fresh DB build (do not rely on fallback caches). VORTEX_CI_DB_CACHE_FALLBACK: 'no' + # Refresh the database dumps in the fetch sources that are dump stores. + VORTEX_PUSH_DB_PROCEED: 1 # Do not build the Drupal front-end. VORTEX_FRONTEND_BUILD_SKIP: 1 diff --git a/.vortex/installer/tests/Fixtures/handler_process/tools_no_behat/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/tools_no_behat/.github/workflows/build-test-deploy.yml index 0b32b5b70d..fd50aad45b 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/tools_no_behat/.github/workflows/build-test-deploy.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/tools_no_behat/.github/workflows/build-test-deploy.yml @@ -9,7 +9,7 @@ - name: Lint module code with NodeJS linters run: docker compose exec -T cli bash -c "npm run lint" continue-on-error: ${{ vars.VORTEX_CI_NODEJS_LINT_IGNORE_FAILURE == '1' }} -@@ -313,12 +309,6 @@ +@@ -315,12 +311,6 @@ VORTEX_CI_IS_JEST_RUNNER: ${{ matrix.instance == 0 || strategy.job-total == 1 }} VORTEX_CI_IS_PHPUNIT_RUNNER: ${{ matrix.instance == 0 || strategy.job-total == 1 }} VORTEX_CI_IS_SDC_DEVEL_RUNNER: ${{ matrix.instance == 0 || strategy.job-total == 1 }} @@ -22,7 +22,7 @@ container: # https://hub.docker.com/r/drevops/ci-runner -@@ -511,19 +501,6 @@ +@@ -513,19 +503,6 @@ exit 1 fi continue-on-error: ${{ vars.VORTEX_CI_SDC_DEVEL_IGNORE_FAILURE == '1' }} diff --git a/.vortex/installer/tests/Fixtures/handler_process/tools_no_behat_circleci/.circleci/config.yml b/.vortex/installer/tests/Fixtures/handler_process/tools_no_behat_circleci/.circleci/config.yml index 194ac3218b..4735766d37 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/tools_no_behat_circleci/.circleci/config.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/tools_no_behat_circleci/.circleci/config.yml @@ -303,6 +303,8 @@ jobs: VORTEX_DEPLOY_SSH_FINGERPRINT: *deploy_ssh_fingerprint # Enforce fresh DB build (do not rely on fallback caches). VORTEX_CI_DB_CACHE_FALLBACK: 'no' + # Refresh the database dumps in the fetch sources that are dump stores. + VORTEX_PUSH_DB_PROCEED: 1 # Do not build the Drupal front-end. VORTEX_FRONTEND_BUILD_SKIP: 1 diff --git a/.vortex/installer/tests/Fixtures/handler_process/tools_no_dclint_circleci/.circleci/config.yml b/.vortex/installer/tests/Fixtures/handler_process/tools_no_dclint_circleci/.circleci/config.yml index 3cf240711d..54dc4477e3 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/tools_no_dclint_circleci/.circleci/config.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/tools_no_dclint_circleci/.circleci/config.yml @@ -298,6 +298,8 @@ jobs: VORTEX_DEPLOY_SSH_FINGERPRINT: *deploy_ssh_fingerprint # Enforce fresh DB build (do not rely on fallback caches). VORTEX_CI_DB_CACHE_FALLBACK: 'no' + # Refresh the database dumps in the fetch sources that are dump stores. + VORTEX_PUSH_DB_PROCEED: 1 # Do not build the Drupal front-end. VORTEX_FRONTEND_BUILD_SKIP: 1 diff --git a/.vortex/installer/tests/Fixtures/handler_process/tools_no_dclint_migration/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/tools_no_dclint_migration/.github/workflows/build-test-deploy.yml index f4e7a01167..03afc8216b 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/tools_no_dclint_migration/.github/workflows/build-test-deploy.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/tools_no_dclint_migration/.github/workflows/build-test-deploy.yml @@ -9,7 +9,7 @@ - name: Build stack run: docker compose up --no-deps --detach cli -@@ -270,6 +266,9 @@ +@@ -272,6 +268,9 @@ echo "db_hash=${{ hashFiles('.data') }}" >> "$GITHUB_ENV" timeout-minutes: 30 @@ -19,7 +19,7 @@ - name: Export DB run: | if [ ! -f /tmp/fetch-db-fresh ]; then echo "==> Database fetch semaphore file is missing. DB export will not proceed."; exit 0; fi -@@ -431,6 +430,11 @@ +@@ -433,6 +432,11 @@ docker compose exec cli mkdir -p .data docker compose --progress quiet cp -L .data/db.sql cli:/app/.data/db.sql rm -f .data/db.sql diff --git a/.vortex/installer/tests/Fixtures/handler_process/tools_no_docker_linters_circleci/.circleci/config.yml b/.vortex/installer/tests/Fixtures/handler_process/tools_no_docker_linters_circleci/.circleci/config.yml index 64bcd3e9ab..fffe322f5d 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/tools_no_docker_linters_circleci/.circleci/config.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/tools_no_docker_linters_circleci/.circleci/config.yml @@ -291,6 +291,8 @@ jobs: VORTEX_DEPLOY_SSH_FINGERPRINT: *deploy_ssh_fingerprint # Enforce fresh DB build (do not rely on fallback caches). VORTEX_CI_DB_CACHE_FALLBACK: 'no' + # Refresh the database dumps in the fetch sources that are dump stores. + VORTEX_PUSH_DB_PROCEED: 1 # Do not build the Drupal front-end. VORTEX_FRONTEND_BUILD_SKIP: 1 diff --git a/.vortex/installer/tests/Fixtures/handler_process/tools_no_eslint_circleci/.circleci/config.yml b/.vortex/installer/tests/Fixtures/handler_process/tools_no_eslint_circleci/.circleci/config.yml index f56bdf626c..b59dde79d2 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/tools_no_eslint_circleci/.circleci/config.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/tools_no_eslint_circleci/.circleci/config.yml @@ -307,6 +307,8 @@ jobs: VORTEX_DEPLOY_SSH_FINGERPRINT: *deploy_ssh_fingerprint # Enforce fresh DB build (do not rely on fallback caches). VORTEX_CI_DB_CACHE_FALLBACK: 'no' + # Refresh the database dumps in the fetch sources that are dump stores. + VORTEX_PUSH_DB_PROCEED: 1 # Do not build the Drupal front-end. VORTEX_FRONTEND_BUILD_SKIP: 1 diff --git a/.vortex/installer/tests/Fixtures/handler_process/tools_no_eslint_no_theme/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/tools_no_eslint_no_theme/.github/workflows/build-test-deploy.yml index da2e5188f3..0cd1802313 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/tools_no_eslint_no_theme/.github/workflows/build-test-deploy.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/tools_no_eslint_no_theme/.github/workflows/build-test-deploy.yml @@ -10,7 +10,7 @@ database: runs-on: ubuntu-latest if: ${{ !inputs.deploy_target && (github.event_name == 'push' || !startsWith(github.head_ref, 'project/')) }} -@@ -312,7 +307,6 @@ +@@ -314,7 +309,6 @@ VORTEX_CI_RUNNER_TOTAL: ${{ strategy.job-total }} VORTEX_CI_IS_JEST_RUNNER: ${{ matrix.instance == 0 || strategy.job-total == 1 }} VORTEX_CI_IS_PHPUNIT_RUNNER: ${{ matrix.instance == 0 || strategy.job-total == 1 }} @@ -18,7 +18,7 @@ # Runs on every instance, using the `p` profile. VORTEX_CI_IS_BEHAT_RUNNER: true # Subtracted from the instance index to derive the profile number. Raise -@@ -499,18 +493,6 @@ +@@ -501,18 +495,6 @@ fi env: VORTEX_CI_CODE_COVERAGE_THRESHOLD: ${{ vars.VORTEX_CI_CODE_COVERAGE_THRESHOLD || '90' }} diff --git a/.vortex/installer/tests/Fixtures/handler_process/tools_no_hadolint_circleci/.circleci/config.yml b/.vortex/installer/tests/Fixtures/handler_process/tools_no_hadolint_circleci/.circleci/config.yml index 37281e8ff5..121b9f5477 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/tools_no_hadolint_circleci/.circleci/config.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/tools_no_hadolint_circleci/.circleci/config.yml @@ -300,6 +300,8 @@ jobs: VORTEX_DEPLOY_SSH_FINGERPRINT: *deploy_ssh_fingerprint # Enforce fresh DB build (do not rely on fallback caches). VORTEX_CI_DB_CACHE_FALLBACK: 'no' + # Refresh the database dumps in the fetch sources that are dump stores. + VORTEX_PUSH_DB_PROCEED: 1 # Do not build the Drupal front-end. VORTEX_FRONTEND_BUILD_SKIP: 1 diff --git a/.vortex/installer/tests/Fixtures/handler_process/tools_no_jest/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/tools_no_jest/.github/workflows/build-test-deploy.yml index a4847a4654..2963f464ca 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/tools_no_jest/.github/workflows/build-test-deploy.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/tools_no_jest/.github/workflows/build-test-deploy.yml @@ -1,4 +1,4 @@ -@@ -310,7 +310,6 @@ +@@ -312,7 +312,6 @@ # https://www.vortextemplate.com/docs/continuous-integration#test-parallelism VORTEX_CI_RUNNER_INDEX: ${{ strategy.job-index }} VORTEX_CI_RUNNER_TOTAL: ${{ strategy.job-total }} @@ -6,7 +6,7 @@ VORTEX_CI_IS_PHPUNIT_RUNNER: ${{ matrix.instance == 0 || strategy.job-total == 1 }} VORTEX_CI_IS_SDC_DEVEL_RUNNER: ${{ matrix.instance == 0 || strategy.job-total == 1 }} # Runs on every instance, using the `p` profile. -@@ -421,7 +420,6 @@ +@@ -423,7 +422,6 @@ docker compose exec $(env | cut -f1 -d= | sed 's/^/-e /') -T cli bash -c " \ if [ -n \"\${PACKAGE_TOKEN:-}\" ]; then composer config --global --auth github-oauth.github.com \"\${PACKAGE_TOKEN}\"; fi && \ COMPOSER_MEMORY_LIMIT=-1 composer --ansi install --prefer-dist" @@ -14,7 +14,7 @@ - name: Provision site run: | -@@ -434,11 +432,6 @@ +@@ -436,11 +434,6 @@ fi docker compose exec $(env | cut -f1 -d= | sed 's/^/-e /') -T cli ./vendor/bin/vortex-provision timeout-minutes: 30 diff --git a/.vortex/installer/tests/Fixtures/handler_process/tools_no_jest_circleci/.circleci/config.yml b/.vortex/installer/tests/Fixtures/handler_process/tools_no_jest_circleci/.circleci/config.yml index 52fdedfed1..8cbd50d31b 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/tools_no_jest_circleci/.circleci/config.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/tools_no_jest_circleci/.circleci/config.yml @@ -307,6 +307,8 @@ jobs: VORTEX_DEPLOY_SSH_FINGERPRINT: *deploy_ssh_fingerprint # Enforce fresh DB build (do not rely on fallback caches). VORTEX_CI_DB_CACHE_FALLBACK: 'no' + # Refresh the database dumps in the fetch sources that are dump stores. + VORTEX_PUSH_DB_PROCEED: 1 # Do not build the Drupal front-end. VORTEX_FRONTEND_BUILD_SKIP: 1 diff --git a/.vortex/installer/tests/Fixtures/handler_process/tools_no_phpcs_circleci/.circleci/config.yml b/.vortex/installer/tests/Fixtures/handler_process/tools_no_phpcs_circleci/.circleci/config.yml index f2aacd0e81..3262246832 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/tools_no_phpcs_circleci/.circleci/config.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/tools_no_phpcs_circleci/.circleci/config.yml @@ -303,6 +303,8 @@ jobs: VORTEX_DEPLOY_SSH_FINGERPRINT: *deploy_ssh_fingerprint # Enforce fresh DB build (do not rely on fallback caches). VORTEX_CI_DB_CACHE_FALLBACK: 'no' + # Refresh the database dumps in the fetch sources that are dump stores. + VORTEX_PUSH_DB_PROCEED: 1 # Do not build the Drupal front-end. VORTEX_FRONTEND_BUILD_SKIP: 1 diff --git a/.vortex/installer/tests/Fixtures/handler_process/tools_no_phpstan_circleci/.circleci/config.yml b/.vortex/installer/tests/Fixtures/handler_process/tools_no_phpstan_circleci/.circleci/config.yml index bf65b2a0ba..8fc685f729 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/tools_no_phpstan_circleci/.circleci/config.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/tools_no_phpstan_circleci/.circleci/config.yml @@ -303,6 +303,8 @@ jobs: VORTEX_DEPLOY_SSH_FINGERPRINT: *deploy_ssh_fingerprint # Enforce fresh DB build (do not rely on fallback caches). VORTEX_CI_DB_CACHE_FALLBACK: 'no' + # Refresh the database dumps in the fetch sources that are dump stores. + VORTEX_PUSH_DB_PROCEED: 1 # Do not build the Drupal front-end. VORTEX_FRONTEND_BUILD_SKIP: 1 diff --git a/.vortex/installer/tests/Fixtures/handler_process/tools_no_phpunit/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/tools_no_phpunit/.github/workflows/build-test-deploy.yml index 83fae1f090..4bf951f78a 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/tools_no_phpunit/.github/workflows/build-test-deploy.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/tools_no_phpunit/.github/workflows/build-test-deploy.yml @@ -1,4 +1,4 @@ -@@ -311,7 +311,6 @@ +@@ -313,7 +313,6 @@ VORTEX_CI_RUNNER_INDEX: ${{ strategy.job-index }} VORTEX_CI_RUNNER_TOTAL: ${{ strategy.job-total }} VORTEX_CI_IS_JEST_RUNNER: ${{ matrix.instance == 0 || strategy.job-total == 1 }} @@ -6,7 +6,7 @@ VORTEX_CI_IS_SDC_DEVEL_RUNNER: ${{ matrix.instance == 0 || strategy.job-total == 1 }} # Runs on every instance, using the `p` profile. VORTEX_CI_IS_BEHAT_RUNNER: true -@@ -439,66 +438,6 @@ +@@ -441,66 +440,6 @@ if: ${{ env.VORTEX_CI_IS_JEST_RUNNER == 'true' }} run: docker compose exec -T cli bash -c "npm test" continue-on-error: ${{ vars.VORTEX_CI_JEST_IGNORE_FAILURE == '1' }} diff --git a/.vortex/installer/tests/Fixtures/handler_process/tools_no_phpunit_circleci/.circleci/config.yml b/.vortex/installer/tests/Fixtures/handler_process/tools_no_phpunit_circleci/.circleci/config.yml index affec2423a..05b7fbe608 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/tools_no_phpunit_circleci/.circleci/config.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/tools_no_phpunit_circleci/.circleci/config.yml @@ -307,6 +307,8 @@ jobs: VORTEX_DEPLOY_SSH_FINGERPRINT: *deploy_ssh_fingerprint # Enforce fresh DB build (do not rely on fallback caches). VORTEX_CI_DB_CACHE_FALLBACK: 'no' + # Refresh the database dumps in the fetch sources that are dump stores. + VORTEX_PUSH_DB_PROCEED: 1 # Do not build the Drupal front-end. VORTEX_FRONTEND_BUILD_SKIP: 1 diff --git a/.vortex/installer/tests/Fixtures/handler_process/tools_no_rector_circleci/.circleci/config.yml b/.vortex/installer/tests/Fixtures/handler_process/tools_no_rector_circleci/.circleci/config.yml index 65af74315e..dfdbda3a2a 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/tools_no_rector_circleci/.circleci/config.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/tools_no_rector_circleci/.circleci/config.yml @@ -303,6 +303,8 @@ jobs: VORTEX_DEPLOY_SSH_FINGERPRINT: *deploy_ssh_fingerprint # Enforce fresh DB build (do not rely on fallback caches). VORTEX_CI_DB_CACHE_FALLBACK: 'no' + # Refresh the database dumps in the fetch sources that are dump stores. + VORTEX_PUSH_DB_PROCEED: 1 # Do not build the Drupal front-end. VORTEX_FRONTEND_BUILD_SKIP: 1 diff --git a/.vortex/installer/tests/Fixtures/handler_process/tools_no_stylelint_circleci/.circleci/config.yml b/.vortex/installer/tests/Fixtures/handler_process/tools_no_stylelint_circleci/.circleci/config.yml index f56bdf626c..b59dde79d2 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/tools_no_stylelint_circleci/.circleci/config.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/tools_no_stylelint_circleci/.circleci/config.yml @@ -307,6 +307,8 @@ jobs: VORTEX_DEPLOY_SSH_FINGERPRINT: *deploy_ssh_fingerprint # Enforce fresh DB build (do not rely on fallback caches). VORTEX_CI_DB_CACHE_FALLBACK: 'no' + # Refresh the database dumps in the fetch sources that are dump stores. + VORTEX_PUSH_DB_PROCEED: 1 # Do not build the Drupal front-end. VORTEX_FRONTEND_BUILD_SKIP: 1 diff --git a/.vortex/installer/tests/Fixtures/handler_process/tools_no_stylelint_no_theme/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/tools_no_stylelint_no_theme/.github/workflows/build-test-deploy.yml index da2e5188f3..0cd1802313 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/tools_no_stylelint_no_theme/.github/workflows/build-test-deploy.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/tools_no_stylelint_no_theme/.github/workflows/build-test-deploy.yml @@ -10,7 +10,7 @@ database: runs-on: ubuntu-latest if: ${{ !inputs.deploy_target && (github.event_name == 'push' || !startsWith(github.head_ref, 'project/')) }} -@@ -312,7 +307,6 @@ +@@ -314,7 +309,6 @@ VORTEX_CI_RUNNER_TOTAL: ${{ strategy.job-total }} VORTEX_CI_IS_JEST_RUNNER: ${{ matrix.instance == 0 || strategy.job-total == 1 }} VORTEX_CI_IS_PHPUNIT_RUNNER: ${{ matrix.instance == 0 || strategy.job-total == 1 }} @@ -18,7 +18,7 @@ # Runs on every instance, using the `p` profile. VORTEX_CI_IS_BEHAT_RUNNER: true # Subtracted from the instance index to derive the profile number. Raise -@@ -499,18 +493,6 @@ +@@ -501,18 +495,6 @@ fi env: VORTEX_CI_CODE_COVERAGE_THRESHOLD: ${{ vars.VORTEX_CI_CODE_COVERAGE_THRESHOLD || '90' }} diff --git a/.vortex/installer/tests/Fixtures/handler_process/tools_no_twig_circleci/.circleci/config.yml b/.vortex/installer/tests/Fixtures/handler_process/tools_no_twig_circleci/.circleci/config.yml index 5001ef54a4..f2890ba107 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/tools_no_twig_circleci/.circleci/config.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/tools_no_twig_circleci/.circleci/config.yml @@ -303,6 +303,8 @@ jobs: VORTEX_DEPLOY_SSH_FINGERPRINT: *deploy_ssh_fingerprint # Enforce fresh DB build (do not rely on fallback caches). VORTEX_CI_DB_CACHE_FALLBACK: 'no' + # Refresh the database dumps in the fetch sources that are dump stores. + VORTEX_PUSH_DB_PROCEED: 1 # Do not build the Drupal front-end. VORTEX_FRONTEND_BUILD_SKIP: 1 diff --git a/.vortex/installer/tests/Fixtures/handler_process/tools_none/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/tools_none/.github/workflows/build-test-deploy.yml index ecbb0ad695..e3c358bb75 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/tools_none/.github/workflows/build-test-deploy.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/tools_none/.github/workflows/build-test-deploy.yml @@ -58,7 +58,7 @@ database: runs-on: ubuntu-latest if: ${{ !inputs.deploy_target && (github.event_name == 'push' || !startsWith(github.head_ref, 'project/')) }} -@@ -310,15 +269,7 @@ +@@ -312,15 +271,7 @@ # https://www.vortextemplate.com/docs/continuous-integration#test-parallelism VORTEX_CI_RUNNER_INDEX: ${{ strategy.job-index }} VORTEX_CI_RUNNER_TOTAL: ${{ strategy.job-total }} @@ -74,7 +74,7 @@ container: # https://hub.docker.com/r/drevops/ci-runner -@@ -421,7 +372,6 @@ +@@ -423,7 +374,6 @@ docker compose exec $(env | cut -f1 -d= | sed 's/^/-e /') -T cli bash -c " \ if [ -n \"\${PACKAGE_TOKEN:-}\" ]; then composer config --global --auth github-oauth.github.com \"\${PACKAGE_TOKEN}\"; fi && \ COMPOSER_MEMORY_LIMIT=-1 composer --ansi install --prefer-dist" @@ -82,7 +82,7 @@ - name: Provision site run: | -@@ -435,71 +385,6 @@ +@@ -437,71 +387,6 @@ docker compose exec $(env | cut -f1 -d= | sed 's/^/-e /') -T cli ./vendor/bin/vortex-provision timeout-minutes: 30 @@ -154,7 +154,7 @@ - name: Validate Single Directory Components if: ${{ env.VORTEX_CI_IS_SDC_DEVEL_RUNNER == 'true' }} run: | -@@ -512,19 +397,6 @@ +@@ -514,19 +399,6 @@ fi continue-on-error: ${{ vars.VORTEX_CI_SDC_DEVEL_IGNORE_FAILURE == '1' }} @@ -174,7 +174,7 @@ - name: Process test logs and artifacts if: always() run: | -@@ -541,16 +413,6 @@ +@@ -543,16 +415,6 @@ path: .logs include-hidden-files: true if-no-files-found: error From e8f36d05e45daca4f241bd584a0c55ea537c5ec6 Mon Sep 17 00:00:00 2001 From: Alex Skrypnyk Date: Wed, 11 Nov 2026 15:14:45 +1100 Subject: [PATCH 15/21] Pushed the latest dump under the remote file name with an optional suffix and an opt-in timestamped copy, in place of the previous copy. --- .circleci/config.yml | 6 +- .env | 10 +- .github/workflows/build-test-deploy.yml | 6 +- .../content/development/database-storage.mdx | 51 ++++----- .vortex/tooling/playground/try-push-db-ftp.sh | 4 +- .vortex/tooling/src/vortex-push-db-ftp | 88 +++++++++------ .vortex/tooling/src/vortex-push-db-s3 | 97 +++++++++-------- .vortex/tooling/tests/_helper.bash | 3 +- .vortex/tooling/tests/unit/push-db-ftp.bats | 102 ++++++++++++++---- .vortex/tooling/tests/unit/push-db-s3.bats | 84 +++++++-------- 10 files changed, 267 insertions(+), 184 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 8e9a014630..23a1515596 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -416,8 +416,10 @@ jobs: # Refresh the database dumps in the fetch sources that are dump stores. VORTEX_PUSH_DB_PROCEED: 1 #;< VORTEX_DEV - # Tell this provider's demo database dumps apart on the test FTP server. - VORTEX_PUSH_DB_FILE_PREFIX: circleci- + # Tell this provider's demo database dumps apart on the test FTP server, + # with a timestamped copy of each push. + VORTEX_PUSH_DB_FILE_SUFFIX: _circleci + VORTEX_PUSH_DB_TIMESTAMPED_COPY: 1 #;> VORTEX_DEV # Do not build the Drupal front-end. VORTEX_FRONTEND_BUILD_SKIP: 1 diff --git a/.env b/.env index 41d5604266..661a9a946b 100644 --- a/.env +++ b/.env @@ -263,10 +263,14 @@ VORTEX_FETCH_DB_S3_REGION=ap-southeast-2 # # Uncomment to make the database fetch source a dump store: the nightly CI # job fetches the dump from this source, sanitizes it and pushes it to the -# fetch source, keeping the dump it replaces as the previous one. The fetch -# script of this source reads its own variables, such as VORTEX_FETCH_DB_URL -# for url. Add them, and keep secrets in CI. +# fetch source as the latest dump. The fetch script of this source reads its +# own variables, such as VORTEX_FETCH_DB_URL for url. Add them, and keep +# secrets in CI. # VORTEX_REFRESH_DB_SOURCE=url + +# Also push a copy of each dump named with a UTC timestamp, such as +# db_20261111-180000.sql, to keep a history of the pushed dumps. +# VORTEX_PUSH_DB_TIMESTAMPED_COPY=1 #;> DB_PUSH # Environment to fetch the database from. diff --git a/.github/workflows/build-test-deploy.yml b/.github/workflows/build-test-deploy.yml index b8f27a0700..00e0d1be9b 100644 --- a/.github/workflows/build-test-deploy.yml +++ b/.github/workflows/build-test-deploy.yml @@ -243,13 +243,15 @@ jobs: #;> MIGRATION_DB_FETCH_SOURCE_S3 #;< VORTEX_DEV # FTP server that receives the demo database on the nightly run, under - # a prefix that tells this provider's dumps apart. + # a suffix that tells this provider's dumps apart, with a timestamped + # copy of each push. VORTEX_PUSH_DB_FTP_HOST: ${{ secrets.VORTEX_PUSH_DB_FTP_HOST }} VORTEX_PUSH_DB_FTP_PORT: ${{ secrets.VORTEX_PUSH_DB_FTP_PORT }} VORTEX_PUSH_DB_FTP_USER: ${{ secrets.VORTEX_PUSH_DB_FTP_USER }} VORTEX_PUSH_DB_FTP_PASS: ${{ secrets.VORTEX_PUSH_DB_FTP_PASS }} VORTEX_PUSH_DB_FTP_FILE: ${{ vars.VORTEX_PUSH_DB_FTP_FILE || 'db.sql' }} - VORTEX_PUSH_DB_FILE_PREFIX: gha- + VORTEX_PUSH_DB_FILE_SUFFIX: _gha + VORTEX_PUSH_DB_TIMESTAMPED_COPY: 1 #;> VORTEX_DEV steps: diff --git a/.vortex/docs/content/development/database-storage.mdx b/.vortex/docs/content/development/database-storage.mdx index e0a7dad597..0780d95a3f 100644 --- a/.vortex/docs/content/development/database-storage.mdx +++ b/.vortex/docs/content/development/database-storage.mdx @@ -78,18 +78,21 @@ refreshes the store from the upstream source: │ provisions the dump, sanitizes it and exports it ▼ Push DB (vortex-push-db) - │ 1. keeps the current dump as db-previous.sql - │ 2. uploads the new dump as db.sql + │ 1. replaces db.sql with the new dump + │ 2. with VORTEX_PUSH_DB_TIMESTAMPED_COPY=1, adds a timestamped copy ▼ FTP server or S3 bucket - ├── db.sql ◀── ahoy fetch-db, branch builds - └── db-previous.sql + ├── db.sql ◀── ahoy fetch-db, branch builds + ├── db_20261110-180000.sql timestamped copies, opt-in + └── db_20261111-180000.sql ``` -The store always holds 2 copies: the current dump and the dump it -replaced. Fetching reads the current dump. To fetch the previous dump from -an FTP store, set `VORTEX_FETCH_DB_FTP_FILE` to its path in `.env.local`, -for example `your_site/db-previous.sql`. +`db.sql` always holds the latest dump, and fetching downloads it. By +default, it's the only file a push creates. To keep a history of the +pushed dumps, set `VORTEX_PUSH_DB_TIMESTAMPED_COPY=1` in `.env`. Each push +then also creates a copy named with its UTC time, identical to `db.sql` +at that moment. Nothing removes the copies, so clear out the old ones +yourself. Every other CI run fetches from the store, so only the nightly run needs the credentials of the upstream source. @@ -105,10 +108,10 @@ Its files sit next to the primary dump in the same store: ```text FTP server or S3 bucket - ├── db.sql primary database (VORTEX_FETCH_DB_FTP_FILE) - ├── db-previous.sql - ├── db2.sql migration database (VORTEX_FETCH_DB2_FTP_FILE) - └── db2-previous.sql + ├── db.sql primary database (VORTEX_FETCH_DB_FTP_FILE) + ├── db_20261111-180000.sql + ├── db2.sql migration database (VORTEX_FETCH_DB2_FTP_FILE) + └── db2_20261111-180000.sql ``` To make the migration source a dump store, set @@ -120,7 +123,7 @@ To make the migration source a dump store, set Before you begin, create an FTP account that can write to a directory on the server. The push uploads the dump under a temporary name, then renames -the files into place. The server must allow the `DELE`, `RNFR` and `RNTO` +it into place, so a failed upload keeps the latest dump. The server must allow the `DELE`, `RNFR` and `RNTO` commands, which standard FTP servers do. 1. In `.env`, set the fetch source and the server: @@ -167,20 +170,11 @@ Before you begin, create a private S3 bucket with public access blocked. "Effect": "Allow", "Action": ["s3:GetObject", "s3:PutObject"], "Resource": "arn:aws:s3:::your-bucket/*" - }, - { - "Effect": "Allow", - "Action": ["s3:ListBucket"], - "Resource": "arn:aws:s3:::your-bucket" } ] } ``` - The push copies the current dump before the upload. Without - `s3:ListBucket`, S3 answers a missing dump with an access error, and - the first push fails. - 2. In `.env`, set the fetch source and the bucket: ```shell @@ -200,15 +194,16 @@ Before you begin, create a private S3 bucket with public access blocked. 6. After the next nightly run, confirm that the **Push DB** step ends with `Finished database push.` and that the bucket holds `db.sql`. -S3 copies a dump of up to 5 GB in 1 request, so the push fails for a -larger dump. +S3 makes the timestamped copy on its side, so the dump is uploaded once. +S3 copies a dump of up to 5 GB in 1 request, so the timestamped copy +fails for a larger dump. On FTP, the timestamped copy is a second upload. ## Tell pushes apart -`VORTEX_PUSH_DB_FILE_PREFIX` adds a prefix to the names of the pushed -files, such as `gha-db.sql` and `gha-db-previous.sql`. Use a different -prefix in each place that pushes to the same store. A fetch reads the -prefixed files only when its remote filename includes the prefix. +`VORTEX_PUSH_DB_FILE_SUFFIX` adds a suffix before the extension of the +pushed files, such as `db_gha.sql` and `db_gha_20261111-180000.sql`. Use a +different suffix in each place that pushes to the same store. A fetch reads +the suffixed file only when its remote filename includes the suffix. ## What's next diff --git a/.vortex/tooling/playground/try-push-db-ftp.sh b/.vortex/tooling/playground/try-push-db-ftp.sh index dda38af769..1716701ae0 100755 --- a/.vortex/tooling/playground/try-push-db-ftp.sh +++ b/.vortex/tooling/playground/try-push-db-ftp.sh @@ -3,8 +3,8 @@ # Manual test script for FTP database push. # # Uploads a database dump file to an FTP server using push-db-ftp. Run it -# twice to check that the second run keeps the first dump as -# 'db-previous.sql' next to the new 'db.sql'. +# twice to check that the second run replaces 'db.sql'. Set +# VORTEX_PUSH_DB_TIMESTAMPED_COPY=1 to also upload a timestamped copy. # # Usage: # export FTP_HOST="ftp.example.com" diff --git a/.vortex/tooling/src/vortex-push-db-ftp b/.vortex/tooling/src/vortex-push-db-ftp index 2979a1e1ab..f2abb91e23 100755 --- a/.vortex/tooling/src/vortex-push-db-ftp +++ b/.vortex/tooling/src/vortex-push-db-ftp @@ -2,13 +2,14 @@ ## # Push DB dump to FTP. # -# Keeps 2 copies of the database dump file on the FTP server: the current -# dump under the remote file name and the dump it replaced with a '-previous' -# suffix, such as 'db.sql' and 'db-previous.sql'. The fetch always finds the -# current dump under the remote file name. +# Uploads the database dump file under the remote file name, such as +# 'db.sql', which always holds the latest dump that the fetch downloads. With +# VORTEX_PUSH_DB_TIMESTAMPED_COPY set to 1, the dump is also uploaded under a +# name with a UTC timestamp, such as 'db_20261111-180000.sql', to keep a +# history. # -# The dump is uploaded under a temporary name and renamed into place only -# after the upload succeeds, so a failed upload keeps the current dump. +# Each file is uploaded under a temporary name and renamed into place only +# after the upload succeeds, so a failed upload keeps the existing file. # # IMPORTANT! This script runs outside the container on the host system. # @@ -64,12 +65,18 @@ _vs="VORTEX_FETCH_DB${_db_index}_FILE" _vss="VORTEX_DB${_db_index}_FILE" VORTEX_PUSH_DB_FTP_DB_FILE="${!_v:-${!_vs:-${!_vss:-${VORTEX_DB_FILE:-db.sql}}}}" -# Prefix for the names of the pushed database dump files. +# Suffix for the names of the pushed database dump files. # -# Use a different prefix in each place that pushes to the same storage, such -# as 'gha-' and 'circleci-', to tell their dumps apart. A fetch reads the -# prefixed name only when its remote file name includes the prefix. -VORTEX_PUSH_DB_FILE_PREFIX="${VORTEX_PUSH_DB_FILE_PREFIX:-}" +# Goes before the extension, so 'db.sql' becomes 'db_gha.sql' with '_gha'. +# Use a different suffix in each place that pushes to the same storage. A +# fetch reads the suffixed name only when its remote file name includes it. +VORTEX_PUSH_DB_FILE_SUFFIX="${VORTEX_PUSH_DB_FILE_SUFFIX:-}" + +# Also push a copy of the database dump file named with a UTC timestamp. +# +# Set to 1 to keep a history of the pushed dumps. The fetch never reads the +# timestamped copies. +VORTEX_PUSH_DB_TIMESTAMPED_COPY="${VORTEX_PUSH_DB_TIMESTAMPED_COPY:-}" # ------------------------------------------------------------------------------ @@ -87,6 +94,22 @@ curl_config_quote() { printf '"%s"' "${value//\"/\\\"}" } +# Upload the database dump file to a name in the remote directory. +# +# The credentials go to curl on stdin, so they never appear in the process +# list. The commands after the upload run in the remote directory: the +# existing file is removed and the upload takes its name. +ftp_upload() { + local name="${1}" + local upload_name="${name}.uploading" + + echo "user = $(curl_config_quote "${VORTEX_PUSH_DB_FTP_USER}:${VORTEX_PUSH_DB_FTP_PASS}")" | curl --silent --show-error --config - --ftp-create-dirs --upload-file "${local_file}" \ + --quote "-*DELE ${name}" \ + --quote "-RNFR ${upload_name}" \ + --quote "-RNTO ${name}" \ + "ftp://${VORTEX_PUSH_DB_FTP_HOST}:${VORTEX_PUSH_DB_FTP_PORT}/${remote_dir}${upload_name}" +} + # shellcheck disable=SC2043 for cmd in curl; do command -v "${cmd}" >/dev/null || fail "Command ${cmd} is not available."; done @@ -105,36 +128,31 @@ if [[ ${remote_name} == */* ]]; then remote_name="${remote_name##*/}" fi -current_name="${VORTEX_PUSH_DB_FILE_PREFIX}${remote_name}" +base="${remote_name}" +ext="" if [[ ${remote_name} == *.* ]]; then - previous_name="${VORTEX_PUSH_DB_FILE_PREFIX}${remote_name%.*}-previous.${remote_name##*.}" -else - previous_name="${VORTEX_PUSH_DB_FILE_PREFIX}${remote_name}-previous" + base="${remote_name%.*}" + ext=".${remote_name##*.}" fi -upload_name="${current_name}.uploading" +current_name="${base}${VORTEX_PUSH_DB_FILE_SUFFIX}${ext}" +timestamped_name="${base}${VORTEX_PUSH_DB_FILE_SUFFIX}_$(date -u +%Y%m%d-%H%M%S)${ext}" info "Started database${_db_index:+ ${_db_index}} dump push to FTP." -note "Local file: ${local_file}" -note "Remote file: ${remote_dir}${current_name}" -note "Previous file: ${remote_dir}${previous_name}" -note "FTP host: ${VORTEX_PUSH_DB_FTP_HOST}:${VORTEX_PUSH_DB_FTP_PORT}" +note "Local file: ${local_file}" +note "Remote file: ${remote_dir}${current_name}" +note "FTP host: ${VORTEX_PUSH_DB_FTP_HOST}:${VORTEX_PUSH_DB_FTP_PORT}" -# The credentials go to curl on stdin, so they never appear in the process -# list. The commands after the upload run in the remote directory: the -# previous dump is replaced by the current one, and the upload takes the -# current name. Only the final rename must succeed, since the first push has -# no current or previous dump. -task "Uploading the database dump file and keeping the dump it replaces." -curl_config="user = $(curl_config_quote "${VORTEX_PUSH_DB_FTP_USER}:${VORTEX_PUSH_DB_FTP_PASS}")" -echo "${curl_config}" | curl --silent --show-error --config - --ftp-create-dirs --upload-file "${local_file}" \ - --quote "-*DELE ${previous_name}" \ - --quote "-*RNFR ${current_name}" \ - --quote "-*RNTO ${previous_name}" \ - --quote "-RNFR ${upload_name}" \ - --quote "-RNTO ${current_name}" \ - "ftp://${VORTEX_PUSH_DB_FTP_HOST}:${VORTEX_PUSH_DB_FTP_PORT}/${remote_dir}${upload_name}" || fail "Unable to push the database dump file to FTP." -pass "Uploaded the database dump file and kept the dump it replaced." +task "Uploading the database dump file." +ftp_upload "${current_name}" || fail "Unable to push the database dump file to FTP." +pass "Uploaded the database dump file." + +if [ "${VORTEX_PUSH_DB_TIMESTAMPED_COPY}" = "1" ]; then + # FTP has no copy command, so the copy is a second upload. + task "Uploading the timestamped copy ${remote_dir}${timestamped_name}." + ftp_upload "${timestamped_name}" || fail "Unable to push the timestamped copy of the database dump file to FTP." + pass "Uploaded the timestamped copy ${remote_dir}${timestamped_name}." +fi # Enable xtrace only after the request so the credentials are never written # to the trace output. diff --git a/.vortex/tooling/src/vortex-push-db-s3 b/.vortex/tooling/src/vortex-push-db-s3 index 4463c14f4f..c58857fa4a 100755 --- a/.vortex/tooling/src/vortex-push-db-s3 +++ b/.vortex/tooling/src/vortex-push-db-s3 @@ -4,13 +4,11 @@ # # Uses AWS Signature Version 4 with curl (no AWS CLI required). # -# Keeps 2 copies of the database dump file in the bucket: the current dump -# under the remote file name and the dump it replaced with a '-previous' -# suffix, such as 'db.sql' and 'db-previous.sql'. The fetch always finds the -# current dump under the remote file name. -# -# The current dump is copied to the previous one on the S3 side before the -# upload, so a failed upload keeps the current dump. +# Uploads the database dump file under the remote file name, such as +# 'db.sql', which always holds the latest dump that the fetch downloads. With +# VORTEX_PUSH_DB_TIMESTAMPED_COPY set to 1, the uploaded object is also copied +# to a name with a UTC timestamp, such as 'db_20261111-180000.sql', to keep a +# history. # # IMPORTANT! This script runs outside the container on the host system. # @@ -77,12 +75,18 @@ VORTEX_PUSH_DB_S3_REMOTE_FILE="${!_v:-${!_vs:-${VORTEX_PUSH_DB_S3_DB_FILE}}}" # S3 storage class. VORTEX_PUSH_DB_S3_STORAGE_CLASS="${VORTEX_PUSH_DB_S3_STORAGE_CLASS:-STANDARD}" -# Prefix for the names of the pushed database dump files. +# Suffix for the names of the pushed database dump files. +# +# Goes before the extension, so 'db.sql' becomes 'db_gha.sql' with '_gha'. +# Use a different suffix in each place that pushes to the same storage. A +# fetch reads the suffixed name only when its remote file name includes it. +VORTEX_PUSH_DB_FILE_SUFFIX="${VORTEX_PUSH_DB_FILE_SUFFIX:-}" + +# Also push a copy of the database dump file named with a UTC timestamp. # -# Use a different prefix in each place that pushes to the same storage, such -# as 'gha-' and 'circleci-', to tell their dumps apart. A fetch reads the -# prefixed name only when its remote file name includes the prefix. -VORTEX_PUSH_DB_FILE_PREFIX="${VORTEX_PUSH_DB_FILE_PREFIX:-}" +# Set to 1 to keep a history of the pushed dumps. The fetch never reads the +# timestamped copies. +VORTEX_PUSH_DB_TIMESTAMPED_COPY="${VORTEX_PUSH_DB_TIMESTAMPED_COPY:-}" # ------------------------------------------------------------------------------ @@ -198,8 +202,8 @@ info "Started database${_db_index:+ ${_db_index}} dump push to S3." [ -n "${VORTEX_PUSH_DB_S3_PREFIX}" ] && VORTEX_PUSH_DB_S3_PREFIX="${VORTEX_PUSH_DB_S3_PREFIX%/}/" -# The file name prefix and the '-previous' suffix apply to the file name, not -# to the directories in the remote file name. +# The suffix and the timestamp apply to the file name, not to the directories +# in the remote file name. remote_dir="" remote_name="${VORTEX_PUSH_DB_S3_REMOTE_FILE}" if [[ ${remote_name} == */* ]]; then @@ -207,46 +211,25 @@ if [[ ${remote_name} == */* ]]; then remote_name="${remote_name##*/}" fi -current_file="${remote_dir}${VORTEX_PUSH_DB_FILE_PREFIX}${remote_name}" +base="${remote_name}" +ext="" if [[ ${remote_name} == *.* ]]; then - previous_file="${remote_dir}${VORTEX_PUSH_DB_FILE_PREFIX}${remote_name%.*}-previous.${remote_name##*.}" -else - previous_file="${remote_dir}${VORTEX_PUSH_DB_FILE_PREFIX}${remote_name}-previous" + base="${remote_name%.*}" + ext=".${remote_name##*.}" fi +current_file="${remote_dir}${base}${VORTEX_PUSH_DB_FILE_SUFFIX}${ext}" +timestamped_file="${remote_dir}${base}${VORTEX_PUSH_DB_FILE_SUFFIX}_$(date -u +%Y%m%d-%H%M%S)${ext}" object_key="$(uri_encode_key "${VORTEX_PUSH_DB_S3_PREFIX}${current_file}")" -previous_key="$(uri_encode_key "${VORTEX_PUSH_DB_S3_PREFIX}${previous_file}")" +timestamped_key="$(uri_encode_key "${VORTEX_PUSH_DB_S3_PREFIX}${timestamped_file}")" note "Local file: ${local_file}" note "Remote file: ${VORTEX_PUSH_DB_S3_PREFIX}${current_file}" -note "Previous file: ${VORTEX_PUSH_DB_S3_PREFIX}${previous_file}" note "S3 bucket: ${VORTEX_PUSH_DB_S3_BUCKET}" note "S3 region: ${VORTEX_PUSH_DB_S3_REGION}" [ -n "${VORTEX_PUSH_DB_S3_PREFIX}" ] && note "S3 prefix: ${VORTEX_PUSH_DB_S3_PREFIX}" note "Storage class: ${VORTEX_PUSH_DB_S3_STORAGE_CLASS}" -# A copy can fail after S3 has sent a 200 status, so a success needs the copy -# result in the body. A missing current dump means this is the first push. -task "Keeping the current database dump as the previous one." -set +e -response=$(s3_put "${previous_key}" "$(hash_sha256 "")" "" "x-amz-copy-source:/${VORTEX_PUSH_DB_S3_BUCKET}/${object_key}") -curl_status=$? -set -e - -[ "${curl_status}" -ne 0 ] && fail "S3 copy request failed with curl exit ${curl_status}." - -http_code=$(echo "${response}" | tail -1) -response_body=$(echo "${response}" | sed '$d') - -if [[ ${http_code} == 2* ]] && [[ ${response_body} == *"/dev/null; then content_type="$(file --brief --mime-type "${local_file}")" else @@ -261,10 +244,6 @@ response=$(s3_put "${object_key}" "${payload_hash}" "${local_file}" "content-typ curl_status=$? set -e -# Enable xtrace only after the signed requests so credentials and the -# signatures are never written to the trace output. -[ "${VORTEX_DEBUG-}" = "1" ] && set -x - [ "${curl_status}" -ne 0 ] && fail "S3 push request failed with curl exit ${curl_status}." http_code=$(echo "${response}" | tail -1) @@ -279,4 +258,30 @@ case "${http_code}" in esac pass "Uploaded the database dump file." +if [ "${VORTEX_PUSH_DB_TIMESTAMPED_COPY}" = "1" ]; then + # The copy runs on the S3 side, so the dump is uploaded only once. A copy + # can fail after S3 has sent a 200 status, so a success needs the copy + # result in the body. + task "Copying the database dump file to ${VORTEX_PUSH_DB_S3_PREFIX}${timestamped_file}." + set +e + response=$(s3_put "${timestamped_key}" "$(hash_sha256 "")" "" "x-amz-copy-source:/${VORTEX_PUSH_DB_S3_BUCKET}/${object_key}") + curl_status=$? + set -e + + [ "${curl_status}" -ne 0 ] && fail "S3 copy request failed with curl exit ${curl_status}." + + http_code=$(echo "${response}" | tail -1) + response_body=$(echo "${response}" | sed '$d') + + if [[ ${http_code} != 2* ]] || [[ ${response_body} != *"/dev/null || exit 1 mkdir -p .data @@ -30,11 +30,11 @@ setup_ftp_fetch_values() { run .vortex/tooling/src/vortex-push-db-ftp assert_success assert_output_contains "[INFO] Started database dump push to FTP." - assert_output_contains "Local file: ./.data/db.sql" - assert_output_contains "Remote file: db.sql" - assert_output_contains "Previous file: db-previous.sql" - assert_output_contains "FTP host: ftp.example.com:21" - assert_output_contains "[ OK ] Uploaded the database dump file and kept the dump it replaced." + assert_output_contains "Local file: ./.data/db.sql" + assert_output_contains "Remote file: db.sql" + assert_output_contains "FTP host: ftp.example.com:21" + assert_output_contains "[ OK ] Uploaded the database dump file." + assert_output_not_contains "timestamped copy" assert_output_contains "[ OK ] Finished database dump push to FTP." assert_output_not_contains "fetch-pass" @@ -42,8 +42,9 @@ setup_ftp_fetch_values() { args="$(mock_get_call_args "${mock_curl}" 1)" assert_string_contains "${args}" "--config - --ftp-create-dirs --upload-file ./.data/db.sql" - # The upload takes a temporary name, and the rename into place comes last. - assert_string_contains "${args}" "--quote -*DELE db-previous.sql --quote -*RNFR db.sql --quote -*RNTO db-previous.sql --quote -RNFR db.sql.uploading --quote -RNTO db.sql" + # The upload takes a temporary name and replaces the latest dump only after + # it succeeds. + assert_string_contains "${args}" "--quote -*DELE db.sql --quote -RNFR db.sql.uploading --quote -RNTO db.sql" assert_string_contains "${args}" "ftp://ftp.example.com:21/db.sql.uploading" # The credentials reach curl on stdin, never as arguments. assert_string_not_contains "${args}" "fetch-pass" @@ -54,7 +55,38 @@ setup_ftp_fetch_values() { popd >/dev/null } -@test "push-db-ftp: Push to a remote directory with a file name prefix" { +@test "push-db-ftp: Upload a timestamped copy when enabled" { + pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 + + mkdir -p .data + echo "database content" >.data/db.sql + + setup_ftp_fetch_values + export VORTEX_PUSH_DB_TIMESTAMPED_COPY=1 + + mock_curl=$(mock_command "curl") + + run .vortex/tooling/src/vortex-push-db-ftp + assert_success + assert_output_contains "[ OK ] Uploaded the database dump file." + assert_output_contains "[ OK ] Uploaded the timestamped copy db_" + assert_output_contains "[ OK ] Finished database dump push to FTP." + + assert_equal "2" "$(mock_get_call_num "${mock_curl}")" + + # The latest dump is uploaded first, then the copy under its own name. + assert_string_contains "$(mock_get_call_args "${mock_curl}" 1)" "ftp://ftp.example.com:21/db.sql.uploading" + copy_args="$(mock_get_call_args "${mock_curl}" 2)" + assert_string_contains "${copy_args}" "--upload-file ./.data/db.sql" + [[ ${copy_args} =~ --quote\ -RNTO\ db_[0-9]{8}-[0-9]{6}\.sql ]] + [[ ${copy_args} =~ ftp://ftp\.example\.com:21/db_[0-9]{8}-[0-9]{6}\.sql\.uploading ]] + + rm -rf .data + + popd >/dev/null +} + +@test "push-db-ftp: Push to a remote directory with a file name suffix" { pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 mkdir -p .data @@ -64,20 +96,22 @@ setup_ftp_fetch_values() { export VORTEX_PUSH_DB_FTP_HOST="push.example.com" export VORTEX_PUSH_DB_FTP_PORT="2121" export VORTEX_PUSH_DB_FTP_FILE="/backups/site/db.sql" - export VORTEX_PUSH_DB_FILE_PREFIX="gha-" + export VORTEX_PUSH_DB_FILE_SUFFIX="_gha" + export VORTEX_PUSH_DB_TIMESTAMPED_COPY=1 mock_curl=$(mock_command "curl") run .vortex/tooling/src/vortex-push-db-ftp assert_success - assert_output_contains "Remote file: backups/site/gha-db.sql" - assert_output_contains "Previous file: backups/site/gha-db-previous.sql" - assert_output_contains "FTP host: push.example.com:2121" + assert_output_contains "Remote file: backups/site/db_gha.sql" + assert_output_contains "FTP host: push.example.com:2121" + assert_output_contains "[ OK ] Uploaded the timestamped copy backups/site/db_gha_" - args="$(mock_get_call_args "${mock_curl}" 1)" # The rename commands run in the remote directory, so they use bare names. - assert_string_contains "${args}" "--quote -*DELE gha-db-previous.sql --quote -*RNFR gha-db.sql --quote -*RNTO gha-db-previous.sql --quote -RNFR gha-db.sql.uploading --quote -RNTO gha-db.sql" - assert_string_contains "${args}" "ftp://push.example.com:2121/backups/site/gha-db.sql.uploading" + args="$(mock_get_call_args "${mock_curl}" 1)" + assert_string_contains "${args}" "--quote -*DELE db_gha.sql --quote -RNFR db_gha.sql.uploading --quote -RNTO db_gha.sql" + assert_string_contains "${args}" "ftp://push.example.com:2121/backups/site/db_gha.sql.uploading" + [[ $(mock_get_call_args "${mock_curl}" 2) =~ ftp://push\.example\.com:2121/backups/site/db_gha_[0-9]{8}-[0-9]{6}\.sql\.uploading ]] rm -rf .data @@ -105,9 +139,8 @@ setup_ftp_fetch_values() { run .vortex/tooling/src/vortex-push-db-ftp assert_success assert_output_contains "[INFO] Started database 2 dump push to FTP." - assert_output_contains "Local file: ./.data/db2.sql" - assert_output_contains "Remote file: db2.sql" - assert_output_contains "Previous file: db2-previous.sql" + assert_output_contains "Local file: ./.data/db2.sql" + assert_output_contains "Remote file: db2.sql" assert_output_contains "[ OK ] Finished database 2 dump push to FTP." assert_string_contains "$(mock_get_call_args "${mock_curl}" 1)" "ftp://ftp2.example.com:21/db2.sql.uploading" @@ -182,13 +215,14 @@ setup_ftp_fetch_values() { popd >/dev/null } -@test "push-db-ftp: Fail when the upload or a rename fails" { +@test "push-db-ftp: Fail when the upload fails" { pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 mkdir -p .data echo "database content" >.data/db.sql setup_ftp_fetch_values + export VORTEX_PUSH_DB_TIMESTAMPED_COPY=1 mock_curl=$(mock_command "curl") mock_set_status "${mock_curl}" 21 1 @@ -196,7 +230,31 @@ setup_ftp_fetch_values() { run .vortex/tooling/src/vortex-push-db-ftp assert_failure assert_output_contains "[FAIL] Unable to push the database dump file to FTP." - assert_output_not_contains "Finished database dump push to FTP." + assert_output_not_contains "Uploaded the timestamped copy" + assert_equal "1" "$(mock_get_call_num "${mock_curl}")" + + rm -rf .data + + popd >/dev/null +} + +@test "push-db-ftp: Fail when the timestamped copy upload fails" { + pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 + + mkdir -p .data + echo "database content" >.data/db.sql + + setup_ftp_fetch_values + export VORTEX_PUSH_DB_TIMESTAMPED_COPY=1 + + mock_curl=$(mock_command "curl") + mock_set_status "${mock_curl}" 0 1 + mock_set_status "${mock_curl}" 25 2 + + run .vortex/tooling/src/vortex-push-db-ftp + assert_failure + assert_output_contains "[ OK ] Uploaded the database dump file." + assert_output_contains "[FAIL] Unable to push the timestamped copy of the database dump file to FTP." rm -rf .data diff --git a/.vortex/tooling/tests/unit/push-db-s3.bats b/.vortex/tooling/tests/unit/push-db-s3.bats index b29fd798a1..f9c066e57f 100644 --- a/.vortex/tooling/tests/unit/push-db-s3.bats +++ b/.vortex/tooling/tests/unit/push-db-s3.bats @@ -21,7 +21,6 @@ load ../_helper.bash "S3 region: ap-southeast-2" "Storage class: STANDARD" - "@curl * # 0 # \n200" "@curl * # 0 # 200" "[ OK ] Finished database dump push to S3." @@ -62,7 +61,6 @@ load ../_helper.bash "Local file: ./.data/db.sql" "Remote file: db.sql" - "@curl * # 0 # \n200" "@curl * # 0 # 200" "[ OK ] Finished database dump push to S3." @@ -102,7 +100,6 @@ load ../_helper.bash "S3 bucket: shortcut-bucket" "S3 region: us-east-1" - "@curl * # 0 # \n200" "@curl * # 0 # 200" "[ OK ] Finished database dump push to S3." @@ -146,7 +143,6 @@ load ../_helper.bash "Remote file: backup/db_latest.sql" - "@curl * # 0 # \n200" "@curl * # 0 # 200" "[ OK ] Finished database dump push to S3." @@ -187,7 +183,6 @@ load ../_helper.bash "Remote file: backups/daily/db.sql" "S3 prefix: backups/daily/" - "@curl * # 0 # \n200" "@curl * # 0 # 200" "[ OK ] Finished database dump push to S3." @@ -228,7 +223,6 @@ load ../_helper.bash "Remote file: backups/daily/db.sql" "S3 prefix: backups/daily/" - "@curl * # 0 # \n200" "@curl * # 0 # 200" "[ OK ] Finished database dump push to S3." @@ -267,7 +261,6 @@ load ../_helper.bash declare -a STEPS=( "[INFO] Started database dump push to S3." - "@curl * # 0 # \n200" "@curl * # 0 # AccessDenied\n403" "[FAIL] Unable to push to S3: HTTP status 403." @@ -446,7 +439,6 @@ load ../_helper.bash "S3 region: eu-west-1" "S3 prefix: store/" - "@curl * # 0 # \n200" "@curl * # 0 # 200" "[ OK ] Finished database dump push to S3." @@ -492,7 +484,6 @@ load ../_helper.bash "Local file: ./.data/db_fetched.sql" "Remote file: db_fetched.sql" - "@curl * # 0 # \n200" "@curl * # 0 # 200" "[ OK ] Finished database dump push to S3." @@ -535,36 +526,27 @@ setup_s3_push() { export VORTEX_PUSH_DB_S3_PREFIX="backups" export VORTEX_PUSH_DB_S3_DB_DIR=".data" export VORTEX_PUSH_DB_S3_DB_FILE="db.sql" - unset VORTEX_PUSH_DB_S3_REMOTE_FILE VORTEX_PUSH_DB_FILE_PREFIX + unset VORTEX_PUSH_DB_S3_REMOTE_FILE VORTEX_PUSH_DB_FILE_SUFFIX VORTEX_PUSH_DB_TIMESTAMPED_COPY } -@test "push-db-s3: Keep the current dump as the previous one before the upload" { +@test "push-db-s3: Push only the latest dump by default" { pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 setup_s3_push mock_curl=$(mock_command "curl") - mock_set_output "${mock_curl}" $'\n200' 1 - mock_set_output "${mock_curl}" "200" 2 + mock_set_output "${mock_curl}" "200" 1 run .vortex/tooling/src/vortex-push-db-s3 assert_success assert_output_contains "Remote file: backups/db.sql" - assert_output_contains "Previous file: backups/db-previous.sql" - assert_output_contains "[ OK ] Kept the current database dump as the previous one." assert_output_contains "[ OK ] Uploaded the database dump file." + assert_output_not_contains "Copying the database dump file" assert_output_contains "[ OK ] Finished database dump push to S3." - assert_equal "2" "$(mock_get_call_num "${mock_curl}")" + assert_equal "1" "$(mock_get_call_num "${mock_curl}")" - # The copy names the current dump as its source and sends no body. - copy_args="$(mock_get_call_args "${mock_curl}" 1)" - assert_string_contains "${copy_args}" "--request PUT" - assert_string_contains "${copy_args}" "--header x-amz-copy-source: /test-bucket/backups/db.sql" - assert_string_not_contains "${copy_args}" "--upload-file" - assert_string_contains "${copy_args}" "https://test-bucket.s3.ap-southeast-2.amazonaws.com/backups/db-previous.sql" - - upload_args="$(mock_get_call_args "${mock_curl}" 2)" + upload_args="$(mock_get_call_args "${mock_curl}" 1)" assert_string_contains "${upload_args}" "--upload-file .data/db.sql" assert_string_contains "${upload_args}" "https://test-bucket.s3.ap-southeast-2.amazonaws.com/backups/db.sql" @@ -573,52 +555,67 @@ setup_s3_push() { popd >/dev/null } -@test "push-db-s3: Upload the first dump when there is no current one" { +@test "push-db-s3: Copy the upload to a timestamped copy when enabled" { pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 setup_s3_push + export VORTEX_PUSH_DB_TIMESTAMPED_COPY=1 mock_curl=$(mock_command "curl") - mock_set_output "${mock_curl}" $'NoSuchKey\n404' 1 - mock_set_output "${mock_curl}" "200" 2 + mock_set_output "${mock_curl}" "200" 1 + mock_set_output "${mock_curl}" $'\n200' 2 run .vortex/tooling/src/vortex-push-db-s3 assert_success - assert_output_contains "[ OK ] Skipped keeping the previous database dump as there is no current one." + assert_output_contains "[ OK ] Uploaded the database dump file." + assert_output_contains "[ OK ] Copied the database dump file to backups/db_" assert_output_contains "[ OK ] Finished database dump push to S3." + assert_equal "2" "$(mock_get_call_num "${mock_curl}")" + # The copy runs after the upload, names the latest dump as its source and + # sends no body. + assert_string_contains "$(mock_get_call_args "${mock_curl}" 1)" "https://test-bucket.s3.ap-southeast-2.amazonaws.com/backups/db.sql" + copy_args="$(mock_get_call_args "${mock_curl}" 2)" + assert_string_contains "${copy_args}" "--header x-amz-copy-source: /test-bucket/backups/db.sql" + assert_string_not_contains "${copy_args}" "--upload-file" + [[ ${copy_args} =~ https://test-bucket\.s3\.ap-southeast-2\.amazonaws\.com/backups/db_[0-9]{8}-[0-9]{6}\.sql ]] + rm -rf .data popd >/dev/null } -@test "push-db-s3: Fail without uploading when the current dump cannot be kept" { +@test "push-db-s3: Fail when the timestamped copy cannot be made" { pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 setup_s3_push + export VORTEX_PUSH_DB_TIMESTAMPED_COPY=1 - # A copy error can come with a 200 status. Each run makes 1 request. + # A copy error can come with a 200 status. Each run makes 2 requests. mock_curl=$(mock_command "curl") - mock_set_output "${mock_curl}" $'InternalError\n200' 1 - mock_set_output "${mock_curl}" $'AccessDenied\n403' 2 + mock_set_output "${mock_curl}" "200" 1 + mock_set_output "${mock_curl}" $'InternalError\n200' 2 + mock_set_output "${mock_curl}" "200" 3 + mock_set_output "${mock_curl}" $'AccessDenied\n403' 4 for code in 200 403; do run .vortex/tooling/src/vortex-push-db-s3 assert_failure + assert_output_contains "[ OK ] Uploaded the database dump file." assert_output_contains "Response: " - assert_output_contains "[FAIL] Unable to keep the current database dump as the previous one: HTTP status ${code}." - assert_output_not_contains "Uploaded the database dump file." + assert_output_contains "[FAIL] Unable to copy the database dump file to the timestamped copy: HTTP status ${code}." + assert_output_not_contains "Finished database dump push to S3." done - assert_equal "2" "$(mock_get_call_num "${mock_curl}")" + assert_equal "4" "$(mock_get_call_num "${mock_curl}")" rm -rf .data popd >/dev/null } -@test "push-db-s3: Push an indexed database with a file name prefix" { +@test "push-db-s3: Push an indexed database with a file name suffix" { pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 mkdir -p .data @@ -632,23 +629,24 @@ setup_s3_push() { export VORTEX_FETCH_DB2_S3_REGION="eu-west-1" export VORTEX_FETCH_DB2_S3_PREFIX="" export VORTEX_FETCH_DB2_FILE="db2.sql" - export VORTEX_PUSH_DB_FILE_PREFIX="circleci-" + export VORTEX_PUSH_DB_FILE_SUFFIX="_circleci" + export VORTEX_PUSH_DB_TIMESTAMPED_COPY=1 mock_curl=$(mock_command "curl") - mock_set_output "${mock_curl}" $'\n200' 1 - mock_set_output "${mock_curl}" "200" 2 + mock_set_output "${mock_curl}" "200" 1 + mock_set_output "${mock_curl}" $'\n200' 2 run .vortex/tooling/src/vortex-push-db-s3 assert_success assert_output_contains "[INFO] Started database 2 dump push to S3." assert_output_contains "Local file: ./.data/db2.sql" - assert_output_contains "Remote file: circleci-db2.sql" - assert_output_contains "Previous file: circleci-db2-previous.sql" + assert_output_contains "Remote file: db2_circleci.sql" assert_output_contains "S3 bucket: db2-bucket" + assert_output_contains "[ OK ] Copied the database dump file to db2_circleci_" assert_output_contains "[ OK ] Finished database 2 dump push to S3." - assert_string_contains "$(mock_get_call_args "${mock_curl}" 1)" "--header x-amz-copy-source: /db2-bucket/circleci-db2.sql" - assert_string_contains "$(mock_get_call_args "${mock_curl}" 2)" "https://db2-bucket.s3.eu-west-1.amazonaws.com/circleci-db2.sql" + assert_string_contains "$(mock_get_call_args "${mock_curl}" 1)" "https://db2-bucket.s3.eu-west-1.amazonaws.com/db2_circleci.sql" + assert_string_contains "$(mock_get_call_args "${mock_curl}" 2)" "--header x-amz-copy-source: /db2-bucket/db2_circleci.sql" rm -rf .data From 4d6795427791f0eda57ee7f79d18b4430f2e53ee Mon Sep 17 00:00:00 2001 From: Alex Skrypnyk Date: Wed, 11 Nov 2026 15:17:50 +1100 Subject: [PATCH 16/21] Updated the database storage docs and the generated CI configs for the timestamped copy. --- .github/workflows/vortex-test-didi.yml | 12 ++++++++---- .../docs/content/development/database-storage.mdx | 5 +++-- .vortex/docs/content/development/variables.mdx | 3 ++- 3 files changed, 13 insertions(+), 7 deletions(-) diff --git a/.github/workflows/vortex-test-didi.yml b/.github/workflows/vortex-test-didi.yml index 0741270dd3..f2a7401592 100644 --- a/.github/workflows/vortex-test-didi.yml +++ b/.github/workflows/vortex-test-didi.yml @@ -82,13 +82,15 @@ jobs: #;> MIGRATION_DB_FETCH_SOURCE_S3 #;< VORTEX_DEV # FTP server that receives the demo database on the nightly run, under - # a prefix that tells this provider's dumps apart. + # a suffix that tells this provider's dumps apart, with a timestamped + # copy of each push. VORTEX_PUSH_DB_FTP_HOST: ${{ secrets.VORTEX_PUSH_DB_FTP_HOST }} VORTEX_PUSH_DB_FTP_PORT: ${{ secrets.VORTEX_PUSH_DB_FTP_PORT }} VORTEX_PUSH_DB_FTP_USER: ${{ secrets.VORTEX_PUSH_DB_FTP_USER }} VORTEX_PUSH_DB_FTP_PASS: ${{ secrets.VORTEX_PUSH_DB_FTP_PASS }} VORTEX_PUSH_DB_FTP_FILE: ${{ vars.VORTEX_PUSH_DB_FTP_FILE || 'db.sql' }} - VORTEX_PUSH_DB_FILE_PREFIX: gha- + VORTEX_PUSH_DB_FILE_SUFFIX: _gha + VORTEX_PUSH_DB_TIMESTAMPED_COPY: 1 #;> VORTEX_DEV VORTEX_FETCH_DB_SOURCE: url VORTEX_FETCH_DB_FORCE: 1 @@ -640,13 +642,15 @@ jobs: #;> MIGRATION_DB_FETCH_SOURCE_S3 #;< VORTEX_DEV # FTP server that receives the demo database on the nightly run, under - # a prefix that tells this provider's dumps apart. + # a suffix that tells this provider's dumps apart, with a timestamped + # copy of each push. VORTEX_PUSH_DB_FTP_HOST: ${{ secrets.VORTEX_PUSH_DB_FTP_HOST }} VORTEX_PUSH_DB_FTP_PORT: ${{ secrets.VORTEX_PUSH_DB_FTP_PORT }} VORTEX_PUSH_DB_FTP_USER: ${{ secrets.VORTEX_PUSH_DB_FTP_USER }} VORTEX_PUSH_DB_FTP_PASS: ${{ secrets.VORTEX_PUSH_DB_FTP_PASS }} VORTEX_PUSH_DB_FTP_FILE: ${{ vars.VORTEX_PUSH_DB_FTP_FILE || 'db.sql' }} - VORTEX_PUSH_DB_FILE_PREFIX: gha- + VORTEX_PUSH_DB_FILE_SUFFIX: _gha + VORTEX_PUSH_DB_TIMESTAMPED_COPY: 1 #;> VORTEX_DEV VORTEX_FETCH_DB_SOURCE: container_registry VORTEX_FETCH_DB_FORCE: 1 diff --git a/.vortex/docs/content/development/database-storage.mdx b/.vortex/docs/content/development/database-storage.mdx index 0780d95a3f..b4f9044238 100644 --- a/.vortex/docs/content/development/database-storage.mdx +++ b/.vortex/docs/content/development/database-storage.mdx @@ -123,8 +123,9 @@ To make the migration source a dump store, set Before you begin, create an FTP account that can write to a directory on the server. The push uploads the dump under a temporary name, then renames -it into place, so a failed upload keeps the latest dump. The server must allow the `DELE`, `RNFR` and `RNTO` -commands, which standard FTP servers do. +it into place, so a failed upload keeps the latest dump. The server must +allow the `DELE`, `RNFR` and `RNTO` commands, which standard FTP servers +do. 1. In `.env`, set the fetch source and the server: diff --git a/.vortex/docs/content/development/variables.mdx b/.vortex/docs/content/development/variables.mdx index b36750e223..cade9da4c0 100644 --- a/.vortex/docs/content/development/variables.mdx +++ b/.vortex/docs/content/development/variables.mdx @@ -536,7 +536,7 @@ The list below is automatically generated with [Shellvar](https://github.com/ale | `VORTEX_PUSH_CONTAINER_REGISTRY_MAP` | Comma-separated map of container services and images to push in format "service1=org/image1,service2=org/image2". | `UNDEFINED` | `.vortex/tooling/src/vortex-push-container-registry`, `.vortex/tooling/src/vortex-push-db-image` | | `VORTEX_PUSH_CONTAINER_REGISTRY_PASS` | The password to login into the container registry. | `${VORTEX_CONTAINER_REGISTRY_PASS}` | `.vortex/tooling/src/vortex-push-container-registry` | | `VORTEX_PUSH_CONTAINER_REGISTRY_USER` | The username to login into the container registry. | `${VORTEX_CONTAINER_REGISTRY_USER}` | `.vortex/tooling/src/vortex-push-container-registry` | -| `VORTEX_PUSH_DB_FILE_PREFIX` | Prefix for the names of the pushed database dump files.

Use a different prefix in each place that pushes to the same storage, such as 'gha-' and 'circleci-', to tell their dumps apart. A fetch reads the prefixed name only when its remote file name includes the prefix. | `UNDEFINED` | `.vortex/tooling/src/vortex-push-db-ftp`, `.vortex/tooling/src/vortex-push-db-s3` | +| `VORTEX_PUSH_DB_FILE_SUFFIX` | Suffix for the names of the pushed database dump files.

Goes before the extension, so 'db.sql' becomes 'db_gha.sql' with '_gha'. Use a different suffix in each place that pushes to the same storage. A fetch reads the suffixed name only when its remote file name includes it. | `UNDEFINED` | `.vortex/tooling/src/vortex-push-db-ftp`, `.vortex/tooling/src/vortex-push-db-s3` | | `VORTEX_PUSH_DB_FTP_DB_DIR` | Directory with database dump file. | `./.data` | `.vortex/tooling/src/vortex-push-db-ftp` | | `VORTEX_PUSH_DB_FTP_DB_FILE` | Database dump file name.

Defaults to the name the database is fetched and refreshed with. | `db.sql` | `.vortex/tooling/src/vortex-push-db-ftp` | | `VORTEX_PUSH_DB_FTP_FILE` | The remote file name, including any directories. | `UNDEFINED` | `.vortex/tooling/src/vortex-push-db-ftp` | @@ -554,6 +554,7 @@ The list below is automatically generated with [Shellvar](https://github.com/ale | `VORTEX_PUSH_DB_S3_REMOTE_FILE` | Remote database dump file name.

Defaults to the remote file name the database is fetched from, then to the database dump file name. | `${VORTEX_PUSH_DB_S3_DB_FILE}` | `.vortex/tooling/src/vortex-push-db-s3` | | `VORTEX_PUSH_DB_S3_SECRET_KEY` | AWS secret key. | `${S3_SECRET_KEY}` | `.vortex/tooling/src/vortex-push-db-s3` | | `VORTEX_PUSH_DB_S3_STORAGE_CLASS` | S3 storage class. | `STANDARD` | `.vortex/tooling/src/vortex-push-db-s3` | +| `VORTEX_PUSH_DB_TIMESTAMPED_COPY` | Also push a copy of the database dump file named with a UTC timestamp.

Set to `1` to keep a history of the pushed dumps. The fetch never reads the timestamped copies. | `UNDEFINED` | `.vortex/tooling/src/vortex-push-db-ftp`, `.vortex/tooling/src/vortex-push-db-s3` | | `VORTEX_REFRESH_DB_PROCEED` | Proceed with the refresh of an indexed database.

The primary database is always refreshed. An indexed database, such as the migration database with VORTEX_DB_INDEX=`2`, is refreshed only when VORTEX_REFRESH_DB2_PROCEED is set to `1`. | `UNDEFINED` | `.vortex/tooling/src/vortex-refresh-db` | | `VORTEX_REFRESH_DB_SOURCE` | Upstream source of the pushed database dump file. | `UNDEFINED` | `.vortex/tooling/src/vortex-push-db`, `.vortex/tooling/src/vortex-refresh-db` | | `VORTEX_RELEASE_VERSION_SCHEME` | Versioning scheme used for releases.

Can be one of: calver, semver, other @see https://www.vortextemplate.com/docs/releasing | `calver` | `.env` | From 1ed342aa462a6aceb2abfc548ffede0021bf4406 Mon Sep 17 00:00:00 2001 From: Alex Skrypnyk Date: Wed, 11 Nov 2026 15:21:11 +1100 Subject: [PATCH 17/21] Updated snapshots. --- .../handler_process/db_fetch_source_ftp/.env | 15 ++++++++++----- .../handler_process/db_fetch_source_s3/.env | 15 ++++++++++----- 2 files changed, 20 insertions(+), 10 deletions(-) diff --git a/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_ftp/.env b/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_ftp/.env index a6c57df4f1..8d266c9412 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_ftp/.env +++ b/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_ftp/.env @@ -1,4 +1,4 @@ -@@ -180,12 +180,27 @@ +@@ -180,13 +180,32 @@ VORTEX_DB_FILE=db.sql # Database fetch source. @@ -23,14 +23,19 @@ -VORTEX_FETCH_DB_URL= +# Uncomment to make the database fetch source a dump store: the nightly CI +# job fetches the dump from this source, sanitizes it and pushes it to the -+# fetch source, keeping the dump it replaces as the previous one. The fetch -+# script of this source reads its own variables, such as VORTEX_FETCH_DB_URL -+# for url. Add them, and keep secrets in CI. ++# fetch source as the latest dump. The fetch script of this source reads its ++# own variables, such as VORTEX_FETCH_DB_URL for url. Add them, and keep ++# secrets in CI. +# VORTEX_REFRESH_DB_SOURCE=url ++# Also push a copy of each dump named with a UTC timestamp, such as ++# db_20261111-180000.sql, to keep a history of the pushed dumps. ++# VORTEX_PUSH_DB_TIMESTAMPED_COPY=1 ++ # Environment to fetch the database from. # -@@ -254,17 +269,3 @@ + # Applies to hosting environments. +@@ -254,17 +273,3 @@ # with optional names in the format "email|name". # Example: "to1@example.com|Jane Doe, to2@example.com|John Doe" VORTEX_NOTIFY_EMAIL_RECIPIENTS="webmaster@star-wars.com|Webmaster" diff --git a/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_s3/.env b/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_s3/.env index 63867d4b6e..aa442ec9c3 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_s3/.env +++ b/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_s3/.env @@ -1,4 +1,4 @@ -@@ -180,12 +180,24 @@ +@@ -180,13 +180,29 @@ VORTEX_DB_FILE=db.sql # Database fetch source. @@ -20,14 +20,19 @@ -VORTEX_FETCH_DB_URL= +# Uncomment to make the database fetch source a dump store: the nightly CI +# job fetches the dump from this source, sanitizes it and pushes it to the -+# fetch source, keeping the dump it replaces as the previous one. The fetch -+# script of this source reads its own variables, such as VORTEX_FETCH_DB_URL -+# for url. Add them, and keep secrets in CI. ++# fetch source as the latest dump. The fetch script of this source reads its ++# own variables, such as VORTEX_FETCH_DB_URL for url. Add them, and keep ++# secrets in CI. +# VORTEX_REFRESH_DB_SOURCE=url ++# Also push a copy of each dump named with a UTC timestamp, such as ++# db_20261111-180000.sql, to keep a history of the pushed dumps. ++# VORTEX_PUSH_DB_TIMESTAMPED_COPY=1 ++ # Environment to fetch the database from. # -@@ -254,17 +266,3 @@ + # Applies to hosting environments. +@@ -254,17 +270,3 @@ # with optional names in the format "email|name". # Example: "to1@example.com|Jane Doe, to2@example.com|John Doe" VORTEX_NOTIFY_EMAIL_RECIPIENTS="webmaster@star-wars.com|Webmaster" From ed124cfacb930c83933992810badb18d00f20a1b Mon Sep 17 00:00:00 2001 From: Alex Skrypnyk Date: Wed, 11 Nov 2026 15:52:16 +1100 Subject: [PATCH 18/21] Addressed code review: sent an empty length on the S3 copy request and read the fetch directory for the S3 push. --- .vortex/tooling/src/vortex-push-db-s3 | 9 +++++++-- .vortex/tooling/tests/unit/push-db-s3.bats | 8 +++++++- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/.vortex/tooling/src/vortex-push-db-s3 b/.vortex/tooling/src/vortex-push-db-s3 index c58857fa4a..0a9ce5ea95 100755 --- a/.vortex/tooling/src/vortex-push-db-s3 +++ b/.vortex/tooling/src/vortex-push-db-s3 @@ -53,8 +53,9 @@ VORTEX_PUSH_DB_S3_PREFIX="${!_v:-${!_vs:-${S3_PREFIX:-}}}" # Directory with database dump file. _v="VORTEX_PUSH_DB${_db_index}_S3_DB_DIR" -_vs="VORTEX_DB${_db_index}_DIR" -VORTEX_PUSH_DB_S3_DB_DIR="${!_v:-${!_vs:-${VORTEX_DB_DIR:-./.data}}}" +_vs="VORTEX_FETCH_DB${_db_index}_DIR" +_vss="VORTEX_DB${_db_index}_DIR" +VORTEX_PUSH_DB_S3_DB_DIR="${!_v:-${!_vs:-${!_vss:-${VORTEX_DB_DIR:-./.data}}}}" # Database dump file name. # @@ -175,8 +176,12 @@ ${canonical_request_hash}" signature=$(aws_signature "${VORTEX_PUSH_DB_S3_SECRET_KEY}" "${date_short}" "${VORTEX_PUSH_DB_S3_REGION}" "s3" "${string_to_sign}") local args=(--silent --show-error --location --proto-redir "=https" --connect-timeout "${VORTEX_PUSH_DB_S3_CONNECT_TIMEOUT:-10}" --max-time "${VORTEX_PUSH_DB_S3_MAX_TIME:-300}" --request PUT --write-out "\n%{http_code}") + # curl sends no length for a PUT without a body, and S3 rejects such a + # request. if [ -n "${upload_file}" ]; then args+=(--upload-file "${upload_file}") + else + args+=(--header "Content-Length: 0") fi local header diff --git a/.vortex/tooling/tests/unit/push-db-s3.bats b/.vortex/tooling/tests/unit/push-db-s3.bats index f9c066e57f..95a5ca50f8 100644 --- a/.vortex/tooling/tests/unit/push-db-s3.bats +++ b/.vortex/tooling/tests/unit/push-db-s3.bats @@ -579,6 +579,8 @@ setup_s3_push() { copy_args="$(mock_get_call_args "${mock_curl}" 2)" assert_string_contains "${copy_args}" "--header x-amz-copy-source: /test-bucket/backups/db.sql" assert_string_not_contains "${copy_args}" "--upload-file" + assert_string_contains "${copy_args}" "--header Content-Length: 0" + assert_string_not_contains "$(mock_get_call_args "${mock_curl}" 1)" "Content-Length: 0" [[ ${copy_args} =~ https://test-bucket\.s3\.ap-southeast-2\.amazonaws\.com/backups/db_[0-9]{8}-[0-9]{6}\.sql ]] rm -rf .data @@ -629,6 +631,10 @@ setup_s3_push() { export VORTEX_FETCH_DB2_S3_REGION="eu-west-1" export VORTEX_FETCH_DB2_S3_PREFIX="" export VORTEX_FETCH_DB2_FILE="db2.sql" + # The dump directory follows the directory the database is fetched to. + mkdir -p .data/db2 + mv .data/db2.sql .data/db2/db2.sql + export VORTEX_FETCH_DB2_DIR=".data/db2" export VORTEX_PUSH_DB_FILE_SUFFIX="_circleci" export VORTEX_PUSH_DB_TIMESTAMPED_COPY=1 @@ -639,7 +645,7 @@ setup_s3_push() { run .vortex/tooling/src/vortex-push-db-s3 assert_success assert_output_contains "[INFO] Started database 2 dump push to S3." - assert_output_contains "Local file: ./.data/db2.sql" + assert_output_contains "Local file: .data/db2/db2.sql" assert_output_contains "Remote file: db2_circleci.sql" assert_output_contains "S3 bucket: db2-bucket" assert_output_contains "[ OK ] Copied the database dump file to db2_circleci_" From b962fd6c6ed6a27a2a19eff06403639a5c93f92a Mon Sep 17 00:00:00 2001 From: Alex Skrypnyk Date: Wed, 11 Nov 2026 16:18:02 +1100 Subject: [PATCH 19/21] Addressed code review: documented setting separate push and fetch filenames for a suffixed dump. --- .../content/development/database-storage.mdx | 25 ++++++++++++++++--- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/.vortex/docs/content/development/database-storage.mdx b/.vortex/docs/content/development/database-storage.mdx index b4f9044238..0269319729 100644 --- a/.vortex/docs/content/development/database-storage.mdx +++ b/.vortex/docs/content/development/database-storage.mdx @@ -87,8 +87,10 @@ refreshes the store from the upstream source: └── db_20261111-180000.sql ``` -`db.sql` always holds the latest dump, and fetching downloads it. By -default, it's the only file a push creates. To keep a history of the +`db.sql` always holds the latest dump, and fetching downloads it. With a +file suffix, the latest dump is the suffixed file, such as `db_gha.sql` +(see [Tell pushes apart](#tell-pushes-apart)). By default, it's the only +file a push creates. To keep a history of the pushed dumps, set `VORTEX_PUSH_DB_TIMESTAMPED_COPY=1` in `.env`. Each push then also creates a copy named with its UTC time, identical to `db.sql` at that moment. Nothing removes the copies, so clear out the old ones @@ -203,8 +205,23 @@ fails for a larger dump. On FTP, the timestamped copy is a second upload. `VORTEX_PUSH_DB_FILE_SUFFIX` adds a suffix before the extension of the pushed files, such as `db_gha.sql` and `db_gha_20261111-180000.sql`. Use a -different suffix in each place that pushes to the same store. A fetch reads -the suffixed file only when its remote filename includes the suffix. +different suffix in each place that pushes to the same store. + +The push takes its remote filename from the fetch, then adds the suffix. To +fetch a suffixed dump from an FTP store, set both filenames, so the suffix +appears once: + +```shell +# The push adds the suffix to this name. +VORTEX_PUSH_DB_FTP_FILE=your_site/db.sql +# The fetch reads the suffixed name. +VORTEX_FETCH_DB_FTP_FILE=your_site/db_gha.sql +``` + +For S3, use the push remote filename `VORTEX_PUSH_DB_S3_REMOTE_FILE` in +place of `VORTEX_PUSH_DB_FTP_FILE`. The S3 fetch names the local dump after +the remote file, so fetching a suffixed dump from S3 also renames the local +dump. ## What's next From 492d8ce6e9145a0b67871be30976ac46f1f1d2a5 Mon Sep 17 00:00:00 2001 From: Alex Skrypnyk Date: Wed, 11 Nov 2026 16:38:53 +1100 Subject: [PATCH 20/21] Addressed code review: documented renaming the dump with 'VORTEX_DB_FILE' for a suffixed S3 fetch. --- .../docs/content/development/database-storage.mdx | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/.vortex/docs/content/development/database-storage.mdx b/.vortex/docs/content/development/database-storage.mdx index 0269319729..f40871cbe2 100644 --- a/.vortex/docs/content/development/database-storage.mdx +++ b/.vortex/docs/content/development/database-storage.mdx @@ -218,10 +218,17 @@ VORTEX_PUSH_DB_FTP_FILE=your_site/db.sql VORTEX_FETCH_DB_FTP_FILE=your_site/db_gha.sql ``` -For S3, use the push remote filename `VORTEX_PUSH_DB_S3_REMOTE_FILE` in -place of `VORTEX_PUSH_DB_FTP_FILE`. The S3 fetch names the local dump after -the remote file, so fetching a suffixed dump from S3 also renames the local -dump. +The S3 fetch names the local dump after the remote file. To fetch a +suffixed dump from an S3 store, rename the dump everywhere with +`VORTEX_DB_FILE`, which the fetch, the refresh and the import all read. +Then set the push remote filename to the unsuffixed name: + +```shell +# The fetch, the refresh and the import use the suffixed name. +VORTEX_DB_FILE=db_gha.sql +# The push adds the suffix to this name. +VORTEX_PUSH_DB_S3_REMOTE_FILE=db.sql +``` ## What's next From aac12f3d85cfa54b5029f3a1f8b13696e4dfb52c Mon Sep 17 00:00:00 2001 From: Alex Skrypnyk Date: Wed, 11 Nov 2026 16:43:20 +1100 Subject: [PATCH 21/21] Reworded the suffixed S3 fetch note to pass the docs spellcheck. --- .vortex/docs/content/development/database-storage.mdx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.vortex/docs/content/development/database-storage.mdx b/.vortex/docs/content/development/database-storage.mdx index f40871cbe2..9b3102336c 100644 --- a/.vortex/docs/content/development/database-storage.mdx +++ b/.vortex/docs/content/development/database-storage.mdx @@ -221,7 +221,7 @@ VORTEX_FETCH_DB_FTP_FILE=your_site/db_gha.sql The S3 fetch names the local dump after the remote file. To fetch a suffixed dump from an S3 store, rename the dump everywhere with `VORTEX_DB_FILE`, which the fetch, the refresh and the import all read. -Then set the push remote filename to the unsuffixed name: +Then set the push remote filename to the name without the suffix: ```shell # The fetch, the refresh and the import use the suffixed name.