From cb5903ac25086eceb6cfb7c6b73d7e7aa1427b60 Mon Sep 17 00:00:00 2001 From: Randy McDermott Date: Fri, 23 Oct 2026 12:09:39 -0400 Subject: [PATCH] Python: mirror FDS env and rebuild when necessary --- .github/requirements.txt | 25 ++++++++++++++++ .gitignore | 2 ++ Utilities/CI/README.md | 8 ++++-- Utilities/CI/run_cfastbot.ps1 | 12 ++++++-- Utilities/CI/setup_python.sh | 22 ++++++-------- Utilities/Python/README.md | 6 +++- Utilities/Python/setup_python_env.bat | 9 ++++++ Utilities/Python/setup_python_env.ps1 | 41 +++++++++++++++++++++++++++ Utilities/Python/setup_python_env.sh | 32 +++++++++++++++++++++ 9 files changed, 138 insertions(+), 19 deletions(-) create mode 100644 .github/requirements.txt create mode 100644 Utilities/Python/setup_python_env.bat create mode 100644 Utilities/Python/setup_python_env.ps1 create mode 100644 Utilities/Python/setup_python_env.sh diff --git a/.github/requirements.txt b/.github/requirements.txt new file mode 100644 index 000000000..ba62a0085 --- /dev/null +++ b/.github/requirements.txt @@ -0,0 +1,25 @@ +# requirements.txt for pip install + +# Mirrored from fds/.github/requirements.txt; keep package requirements synchronized. + +numpy<2 +matplotlib +pandas +scipy +seaborn +pygments +ipython +cantera +pyfdstools +bibtexautocomplete +PySide6 + +# added for bundle scripts +pyinstaller +dmgbuild +pillow + +# Local utility packages (imports fdsplotlib) +-e ../../fds/Utilities/Python + + diff --git a/.gitignore b/.gitignore index e28a61976..e1f131e28 100644 --- a/.gitignore +++ b/.gitignore @@ -65,6 +65,8 @@ fort.3 # Python bytecode __pycache__/ *.py[cod] +.github/cfast_python_env/ +.github/cfast_matplotlib/ # CEditQt app build output Build/CeditQt/icons/ diff --git a/Utilities/CI/README.md b/Utilities/CI/README.md index 1d9c75ca1..1453422bc 100644 --- a/Utilities/CI/README.md +++ b/Utilities/CI/README.md @@ -39,7 +39,7 @@ On Windows, use a compiler-enabled PowerShell terminal from `cfast/Utilities/CI` .\run_cfastbot.ps1 -Compiler gnu -Python C:\path\to\python.exe ``` -The Windows launcher runs the full pipeline locally and sequentially, without Bash or Slurm. It requires GNU Make, Intel C/C++ for Smokeview, Intel Fortran by default (or GNU Fortran with `-Compiler gnu`), Python with the FDS plotting and CEditQt dependencies, and `pdflatex`/`biber`. Activate the compiler environment before launching. Python must already be provisioned; `CFAST_CI_PYTHON` overrides `-Python`. Use `-StateDirectory` to override the external log directory. Stage logs, the final `exit_code`, validation statistics, and successful guide PDFs are saved under the run directory. Per-case `.stdout`, `.err`, and `.exit` files remain beside the inputs; UI checks use a separate `.ui` suffix. The Windows launcher saves results locally and does not send email or publish bundles. +The Windows launcher runs the full pipeline locally and sequentially, without Bash or Slurm. It requires GNU Make, Intel C/C++ for Smokeview, Intel Fortran by default (or GNU Fortran with `-Compiler gnu`), Python with the FDS plotting and CEditQt dependencies, and `pdflatex`/`biber`. Activate the compiler environment before launching. Python selection uses `CFAST_CI_PYTHON`, then an explicit `-Python`, then `fds/.github/fds_python_env/Scripts/python.exe`. If none is supplied or available, the launcher provisions `cfast/.github/cfast_python_env` from the mirrored CFAST requirements. Use `-StateDirectory` to override the external log directory. Stage logs, the final `exit_code`, validation statistics, and successful guide PDFs are saved under the run directory. Per-case `.stdout`, `.err`, and `.exit` files remain beside the inputs; UI checks use a separate `.ui` suffix. The Windows launcher saves results locally and does not send email or publish bundles. Progress and command output are printed to stderr as the run proceeds and saved in the stage logs. The launcher prints the log directory at startup. Batch jobs report submission and suite progress on screen; individual case output remains in the case log files. CFAST writes its console output to each case's `.err` file and its model log to `.log`. Follow a running case from another terminal, for example: @@ -72,9 +72,11 @@ Create `/home/firebot/.cfastbot` before using that log redirection. The log capt ## State and Python -State defaults to `$HOME/.cfastbot`; override it with `CFAST_CI_STATE_DIR`. It must be outside the cleaned checkouts. It contains per-run logs under `runs/`, history, and the Python environment. `latest_run` records the most recent log directory. Each run records repository and submodule revisions and its final exit code. Successful runs save PDFs and revision metadata in `VERSION`; `VERSION_LATEST` contains metadata for the current run. +State defaults to `$HOME/.cfastbot`; override it with `CFAST_CI_STATE_DIR`. It must be outside the cleaned checkouts. It contains per-run logs under `runs/` and history. Python environments and the Matplotlib cache live in the repository collection so compute nodes can access them. `latest_run` records the most recent log directory. Each run records repository and submodule revisions and its final exit code. Successful runs save PDFs and revision metadata in `VERSION`; `VERSION_LATEST` contains metadata for the current run. -The managed Python environment is installed from `fds/.github/requirements.txt` and reused until those requirements change. `CFAST_CI_PYTHON` may select an already-provisioned interpreter instead. FDS's plotting module is imported from the selected FDS checkout. Linux bundle builds also require PyInstaller in this environment. +On Linux and macOS, CI uses `CFAST_CI_PYTHON` when supplied, otherwise the existing sibling `fds/.github/fds_python_env/bin/python`. Only if that FDS interpreter is absent does it run `Utilities/Python/setup_python_env.sh --batchmode` to provision or reuse `cfast/.github/cfast_python_env`. Windows follows the same selection with `Scripts/python.exe` and the PowerShell setup script. An existing FDS environment is checked for required imports but is never rebuilt or updated by CFAST CI; a failed import stops the run and reports the selected interpreter. + +CFAST's `.github/requirements.txt` mirrors FDS's package list, with the editable `fdsplotlib` path adjusted to the sibling FDS checkout. Keep these lists synchronized when dependencies change. CFAST setup reuses its environment and installs changed requirements in place. The Matplotlib cache is `cfast/.github/cfast_matplotlib`. With Slurm, the selected interpreter, repository collection, and cache must be accessible from compute nodes. Repository preparation deletes ignored files, including these environments; selection occurs afterward. Linux bundle builds also require PyInstaller, which is included in the mirrored requirements. ## Bundles and publishing diff --git a/Utilities/CI/run_cfastbot.ps1 b/Utilities/CI/run_cfastbot.ps1 index 08720672d..bea863967 100644 --- a/Utilities/CI/run_cfastbot.ps1 +++ b/Utilities/CI/run_cfastbot.ps1 @@ -2,7 +2,7 @@ [CmdletBinding()] param( [ValidateSet('intel', 'gnu')][string]$Compiler = 'intel', - [string]$Python = 'python', + [string]$Python = '', [string]$StateDirectory = $(if ($env:CFAST_CI_STATE_DIR) { $env:CFAST_CI_STATE_DIR } else { Join-Path $HOME '.cfastbot' }), [switch]$Help ) @@ -137,11 +137,19 @@ try { $fortran = if ($Compiler -eq 'intel') { 'ifx' } else { 'gfortran' } if (-not (Get-Command $fortran -CommandType Application -ErrorAction SilentlyContinue)) { throw "Missing prerequisite: $fortran" } if ($env:CFAST_CI_PYTHON) { $Python = $env:CFAST_CI_PYTHON } + if (-not $Python) { + $Python = Join-Path $root 'fds/.github/fds_python_env/Scripts/python.exe' + if (-not (Test-Path -LiteralPath $Python -PathType Leaf)) { + . (Join-Path $script:Repo 'Utilities/Python/setup_python_env.ps1') --batchmode + $Python = Join-Path $script:Repo '.github/cfast_python_env/Scripts/python.exe' + } + } $script:PythonExe = (Get-Command $Python -CommandType Application -ErrorAction Stop | Select-Object -First 1).Source + Write-Host "CFAST Python: $script:PythonExe" $env:PYTHONPATH = (Join-Path $root 'fds/Utilities/Python') + [IO.Path]::PathSeparator + $env:PYTHONPATH $env:QT_QPA_PLATFORM = 'offscreen' $env:MPLBACKEND = 'Agg' - $env:MPLCONFIGDIR = Join-Path $state 'matplotlib' + $env:MPLCONFIGDIR = Join-Path $script:Repo '.github/cfast_matplotlib' New-Item -ItemType Directory -Path $env:MPLCONFIGDIR -Force | Out-Null Invoke-Stage 'python_check' $script:Repo $script:PythonExe @('-c', 'import fdsplotlib, numpy, pandas, scipy, matplotlib, PySide6') diff --git a/Utilities/CI/setup_python.sh b/Utilities/CI/setup_python.sh index 0abd8e926..5a3bf6239 100755 --- a/Utilities/CI/setup_python.sh +++ b/Utilities/CI/setup_python.sh @@ -1,24 +1,20 @@ #!/usr/bin/env bash -# Source with the repository collection root. Keep the environment outside checkouts. +# Prefer the sibling FDS environment; provision CFAST's environment only as a fallback. setup_cfast_python() { - local root=$1 state=${CFAST_CI_STATE_DIR:-$HOME/.cfastbot} requirements venv - requirements="$root/fds/.github/requirements.txt" - venv="$state/python_env" + local root=$1 if [[ -n ${CFAST_CI_PYTHON:-} ]]; then # Explicitly supplied, already provisioned interpreter (also used for local tests). export CFAST_PYTHON=$CFAST_CI_PYTHON + elif [[ -x $root/fds/.github/fds_python_env/bin/python ]]; then + export CFAST_PYTHON="$root/fds/.github/fds_python_env/bin/python" else - [[ -f $requirements ]] || { echo "Missing Python requirements: $requirements" >&2; return 1; } - [[ -x $venv/bin/python ]] || python3 -m venv "$venv" || return 1 - if ! cmp -s "$requirements" "$venv/cfast_requirements.txt"; then - (cd "$root/fds/.github" && "$venv/bin/python" -m pip install -r requirements.txt) || return 1 - cp "$requirements" "$venv/cfast_requirements.txt" - fi - export CFAST_PYTHON="$venv/bin/python" + source "$root/cfast/Utilities/Python/setup_python_env.sh" --batchmode || return 1 + export CFAST_PYTHON="$root/cfast/.github/cfast_python_env/bin/python" fi + echo "CFAST Python: $CFAST_PYTHON" export PATH="$(dirname "$CFAST_PYTHON"):$PATH" export PYTHONPATH="$root/fds/Utilities/Python${PYTHONPATH:+:$PYTHONPATH}" - export MPLCONFIGDIR="$state/matplotlib" - mkdir -p "$MPLCONFIGDIR" + export MPLCONFIGDIR="$root/cfast/.github/cfast_matplotlib" + mkdir -p "$MPLCONFIGDIR" || return 1 "$CFAST_PYTHON" -c 'import fdsplotlib, numpy, pandas, scipy, matplotlib, PySide6' || return 1 } diff --git a/Utilities/Python/README.md b/Utilities/Python/README.md index b2e5c1e2d..d5f969ce4 100644 --- a/Utilities/Python/README.md +++ b/Utilities/Python/README.md @@ -2,7 +2,11 @@ This folder contains Python scripts that read the output of the CFAST verification and validation cases and create figures for the CFAST Manuals. Before running these scripts, run the [verification cases](https://github.com/firemodels/cfast/tree/master/Verification/scripts) and the [validation cases](https://github.com/firemodels/cfast/tree/master/Validation/scripts). -After the V&V cases have completed, follow the [instructions](https://github.com/firemodels/fds/wiki/Python-Setting-Up-Your-Environment) for installing and activating a special Python environment used by CFAST and FDS for post-processing. This special evnironment ensures that the proper modules are installed on your system. Once that is done, then run the scripts: +CFAST uses the same Python package requirements as FDS. Keep an FDS checkout beside CFAST for the shared `fdsplotlib` module. From this directory, create and activate CFAST's environment on Linux/macOS with `source ./setup_python_env.sh`; in PowerShell use `. .\setup_python_env.ps1`, or in Command Prompt use `call setup_python_env.bat`. The environment is stored in `cfast/.github/cfast_python_env` and installed from `cfast/.github/requirements.txt`. Setup reuses existing environments and installs changed requirements in place; `--batchmode` is accepted for unattended use. Python 3.7 is the setup minimum inherited from FDS, but current packages may require a newer Python version. + +An already provisioned FDS environment can also be activated and used directly. CFAST CI prefers that sibling FDS environment and only sets up CFAST's environment when the FDS interpreter is absent; `CFAST_CI_PYTHON` explicitly overrides that selection. + +After the V&V cases have completed and the environment is active, run: ``` python CFAST_verification_script.py python CFAST_validation_script.py diff --git a/Utilities/Python/setup_python_env.bat b/Utilities/Python/setup_python_env.bat new file mode 100644 index 000000000..5dc97bdef --- /dev/null +++ b/Utilities/Python/setup_python_env.bat @@ -0,0 +1,9 @@ +@echo off +REM Configure CFAST's environment, then activate it in this Command Prompt. +REM Usage: call setup_python_env.bat [--batchmode] +powershell.exe -NoProfile -File "%~dp0setup_python_env.ps1" %* +if errorlevel 1 exit /b 1 +call "%~dp0..\..\.github\cfast_python_env\Scripts\activate.bat" +if errorlevel 1 exit /b 1 +set "PYTHONPATH=%~dp0..\..\..\fds\Utilities\Python;%PYTHONPATH%" +exit /b 0 diff --git a/Utilities/Python/setup_python_env.ps1 b/Utilities/Python/setup_python_env.ps1 new file mode 100644 index 000000000..19ff80b13 --- /dev/null +++ b/Utilities/Python/setup_python_env.ps1 @@ -0,0 +1,41 @@ +# Dot-source to configure and activate CFAST's Python environment on Windows: +# . .\setup_python_env.ps1 [--batchmode] +# Existing environments are reused; changed requirements are installed in place. +function Initialize-CfastPythonEnv { + param([string[]]$Options) + $ErrorActionPreference = 'Stop' + foreach ($Option in $Options) { + if ($Option -notin @('--batchmode', '-BatchMode')) { throw "Unknown argument: $Option" } + } + $Repo = [IO.Path]::GetFullPath((Join-Path $PSScriptRoot '../..')) + $GitHubDir = Join-Path $Repo '.github' + $Requirements = Join-Path $GitHubDir 'requirements.txt' + $Venv = Join-Path $GitHubDir 'cfast_python_env' + $Interpreter = Join-Path $Venv 'Scripts/python.exe' + if (-not (Test-Path -LiteralPath $Requirements -PathType Leaf)) { throw "Missing Python requirements: $Requirements" } + if (-not (Test-Path -LiteralPath $Interpreter -PathType Leaf)) { + & python -c 'import sys; sys.exit(sys.version_info < (3, 7))' + if ($LASTEXITCODE -ne 0) { throw 'Python 3.7 or higher is required (packages may require a newer version).' } + & python -m venv $Venv + if ($LASTEXITCODE -ne 0) { throw 'Failed to create CFAST Python environment.' } + } + $InstalledRequirements = Join-Path $Venv 'cfast_requirements.txt' + if (-not (Test-Path -LiteralPath $InstalledRequirements) -or + (Get-FileHash -LiteralPath $Requirements).Hash -ne (Get-FileHash -LiteralPath $InstalledRequirements).Hash) { + & $Interpreter -m pip install --upgrade pip + if ($LASTEXITCODE -ne 0) { throw 'Failed to upgrade pip.' } + Push-Location -LiteralPath $GitHubDir + try { + & $Interpreter -m pip install -r requirements.txt + if ($LASTEXITCODE -ne 0) { throw 'Failed to install CFAST Python requirements.' } + } finally { Pop-Location } + Copy-Item -LiteralPath $Requirements -Destination $InstalledRequirements -ErrorAction Stop + } + . (Join-Path $Venv 'Scripts/Activate.ps1') + $env:PYTHONPATH = (Join-Path $Repo '../fds/Utilities/Python') + [IO.Path]::PathSeparator + $env:PYTHONPATH + & $Interpreter -c 'import fdsplotlib, numpy, pandas, scipy, matplotlib, PySide6' + if ($LASTEXITCODE -ne 0) { throw 'CFAST Python dependency check failed.' } + Write-Host "CFAST Python environment ready: $Venv" +} + +Initialize-CfastPythonEnv -Options $args diff --git a/Utilities/Python/setup_python_env.sh b/Utilities/Python/setup_python_env.sh new file mode 100644 index 000000000..796ffc67b --- /dev/null +++ b/Utilities/Python/setup_python_env.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +# Configure and activate CFAST's environment, using the FDS package requirements. +# Usage (Linux/macOS): source ./setup_python_env.sh [--batchmode] +# Existing environments are reused; changed requirements are installed in place. + +setup_cfast_python_env() { + local arg repo requirements venv + for arg in "$@"; do + [[ $arg == --batchmode ]] || { echo "Unknown argument: $arg" >&2; return 1; } + done + repo=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd) || return 1 + requirements="$repo/.github/requirements.txt" + venv="$repo/.github/cfast_python_env" + [[ -f $requirements ]] || { echo "Missing Python requirements: $requirements" >&2; return 1; } + if [[ ! -x $venv/bin/python ]]; then + python3 -c 'import sys; sys.exit(sys.version_info < (3, 7))' || { + echo 'Python 3.7 or higher is required (packages may require a newer version).' >&2; return 1; + } + python3 -m venv "$venv" || return 1 + fi + if ! cmp -s "$requirements" "$venv/cfast_requirements.txt"; then + "$venv/bin/python" -m pip install --upgrade pip || return 1 + (cd "$repo/.github" && "$venv/bin/python" -m pip install -r requirements.txt) || return 1 + cp "$requirements" "$venv/cfast_requirements.txt" || return 1 + fi + source "$venv/bin/activate" || return 1 + export PYTHONPATH="$repo/../fds/Utilities/Python${PYTHONPATH:+:$PYTHONPATH}" + "$venv/bin/python" -c 'import fdsplotlib, numpy, pandas, scipy, matplotlib, PySide6' || return 1 + echo "CFAST Python environment ready: $venv" +} + +setup_cfast_python_env "$@"