diff --git a/.github/actions/docker-build/action.yaml b/.github/actions/docker-build/action.yaml index e1df6baf8d..e2aad49d4d 100644 --- a/.github/actions/docker-build/action.yaml +++ b/.github/actions/docker-build/action.yaml @@ -4,6 +4,10 @@ author: "Devito" inputs: # The only supported GHA input type is string + working-directory: + description: "Change to this directory before executing build" + required: true + default: "." file: description: "Dockerfile containing build instructions" required: true @@ -18,6 +22,10 @@ inputs: description: "Arguments to pass to `docker build`" required: true default: "" + local: + description: "Set true to prevent attempting to pull the image" + required: true + default: "" outputs: unique: @@ -40,9 +48,11 @@ runs: shell: bash env: BASE: ${{ inputs.base }} + LOCAL: ${{ inputs.local }} run: | + cd ${{ inputs.working-directory }} docker build \ - --pull \ + $([ "${{ inputs.local }}" = true ] && echo "" || echo "--pull") \ --file ${{ inputs.file }} \ --tag ${{ inputs.tag }}_${{ steps.uniquetag.outputs.unique }} \ ${BASE:+--build-arg base=$BASE} \ diff --git a/.github/actions/docker-clean/action.yaml b/.github/actions/docker-clean/action.yaml index 56bf2f88ab..32dc3b89d6 100644 --- a/.github/actions/docker-clean/action.yaml +++ b/.github/actions/docker-clean/action.yaml @@ -1,5 +1,5 @@ name: Docker cleanup action -description: Composite action for removing Docker images +description: Composite action for removing Docker images and containers author: "Devito" inputs: @@ -10,12 +10,36 @@ inputs: tag: description: "Tag of the built image to use" required: true + container_hashes: + description: "Hash (or list of hashes) of the container that was run using the docker-run action" + required: true + default: "" runs: using: "composite" steps: - - id: dockerclean + - id: dockercontainerkill + name: "Kill hanging containers" + shell: bash + run: | + for HASH in ${{ inputs.container_hashes }}; do + # Kill the docker container + # This should not be necessary if the container exited cleanly + docker kill $HASH || true + done + + - id: dockercontainerrm + name: "Remove remaining containers" + shell: bash + run: | + for HASH in ${{ inputs.container_hashes }}; do + # Remove the docker container + # This should not be necessary if the `--rm` flag is passed + docker container rm -f $HASH || true + done + + - id: dockerimageclean name: "Cleanup docker image" shell: bash run: | - docker image rm -f "${{ inputs.tag }}_${{ inputs.uid }}" + docker image rm -f "${{ inputs.tag }}_${{ inputs.uid }}" diff --git a/.github/actions/docker-run/action.yaml b/.github/actions/docker-run/action.yaml index 375a21c7ce..90a0482960 100644 --- a/.github/actions/docker-run/action.yaml +++ b/.github/actions/docker-run/action.yaml @@ -4,6 +4,10 @@ author: "Devito" inputs: # The only supported GHA input type is string + working-directory: + description: "Change to this directory before executing build" + required: true + default: "." uid: description: "Unique identifier output from docker-build action" required: true @@ -25,6 +29,11 @@ inputs: description: "Command to execute inside of the docker container" required: true +outputs: + hash: + description: "Docker hash for the running container" + value: ${{ steps.postrun.outputs.hash }} + runs: using: "composite" steps: @@ -51,7 +60,9 @@ runs: env: NAME: ${{ inputs.name }} run: | + cd ${{ inputs.working-directory }} docker run \ + --cidfile "${{ inputs.uid }}.cid" \ --init -t --rm \ ${{ inputs.args }} \ --name "ci-${NAME:-${{ inputs.tag }}}-${{ inputs.uid }}" \ @@ -59,3 +70,13 @@ runs: ${{ steps.processenv.outputs.docker_environment_args }} \ "${{ inputs.tag }}_${{ inputs.uid }}" \ ${{ inputs.command }} + + - id: postrun + name: "Collect hash of launched container" + if: always() + shell: bash + run: | + cd ${{ inputs.working-directory }} + HASH=$(cat "${{ inputs.uid }}.cid") + rm "${{ inputs.uid }}.cid" + echo "hash=${HASH}" >> "$GITHUB_OUTPUT" diff --git a/.github/actions/readme.md b/.github/actions/readme.md index d6f249963d..f170a6dc94 100644 --- a/.github/actions/readme.md +++ b/.github/actions/readme.md @@ -13,11 +13,12 @@ uses: ./.github/actions/docker-run ## `docker-build` Inputs: - +- `working-directory`: Change to this directory before executing build command - `file`: Dockerfile containing build instructions (default: `Dockerfile`) - `tag`: Tag to add to the built image - `base`: Base docker image to build on top of - `args`: Arguments to pass to `docker build` +- `local`: Set this to `true` to prevent attempting to pull the image from a remote Outputs: @@ -43,12 +44,17 @@ jobs: Inputs: +- `working-directory`: Change to this directory before executing run command - `uid`: Unique identifier output from docker-build action - `tag`: Tag of the built image to use - `name`: Name substring for docker to use when running the command (optional) - `args`: Arguments to pass to `docker run`, `--init -t --rm` are always added - `env`: Environment variables to set inside the docker container, one environment variable per line --command: Command to execute inside of the docker container +- `command`: Command to execute inside of the docker container + +Outputs: + +- `hash`: The Docker hash for the running container (ie: `${{ steps.runA.outputs.hash }}) ### Notes @@ -65,7 +71,8 @@ Example: jobs: test: steps: - - name: Run a command in a previously built Docker container + - id: runA + name: Run a command in a previously built Docker container uses: ./.github/actions/docker-run with: uid: ${{ steps.build.outputs.unique }} @@ -87,6 +94,7 @@ Inputs: - `uid`: Unique identifier output from docker-build action - `tag`: Tag of the built image to use +- `container_hashes`: Hash (or list of hashes) of the container that was run using the docker-run action ### Notes @@ -106,4 +114,7 @@ jobs: with: uid: ${{ steps.build.outputs.unique }} tag: tag-related-to-test-config + container_hashes: >- + ${{ steps.runA.outputs.hash }} + ${{ steps.runB.outputs.hash }} ``` diff --git a/.github/workflows/pytest-core-mpi.yaml b/.github/workflows/pytest-core-mpi.yaml index 2b82a5e796..1c18f64525 100644 --- a/.github/workflows/pytest-core-mpi.yaml +++ b/.github/workflows/pytest-core-mpi.yaml @@ -105,7 +105,8 @@ jobs: tag: ${{ matrix.name }} base: devitocodes/bases:cpu-${{ matrix.arch }} - - name: Test with pytest + - id: pytest + name: Test with pytest uses: ./.github/actions/docker-run with: uid: ${{ steps.build.outputs.unique }} @@ -121,7 +122,8 @@ jobs: --cov-report=xml \ tests/test_mpi.py - - name: Test acoustic and TTI examples with MPI + - id: examples + name: Test acoustic and TTI examples with MPI uses: ./.github/actions/docker-run with: uid: ${{ steps.build.outputs.unique }} @@ -142,3 +144,6 @@ jobs: with: uid: ${{ steps.build.outputs.unique }} tag: ${{ matrix.name }} + container_hashes: >- + ${{ steps.pytest.outputs.hash }} + ${{ steps.examples.outputs.hash }} diff --git a/.github/workflows/pytest-core-nompi.yaml b/.github/workflows/pytest-core-nompi.yaml index 7c4657247f..6e4e7232a2 100644 --- a/.github/workflows/pytest-core-nompi.yaml +++ b/.github/workflows/pytest-core-nompi.yaml @@ -237,7 +237,8 @@ jobs: tag: ${{ matrix.name }} base: devitocodes/bases:cpu-${{ matrix.arch }} - - name: Test with pytest + - id: pytest + name: Test with pytest uses: ./.github/actions/docker-run with: uid: ${{ steps.build.outputs.unique }} @@ -260,3 +261,5 @@ jobs: with: uid: ${{ steps.build.outputs.unique }} tag: ${{ matrix.name }} + container_hashes: >- + ${{ steps.pytest.outputs.hash }} diff --git a/.github/workflows/pytest-gpu.yaml b/.github/workflows/pytest-gpu.yaml index 3ade43ecad..b08aa51764 100644 --- a/.github/workflows/pytest-gpu.yaml +++ b/.github/workflows/pytest-gpu.yaml @@ -83,11 +83,8 @@ jobs: tag: ${{ matrix.name }} base: ${{ matrix.base }} - - name: Runner environment - run: | - printenv - - - name: Probe GPU + - id: probe + name: Probe GPU uses: ./.github/actions/docker-run with: uid: ${{ steps.build.outputs.unique }} @@ -95,7 +92,8 @@ jobs: args: ${{ matrix.dockerflags }} command: ${{ matrix.test_drive_cmd }} - - name: Test with pytest + - id: pytest + name: Test with pytest uses: ./.github/actions/docker-run with: uid: ${{ steps.build.outputs.unique }} @@ -122,7 +120,8 @@ jobs: --cov-report=xml \ ${{ matrix.test_files }} - - name: Test examples + - id: examples + name: Test examples uses: ./.github/actions/docker-run with: uid: ${{ steps.build.outputs.unique }} @@ -135,7 +134,8 @@ jobs: -v \ ${{ matrix.test_examples }} - - name: Test examples with MPI + - id: mpi + name: Test examples with MPI uses: ./.github/actions/docker-run with: uid: ${{ steps.build.outputs.unique }} @@ -157,3 +157,8 @@ jobs: with: uid: ${{ steps.build.outputs.unique }} tag: ${{ matrix.name }} + container_hashes: >- + ${{ steps.probe.outputs.hash }} + ${{ steps.pytest.outputs.hash }} + ${{ steps.examples.outputs.hash }} + ${{ steps.mpi.outputs.hash }} diff --git a/.github/workflows/tutorials.yaml b/.github/workflows/tutorials.yaml index 57e956c296..0ecc5a982a 100644 --- a/.github/workflows/tutorials.yaml +++ b/.github/workflows/tutorials.yaml @@ -120,7 +120,8 @@ jobs: file: docker/Dockerfile.devito tag: ${{ matrix.name }} - - name: Test seismic notebooks + - id: seismic + name: Test seismic notebooks uses: ./.github/actions/docker-run with: uid: ${{ steps.build.outputs.unique }} @@ -136,7 +137,8 @@ jobs: -k 'not dask and not synthetics and not mpi' \ examples - - name: Failing notebooks + - id: failing + name: Failing notebooks continue-on-error: true uses: ./.github/actions/docker-run with: @@ -149,7 +151,8 @@ jobs: command: | pytest -v --nbval examples/seismic/tutorials/14_creating_synthetics.ipynb - - name: Dask notebooks + - id: dask + name: Dask notebooks uses: ./.github/actions/docker-run with: uid: ${{ steps.build.outputs.unique }} @@ -168,3 +171,7 @@ jobs: with: uid: ${{ steps.build.outputs.unique }} tag: ${{ matrix.name }} + container_hashes: >- + ${{ steps.seismic.outputs.hash }} + ${{ steps.failing.outputs.hash }} + ${{ steps.dask.outputs.hash }}