From 7e414c991ca9841883d9a2df82f106e723abd11a Mon Sep 17 00:00:00 2001 From: JDBetteridge Date: Fri, 6 Mar 2026 22:42:42 +0000 Subject: [PATCH 01/11] ci: Add composite actions and test in core-mpi --- .github/actions/docker-build/action.yaml | 55 ++++++++ .github/actions/docker-clean/action.yaml | 21 +++ .github/actions/docker-run/action.yaml | 56 ++++++++ .github/workflows/pytest-core-mpi.yaml | 157 +++++++++++------------ docker/Dockerfile.devito | 1 + docker/coverage.env | 23 ++++ 6 files changed, 228 insertions(+), 85 deletions(-) create mode 100644 .github/actions/docker-build/action.yaml create mode 100644 .github/actions/docker-clean/action.yaml create mode 100644 .github/actions/docker-run/action.yaml create mode 100644 docker/coverage.env diff --git a/.github/actions/docker-build/action.yaml b/.github/actions/docker-build/action.yaml new file mode 100644 index 0000000000..004ee0d0d2 --- /dev/null +++ b/.github/actions/docker-build/action.yaml @@ -0,0 +1,55 @@ +name: Docker build action +description: Composite action for building Devito Docker containers +author: "Devito" + +inputs: +# The only supported GHA input type is string + file: + description: "Dockerfile containing build instructions" + required: true + default: Dockerfile + tag: + description: "Tag to add to the built image" + required: true + base: + description: "Base docker image to build on top of" + required: true + # Update default if docker/Dockerfile.devito ever changes default + default: "devitocodes/bases:cpu-gcc" + +outputs: + unique: + description: "Unique identifier for the CI run" + value: ${{ steps.uniquetag.outputs.unique }} + +runs: + using: "composite" + steps: + - id: uniquetag + name: "Generate unique CI tag" + shell: bash + run: | + UNIQUE=$(echo "${GITHUB_RUN_ID}_${GITHUB_RUN_ATTEMPT}" | cksum | cut -f 1 -d " ") + echo "Unique ID: ${UNIQUE}" + echo "unique=${UNIQUE}" >> "$GITHUB_OUTPUT" + + - id: dockerbuild + name: "Build docker container" + shell: bash + run: | + docker build \ + --pull \ + --file ${{ inputs.file }} \ + --tag ${{ inputs.tag }}_${{ steps.uniquetag.outputs.unique }} \ + --build-arg base=${{ inputs.base }} \ + . + +# Do we need to be more specific? +#~ docker buildx build . \ + #~ --builder "${RUNNER_NAME// /_}" \ + #~ --load \ + #~ --label ci-run="$GITHUB_RUN_ID" \ + #~ --rm --pull \ + #~ --file docker/Dockerfile.devito \ + #~ --tag "${DOCKER_IMAGE}" \ + #~ --build-arg base="${{ matrix.base }}" diff --git a/.github/actions/docker-clean/action.yaml b/.github/actions/docker-clean/action.yaml new file mode 100644 index 0000000000..56bf2f88ab --- /dev/null +++ b/.github/actions/docker-clean/action.yaml @@ -0,0 +1,21 @@ +name: Docker cleanup action +description: Composite action for removing Docker images +author: "Devito" + +inputs: +# The only supported GHA input type is string + uid: + description: "Unique identifier output from docker-build action" + required: true + tag: + description: "Tag of the built image to use" + required: true + +runs: + using: "composite" + steps: + - id: dockerclean + name: "Cleanup docker image" + shell: bash + run: | + docker image rm -f "${{ inputs.tag }}_${{ inputs.uid }}" diff --git a/.github/actions/docker-run/action.yaml b/.github/actions/docker-run/action.yaml new file mode 100644 index 0000000000..70218699b2 --- /dev/null +++ b/.github/actions/docker-run/action.yaml @@ -0,0 +1,56 @@ +name: Docker run action +description: Composite action for running commands in Docker containers +author: "Devito" + +inputs: +# The only supported GHA input type is string + uid: + description: "Unique identifier output from docker-build action" + required: true + args: + description: "Arguments to pass to `docker run`" + required: true + default: "" + env: + description: "Environment variables to set inside the docker container, one environment variable per line" + required: true + default: Dockerfile + tag: + description: "Tag of the built image to use" + required: true + command: + description: "Command to execute inside of the docker container" + required: true + +runs: + using: "composite" + steps: + - id: processenv + name: Process environment variable list + shell: bash + env: + ENV_INPUT: ${{ inputs.env }} + run: | + ENV_STRING="" + # Read line by line with here string, safely handling spaces within the values + while IFS= read -r LINE; do + if [[ -n "$LINE" ]]; then + ENV_STRING="$ENV_STRING --env $LINE" + fi + done <<< "$ENV_INPUT" + # Remove the leading space from the first concatenation + ENV_STRING="${ENV_STRING# }" + echo "env=$ENV_STRING" >> "$GITHUB_OUTPUT" + + - id: dockerrun + name: "Run command ${{ inputs.command }} in ${{ inputs.tag }} docker container" + shell: bash + run: | + docker run \ + --init -t --rm \ + ${{ inputs.args }} \ + --name "ci-${{ inputs.name }}-${{ inputs.uid }}" \ + --env-file=docker/coverage.env \ + ${{ steps.processenv.outputs.env }} \ + "${{ inputs.tag }}_${{ inputs.uid }}" \ + ${{ inputs.command }} diff --git a/.github/workflows/pytest-core-mpi.yaml b/.github/workflows/pytest-core-mpi.yaml index 6923691723..f21851b0a8 100644 --- a/.github/workflows/pytest-core-mpi.yaml +++ b/.github/workflows/pytest-core-mpi.yaml @@ -12,10 +12,10 @@ on: # but only for the main branch push: branches: - - main + - main pull_request: branches: - - main + - main jobs: test-mpi-basic: @@ -63,86 +63,73 @@ jobs: name: pytest-mpi test-mpi-docker: - name: pytest-mpi - runs-on: ${{ matrix.os }} - outputs: - unique : ${{ steps.uniquetag.outputs.unique }} - strategy: - matrix: - name: [gcc, gcc-arm, icx] - include: - - name: gcc - arch: gcc - os: ubuntu-latest - mpiflag: "" - - - name: gcc-arm - arch: gcc - os: ubuntu-24.04-arm - mpiflag: "" - - - name: icx - arch: icx - os: ubuntu-latest - # Need safe math for icx due to inaccuracy with mpi+sinc interpolation - mpiflag: "-e DEVITO_SAFE_MATH=1" - - steps: - - name: Checkout devito - uses: actions/checkout@v6 - - - name: Generate unique CI tag - id: uniquetag - run: | - UNIQUE=$(echo "${GITHUB_RUN_ID}_${GITHUB_RUN_ATTEMPT}" | cksum | cut -f 1 -d " ") - echo "Unique ID: ${UNIQUE}" - echo "unique=${UNIQUE}" >> "$GITHUB_OUTPUT" - - - name: Build docker image - env: - UNIQUE: ${{ steps.uniquetag.outputs.unique }} - run: | - docker build \ - --file docker/Dockerfile.devito \ - --tag "devito_img${UNIQUE}" \ - --build-arg base=devitocodes/bases:cpu-${{ matrix.arch }} \ - . - - - name: Test with pytest - env: - UNIQUE: ${{ steps.uniquetag.outputs.unique }} - run: | - docker run \ - --init -t --rm \ - --env CODECOV_TOKEN=${{ secrets.CODECOV_TOKEN }} \ - --env OMP_NUM_THREADS=1 \ - --name testrun \ - "devito_img${UNIQUE}" \ - pytest tests/test_mpi.py - - - name: Test examples with MPI - env: - UNIQUE: ${{ steps.uniquetag.outputs.unique }} - run: | - docker run \ - --init -t --rm \ - ${{ matrix.mpiflag }} \ - --env DEVITO_MPI=1 \ - --env OMP_NUM_THREADS=1 \ - --name examplerun \ - "devito_img${UNIQUE}" \ - mpiexec -n 2 pytest examples/seismic/acoustic - # - docker run \ - --init -t --rm \ - --env DEVITO_MPI=1 \ - --env OMP_NUM_THREADS=1 \ - --name examplerun \ - "devito_img${UNIQUE}" \ - mpiexec -n 2 pytest examples/seismic/tti - - - name: Cleanup - env: - UNIQUE: ${{ steps.uniquetag.outputs.unique }} - run: | - docker image rm -f "devito_img${UNIQUE}" + name: pytest-mpi + runs-on: ${{ matrix.os }} + strategy: + matrix: + name: [ + pytest-mpi-docker-gcc, + pytest-mpi-docker-gcc-arm, + pytest-mpi-docker-icx + ] + include: + - name: pytest-mpi-docker-gcc + arch: gcc + os: ubuntu-latest + + - name: pytest-mpi-docker-gcc-arm + arch: gcc + os: ubuntu-24.04-arm + + - name: pytest-mpi-docker-icx + arch: icx + os: ubuntu-latest + + steps: + - name: Checkout devito + uses: actions/checkout@v6 + + - id: build + name: Build docker image + uses: ./.github/actions/docker-build + with: + file: docker/Dockerfile.devito + tag: ${{ matrix.name }} + base: devitocodes/bases:cpu-${{ matrix.arch }} + + - name: Test with pytest + uses: ./.github/actions/docker-run + with: + uid: ${{ steps.build.outputs.unique }} + tag: ${{ matrix.name }} + env: | + CODECOV_TOKEN=${{ secrets.CODECOV_TOKEN }} + OMP_NUM_THREADS=1 + command: "pytest tests/test_mpi.py" + + - name: Test acoustic example with MPI + uses: ./.github/actions/docker-run + with: + uid: ${{ steps.build.outputs.unique }} + tag: ${{ matrix.name }} + env: | + DEVITO_MPI=1 + OMP_NUM_THREADS=1 + command: "mpiexec -n 2 pytest examples/seismic/acoustic" + + - name: Test tti example with MPI + uses: ./.github/actions/docker-run + with: + uid: ${{ steps.build.outputs.unique }} + tag: ${{ matrix.name }} + env: | + DEVITO_MPI=1 + OMP_NUM_THREADS=1 + command: "mpiexec -n 2 pytest examples/seismic/tti" + + - name: Cleanup docker image + if: always() + uses: ./.github/actions/docker-clean + with: + uid: ${{ steps.build.outputs.unique }} + tag: ${{ matrix.name }} diff --git a/docker/Dockerfile.devito b/docker/Dockerfile.devito index 672daf0a9b..b23e26dcf0 100644 --- a/docker/Dockerfile.devito +++ b/docker/Dockerfile.devito @@ -3,6 +3,7 @@ ############################################################## # Base image with compilers +# Update default in .github/actions/docker-build if this value ever changes ARG base=devitocodes/bases:cpu-gcc FROM $base AS builder diff --git a/docker/coverage.env b/docker/coverage.env new file mode 100644 index 0000000000..f954033159 --- /dev/null +++ b/docker/coverage.env @@ -0,0 +1,23 @@ +# Generic code coverage variables +CODECOV_ENV +CODECOV_TOKEN +CODECOV_URL +CODECOV_SLUG +VCS_COMMIT_ID +VCS_BRANCH_NAME +VCS_PULL_REQUEST +VCS_SLUG +VCS_TAG +CI_BUILD_URL +CI_BUILD_ID +CI_JOB_ID + +# Github specific code coverage variables +GITHUB_ACTIONS +GITHUB_HEAD_REF +GITHUB_REF +GITHUB_REPOSITORY +GITHUB_RUN_ID +GITHUB_SERVER_URL +GITHUB_SHA +GITHUB_WORKFLOW From 0d6bda8407fde478fa9389b6f3d6eb69688ab3b4 Mon Sep 17 00:00:00 2001 From: JDBetteridge Date: Sat, 7 Mar 2026 02:18:20 +0000 Subject: [PATCH 02/11] ci: Test new actions in nompi workflow --- .github/workflows/pytest-core-nompi.yaml | 114 ++++++++++++----------- 1 file changed, 60 insertions(+), 54 deletions(-) diff --git a/.github/workflows/pytest-core-nompi.yaml b/.github/workflows/pytest-core-nompi.yaml index 9fa6284f13..0debf0dc84 100644 --- a/.github/workflows/pytest-core-nompi.yaml +++ b/.github/workflows/pytest-core-nompi.yaml @@ -27,9 +27,6 @@ jobs: DEVITO_LANGUAGE: ${{ matrix.language }} OMP_NUM_THREADS: 2 - outputs: - unique : ${{ steps.uniquetag.outputs.unique }} - strategy: # Prevent all build to stop if a single one fails fail-fast: false @@ -139,9 +136,11 @@ jobs: set: adjoint steps: + # TODO: This workflow should really be split into 2 (Docker and non-Docker) - name: Checkout devito uses: actions/checkout@v6 + # Bare metal (not Docker) - name: Set up Python ${{ matrix.python-version }} if: "!contains(matrix.name, 'docker')" uses: actions/setup-python@v6 @@ -149,45 +148,13 @@ jobs: python-version: ${{ matrix.python-version }} allow-prereleases: true - - name: Generate unique CI tag - id: uniquetag - run: | - UNIQUE=$(echo "${GITHUB_RUN_ID}_${GITHUB_RUN_ATTEMPT}" | cksum | cut -f 1 -d " ") - echo "Unique ID: ${UNIQUE}" - echo "unique=${UNIQUE}" >> "$GITHUB_OUTPUT" - - - name: Build docker image - if: contains(matrix.name, 'docker') - env: - UNIQUE: ${{ steps.uniquetag.outputs.unique }} - run: | - docker build \ - --file docker/Dockerfile.devito \ - --tag "devito_img${UNIQUE}" \ - --build-arg base=devitocodes/bases:cpu-${{ matrix.arch }} \ - . - - - name: Set run prefix - env: - UNIQUE: ${{ steps.uniquetag.outputs.unique }} - run: | - if [[ "${{ matrix.name }}" =~ "docker" ]]; then - echo "RUN_CMD=docker run \ - --init -t --rm \ - --env CODECOV_TOKEN=${{ secrets.CODECOV_TOKEN }} \ - --name testrun \ - devito_img${UNIQUE}" >> "$GITHUB_ENV" - else - echo "RUN_CMD=" >> "$GITHUB_ENV" - fi - id: set-run - - name: Install ${{ matrix.arch }} compiler if: "runner.os == 'linux' && !contains(matrix.name, 'docker') && matrix.arch !='custom' " run : | sudo apt-get install -y ${{ matrix.arch }} - name: Set tests (reduced number for OSX) + if: "!contains(matrix.name, 'docker')" run : | if [ "${{ runner.os }}" == 'macOS' ]; then brew install llvm libomp @@ -196,6 +163,7 @@ jobs: id: set-tests - name: Set pip flags for latest python (3.12) + if: "!contains(matrix.name, 'docker')" run: | ver="${{ matrix.python-version }}" major=${ver%%.*} @@ -215,27 +183,17 @@ jobs: if: matrix.name == 'pytest-ubuntu-py310-gcc14-omp' run: python3 -m pip install ${{ env.PIPFLAGS }} numpy==1.26 - - name: Check Docker image Python version - if: "contains(matrix.name, 'docker')" - run: | - declared_pyver="${{ matrix.python-version }}" - actual_pyver=$(${{ env.RUN_CMD }} python3 --version | grep "Python " | cut -d' ' -f2 | cut -d'.' -f1,2) - echo "Declared Python version: $declared_pyver" - echo "Actual Python version: $actual_pyver" - if [ "$declared_pyver" != "$actual_pyver" ]; then - echo "Python version mismatch: declared $declared_pyver, image has $actual_pyver" - exit 1 - fi - - name: Check configuration + if: "!contains(matrix.name, 'docker')" run: | - ${{ env.RUN_CMD }} python3 \ + python3 \ -c "from devito import configuration; \ print(''.join(['%s: %s \n' % (k, v) for (k, v) in configuration.items()]))" - name: Test with pytest + if: "!contains(matrix.name, 'docker')" run: | - ${{ env.RUN_CMD }} pytest \ + pytest \ -k "${{ matrix.test-set }}" \ -m "not parallel" \ --cov \ @@ -250,9 +208,57 @@ jobs: token: ${{ secrets.CODECOV_TOKEN }} name: ${{ matrix.name }} - - name: Cleanup Docker + # Docker + - id: build + name: Build docker image + if: contains(matrix.name, 'docker') + uses: ./.github/actions/docker-build + with: + file: docker/Dockerfile.devito + tag: devito_img + base: devitocodes/bases:cpu-${{ matrix.arch }} + + - name: Get the Docker image Python version + if: "contains(matrix.name, 'docker')" + uses: ./.github/actions/docker-run + with: + uid: ${{ steps.build.outputs.unique }} + tag: devito_img + command: > + python3 --version | grep "Python " | cut -d' ' -f2 | cut -d'.' -f1,2 > dockerpythonversion.txt + + - name: Check Docker image Python version if: "contains(matrix.name, 'docker')" - env: - UNIQUE: ${{ steps.uniquetag.outputs.unique }} run: | - docker image rm -f "devito_img${UNIQUE}" + declared_pyver="${{ matrix.python-version }}" + actual_pyver=$(cat dockerpythonversion.txt) + echo "Declared Python version: $declared_pyver" + echo "Actual Python version: $actual_pyver" + if [ "$declared_pyver" != "$actual_pyver" ]; then + echo "Python version mismatch: declared $declared_pyver, image has $actual_pyver" + exit 1 + fi + + - name: Test with pytest + if: "contains(matrix.name, 'docker')" + uses: ./.github/actions/docker-run + with: + uid: ${{ steps.build.outputs.unique }} + env: | + CODECOV_TOKEN=${{ secrets.CODECOV_TOKEN }} + tag: devito_img + command: | + pytest \ + -k "${{ matrix.test-set }}" \ + -m "not parallel" \ + --cov \ + --cov-config=.coveragerc \ + --cov-report=xml \ + tests/ + + - name: Cleanup docker image + if: always() + uses: ./.github/actions/docker-clean + with: + uid: ${{ steps.build.outputs.unique }} + tag: devito_img From e07905720a1886bed61b349a1737286352f153cb Mon Sep 17 00:00:00 2001 From: JDBetteridge Date: Mon, 9 Mar 2026 18:32:05 +0000 Subject: [PATCH 03/11] ci: Split core-nompi into basic and docker --- .github/workflows/pytest-core-nompi.yaml | 111 +++++++++++++---------- 1 file changed, 63 insertions(+), 48 deletions(-) diff --git a/.github/workflows/pytest-core-nompi.yaml b/.github/workflows/pytest-core-nompi.yaml index 0debf0dc84..70f31fa7d6 100644 --- a/.github/workflows/pytest-core-nompi.yaml +++ b/.github/workflows/pytest-core-nompi.yaml @@ -18,7 +18,7 @@ on: - main jobs: - pytest: + test-nompi-basic: name: ${{ matrix.name }}-${{ matrix.set }} runs-on: "${{ matrix.os }}" @@ -33,16 +33,13 @@ jobs: matrix: name: [ - pytest-ubuntu-py311-gcc11-cxxnoomp, - pytest-ubuntu-py312-gcc12-cxxomp, + pytest-osx-py312-clang-omp, pytest-ubuntu-py310-gcc14-omp, + pytest-ubuntu-py310-gcc9-omp, pytest-ubuntu-py311-gcc10-noomp, + pytest-ubuntu-py311-gcc11-cxxnoomp, + pytest-ubuntu-py312-gcc12-cxxomp, pytest-ubuntu-py312-gcc13-omp, - pytest-ubuntu-py310-gcc9-omp, - pytest-osx-py312-clang-omp, - pytest-docker-py310-gcc-omp, - pytest-docker-py310-gcc-omp-arm64, - pytest-docker-py310-icx-omp, pytest-ubuntu-py313-gcc14-omp ] set: [base, adjoint] @@ -97,27 +94,6 @@ jobs: language: "openmp" sympy: "1.12" - - name: pytest-docker-py310-gcc-omp - python-version: '3.10' - os: ubuntu-latest - arch: "gcc" - language: "openmp" - sympy: "1.13" - - - name: pytest-docker-py310-gcc-omp-arm64 - python-version: '3.10' - os: ubuntu-24.04-arm - arch: "gcc" - language: "openmp" - sympy: "1.13" - - - name: pytest-docker-py310-icx-omp - python-version: '3.10' - os: ubuntu-latest - arch: "icx" - language: "openmp" - sympy: "1.13" - - name: pytest-ubuntu-py313-gcc14-omp python-version: '3.13' os: ubuntu-24.04 @@ -136,25 +112,21 @@ jobs: set: adjoint steps: - # TODO: This workflow should really be split into 2 (Docker and non-Docker) - name: Checkout devito uses: actions/checkout@v6 - # Bare metal (not Docker) - name: Set up Python ${{ matrix.python-version }} - if: "!contains(matrix.name, 'docker')" uses: actions/setup-python@v6 with: python-version: ${{ matrix.python-version }} allow-prereleases: true - name: Install ${{ matrix.arch }} compiler - if: "runner.os == 'linux' && !contains(matrix.name, 'docker') && matrix.arch !='custom' " + if: "runner.os == 'linux' && matrix.arch !='custom' " run : | sudo apt-get install -y ${{ matrix.arch }} - name: Set tests (reduced number for OSX) - if: "!contains(matrix.name, 'docker')" run : | if [ "${{ runner.os }}" == 'macOS' ]; then brew install llvm libomp @@ -163,7 +135,6 @@ jobs: id: set-tests - name: Set pip flags for latest python (3.12) - if: "!contains(matrix.name, 'docker')" run: | ver="${{ matrix.python-version }}" major=${ver%%.*} @@ -173,7 +144,6 @@ jobs: fi - name: Install dependencies - if: "!contains(matrix.name, 'docker')" run: | python3 -m pip install ${{ env.PIPFLAGS }} --upgrade pip python3 -m pip install ${{ env.PIPFLAGS }} -e ".[tests,extras]" @@ -184,14 +154,12 @@ jobs: run: python3 -m pip install ${{ env.PIPFLAGS }} numpy==1.26 - name: Check configuration - if: "!contains(matrix.name, 'docker')" run: | python3 \ -c "from devito import configuration; \ print(''.join(['%s: %s \n' % (k, v) for (k, v) in configuration.items()]))" - name: Test with pytest - if: "!contains(matrix.name, 'docker')" run: | pytest \ -k "${{ matrix.test-set }}" \ @@ -202,33 +170,81 @@ jobs: tests/ - name: Upload coverage to Codecov - if: "!contains(matrix.name, 'docker')" uses: codecov/codecov-action@v5 with: token: ${{ secrets.CODECOV_TOKEN }} name: ${{ matrix.name }} - # Docker + test-nompi-docker: + name: ${{ matrix.name }}-${{ matrix.set }} + runs-on: "${{ matrix.os }}" + + env: + DEVITO_ARCH: "${{ matrix.arch }}" + DEVITO_LANGUAGE: ${{ matrix.language }} + OMP_NUM_THREADS: 2 + + strategy: + # Prevent all build to stop if a single one fails + fail-fast: false + + matrix: + name: [ + pytest-docker-py310-gcc-omp, + pytest-docker-py310-gcc-omp-arm64, + pytest-docker-py310-icx-omp, + ] + set: [base, adjoint] + + include: + - name: pytest-docker-py310-gcc-omp + python-version: '3.10' + os: ubuntu-latest + arch: "gcc" + language: "openmp" + sympy: "1.13" + + - name: pytest-docker-py310-gcc-omp-arm64 + python-version: '3.10' + os: ubuntu-24.04-arm + arch: "gcc" + language: "openmp" + sympy: "1.13" + + - name: pytest-docker-py310-icx-omp + python-version: '3.10' + os: ubuntu-latest + arch: "icx" + language: "openmp" + sympy: "1.13" + + - set: base + test-set: 'not adjoint' + + - set: adjoint + test-set: 'adjoint' + + steps: + - name: Checkout devito + uses: actions/checkout@v6 + - id: build name: Build docker image - if: contains(matrix.name, 'docker') uses: ./.github/actions/docker-build with: file: docker/Dockerfile.devito - tag: devito_img + tag: ${{ matrix.name }} base: devitocodes/bases:cpu-${{ matrix.arch }} - name: Get the Docker image Python version - if: "contains(matrix.name, 'docker')" uses: ./.github/actions/docker-run with: uid: ${{ steps.build.outputs.unique }} - tag: devito_img + tag: ${{ matrix.name }} command: > python3 --version | grep "Python " | cut -d' ' -f2 | cut -d'.' -f1,2 > dockerpythonversion.txt - name: Check Docker image Python version - if: "contains(matrix.name, 'docker')" run: | declared_pyver="${{ matrix.python-version }}" actual_pyver=$(cat dockerpythonversion.txt) @@ -240,13 +256,12 @@ jobs: fi - name: Test with pytest - if: "contains(matrix.name, 'docker')" uses: ./.github/actions/docker-run with: uid: ${{ steps.build.outputs.unique }} + tag: ${{ matrix.name }} env: | CODECOV_TOKEN=${{ secrets.CODECOV_TOKEN }} - tag: devito_img command: | pytest \ -k "${{ matrix.test-set }}" \ @@ -261,4 +276,4 @@ jobs: uses: ./.github/actions/docker-clean with: uid: ${{ steps.build.outputs.unique }} - tag: devito_img + tag: ${{ matrix.name }} From b2916071f513430c02b5eabb1d12b86ea55aa067 Mon Sep 17 00:00:00 2001 From: JDBetteridge Date: Mon, 9 Mar 2026 19:57:08 +0000 Subject: [PATCH 04/11] ci: Use composite actions for GPU tests --- .github/actions/docker-run/action.yaml | 2 +- .github/workflows/pytest-gpu.yaml | 236 ++++++++++--------------- 2 files changed, 98 insertions(+), 140 deletions(-) diff --git a/.github/actions/docker-run/action.yaml b/.github/actions/docker-run/action.yaml index 70218699b2..b435db8242 100644 --- a/.github/actions/docker-run/action.yaml +++ b/.github/actions/docker-run/action.yaml @@ -14,7 +14,7 @@ inputs: env: description: "Environment variables to set inside the docker container, one environment variable per line" required: true - default: Dockerfile + default: "" tag: description: "Tag of the built image to use" required: true diff --git a/.github/workflows/pytest-gpu.yaml b/.github/workflows/pytest-gpu.yaml index 993fc53b1e..1faa0fa666 100644 --- a/.github/workflows/pytest-gpu.yaml +++ b/.github/workflows/pytest-gpu.yaml @@ -1,12 +1,8 @@ -# Runner information: +# Workflow information: # - OpenACC/OpenMP on NVIDIA runs on runners labeled `nvidiagpu` # - OpenMP on AMD runs on runners labeled `amdgpu` -# -# Changes vs original: -# * Respect CUDA_VISIBLE_DEVICES for NVIDIA jobs by passing it AND restricting Docker with --gpus "device=…" -# * Tag images and container names with ${{ runner.name }} to avoid cross-runner races and maximize cache reuse -# * Remove docker prune / global container deletes (we assume disk space is fine) -# * Add comments throughout +# - Respect CUDA_VISIBLE_DEVICES for NVIDIA jobs by passing it AND restricting Docker with --gpus "device=…" +# - Tag images and container names to avoid cross-runner races and maximize cache reuse name: CI-gpu @@ -31,15 +27,12 @@ on: description: "Run GPU tests" jobs: - build: + test-gpu-docker: name: ${{ matrix.name }} runs-on: - self-hosted - ${{ matrix.runner_label }} - outputs: - unique : ${{ steps.uniquetag.outputs.unique }} - strategy: fail-fast: false matrix: @@ -47,133 +40,98 @@ jobs: test_examples: ["examples/seismic/tti/tti_example.py examples/seismic/acoustic/acoustic_example.py examples/seismic/viscoacoustic/viscoacoustic_example.py examples/seismic/viscoelastic/viscoelastic_example.py examples/seismic/elastic/elastic_example.py"] include: - # -------------------- NVIDIA job -------------------- - - name: pytest-gpu-acc-nvidia - test_files: "tests/test_adjoint.py tests/test_gpu_common.py tests/test_gpu_openacc.py tests/test_operator.py::TestEstimateMemory" - base: "devitocodes/bases:nvidia-nvc12" - runner_label: nvidiagpu - test_drive_cmd: "nvidia-smi" - # Respect CUDA_VISIBLE_DEVICES and also hard-limit Docker to that device. - # NOTE: CUDA_VISIBLE_DEVICES must be set by the runner (systemd drop-in etc.). - dockerflags: >- - --init --rm -t - --name ${CONTAINER_BASENAME} - --gpus "device=${CUDA_VISIBLE_DEVICES:-all}" - - # -------------------- AMD job ----------------------- - - name: pytest-gpu-omp-amd - test_files: "tests/test_adjoint.py tests/test_gpu_common.py tests/test_gpu_openmp.py tests/test_operator.py::TestEstimateMemory" - runner_label: amdgpu - base: "devitocodes/bases:amd" - test_drive_cmd: "rocm-smi" - # Unchanged, still passes through required /dev nodes etc. - dockerflags: >- - --init --network=host - --device=/dev/kfd --device=/dev/dri - --ipc=host - --group-add video --group-add "$(getent group render | cut -d: -f3)" - --cap-add=SYS_PTRACE --security-opt seccomp=unconfined - --rm -t - --name ${CONTAINER_BASENAME} + # -------------------- NVIDIA job -------------------- + - name: pytest-gpu-acc-nvidia + test_files: "tests/test_adjoint.py tests/test_gpu_common.py tests/test_gpu_openacc.py tests/test_operator.py::TestEstimateMemory" + base: "devitocodes/bases:nvidia-nvc12" + runner_label: nvidiagpu + test_drive_cmd: "nvidia-smi" + # Respect CUDA_VISIBLE_DEVICES and also hard-limit Docker to that device. + # NOTE: CUDA_VISIBLE_DEVICES must be set by the runner (systemd drop-in etc.). + dockerflags: --gpus "device=${CUDA_VISIBLE_DEVICES:-all}" + + # -------------------- AMD job ----------------------- + - name: pytest-gpu-omp-amd + test_files: "tests/test_adjoint.py tests/test_gpu_common.py tests/test_gpu_openmp.py tests/test_operator.py::TestEstimateMemory" + runner_label: amdgpu + base: "devitocodes/bases:amd" + test_drive_cmd: "rocm-smi" + # Unchanged, still passes through required /dev nodes etc. + dockerflags: >- + --network=host + --device=/dev/kfd --device=/dev/dri + --ipc=host + --group-add video --group-add "$(getent group render | cut -d: -f3)" + --cap-add=SYS_PTRACE --security-opt seccomp=unconfined steps: - - name: Checkout devito - uses: actions/checkout@v6 - - - name: Generate unique CI tag - id: uniquetag - run: | - UNIQUE=$(echo "${GITHUB_RUN_ID}_${GITHUB_RUN_ATTEMPT}" | cksum | cut -f 1 -d " ") - echo "Unique ID: ${UNIQUE}" - echo "unique=${UNIQUE}" >> "$GITHUB_OUTPUT" - - - name: Set per-runner tags - env: - UNIQUE: ${{ steps.uniquetag.outputs.unique }} - run: | - echo "DOCKER_IMAGE=${{ matrix.name }}-${RUNNER_NAME// /_}-${UNIQUE}" >> "$GITHUB_ENV" - echo "CONTAINER_BASENAME=testrun-${{ matrix.name }}-${RUNNER_NAME// /_}-${{ github.sha }}" >> "$GITHUB_ENV" - - - name: Ensure buildx builder - run: | - docker buildx inspect "${RUNNER_NAME// /_}" >/dev/null 2>&1 || \ - docker buildx create --name "${RUNNER_NAME// /_}" --driver docker-container - docker buildx use "${RUNNER_NAME// /_}" - - - name: Build docker image - run: | - docker buildx build . \ - --builder "${RUNNER_NAME// /_}" \ - --load \ - --label ci-run="$GITHUB_RUN_ID" \ - --rm --pull \ - --file docker/Dockerfile.devito \ - --tag "${DOCKER_IMAGE}" \ - --build-arg base="${{ matrix.base }}" - - - name: Export CODECOV token - run: echo "CODECOV_TOKEN=${{ secrets.CODECOV_TOKEN }}" >> "$GITHUB_ENV" - - - name: Probe gpu - run: | - # Make sure CUDA_VISIBLE_DEVICES is at least *something* on NVIDIA - # runners; fall back to "all" so the driver probe does not fail. - if [[ "${{ matrix.runner_label }}" == "nvidiagpu" && -z "${CUDA_VISIBLE_DEVICES:-}" ]]; then - echo "CUDA_VISIBLE_DEVICES=all" >> "$GITHUB_ENV" - fi - - # Run a simple driver-probe command (nvidia-smi / rocm-smi) - docker rm -f "${CONTAINER_BASENAME}" 2>/dev/null || true - docker run ${{ matrix.dockerflags }} "${DOCKER_IMAGE}" ${{ matrix.test_drive_cmd }} - - - name: Test with pytest - env: - # Exported earlier in the job; needed inside the container for codecov - CODECOV_TOKEN: ${{ env.CODECOV_TOKEN }} - run: | - # Add Codecov’s environment variables (GITHUB_SHA, etc.) - ci_env=$(bash <(curl -s https://codecov.io/env)) - - # Run the test suite using the matrix-defined flags - docker run \ - ${{ matrix.dockerflags }} \ - "${ci_env}" \ - --env CI=true \ - --env PYTHONFAULTHANDLER=1 \ - --env DEVITO_LOGGING=DEBUG \ - --env CODECOV_TOKEN \ - "${DOCKER_IMAGE}" \ - pytest -vvv --capture=no --showlocals \ - --log-cli-level=DEBUG -o log_cli=true \ - --full-trace --durations=10 \ - --cov --cov-config=.coveragerc --cov-report=xml \ - ${{ matrix.test_files }} - - - name: Test examples - run: | - docker run \ - ${{ matrix.dockerflags }} \ - "${DOCKER_IMAGE}" \ - pytest ${{ matrix.test_examples }} - - - name: Test examples with MPI - run: | - docker run \ - ${{ matrix.dockerflags }} \ - --env DEVITO_MPI=1 \ - "${DOCKER_IMAGE}" \ - mpiexec -n 2 pytest ${{ matrix.test_examples }} - - - name: Builder & image cleanup (keep 3 days of cache) - if: always() - run: | - # Remove only the test image we built - docker rmi -f "${DOCKER_IMAGE}" || true - - # Classic image layers created in this job - docker image prune -f --filter label=ci-run="$GITHUB_RUN_ID" - - # BuildKit cache: target the per-runner builder explicitly - docker builder prune --builder "${RUNNER_NAME// /_}" \ - -f \ - --filter "until=72h" + - name: Checkout devito + uses: actions/checkout@v6 + + - id: build + name: Build docker image + uses: ./.github/actions/docker-build + with: + file: docker/Dockerfile.devito + tag: ${{ matrix.name }} + base: ${{ matrix.base }} + + - name: Probe GPU + uses: ./.github/actions/docker-run + with: + uid: ${{ steps.build.outputs.unique }} + tag: ${{ matrix.name }} + args: ${{ matrix.dockerflags }} + command: ${{ matrix.test_drive_cmd }} + + - name: Test with pytest + uses: ./.github/actions/docker-run + with: + uid: ${{ steps.build.outputs.unique }} + tag: ${{ matrix.name }} + args: ${{ matrix.dockerflags }} + env: | + CI=true + CODECOV_TOKEN=${{ secrets.CODECOV_TOKEN }} + DEVITO_LOGGING=DEBUG + PYTHONFAULTHANDLER=1 + command: | + pytest \ + -vvv \ + --capture=no \ + --showlocals \ + --log-cli-level=DEBUG \ + -o log_cli=true \ + --full-trace \ + --durations=10 \ + --cov \ + --cov-config=.coveragerc \ + --cov-report=xml \ + ${{ matrix.test_files }} + + - name: Test examples + uses: ./.github/actions/docker-run + with: + uid: ${{ steps.build.outputs.unique }} + tag: ${{ matrix.name }} + args: ${{ matrix.dockerflags }} + env: | + command: pytest ${{ matrix.test_examples }} + + - name: Test examples with MPI + uses: ./.github/actions/docker-run + with: + uid: ${{ steps.build.outputs.unique }} + tag: ${{ matrix.name }} + args: ${{ matrix.dockerflags }} + env: | + DEVITO_LOGGING=DEBUG + DEVITO_MPI=1 + command: mpiexec -n 2 pytest ${{ matrix.test_examples }} + + - name: Cleanup docker image + if: always() + uses: ./.github/actions/docker-clean + with: + uid: ${{ steps.build.outputs.unique }} + tag: ${{ matrix.name }} From 8390d9ca4f22d38f4a2ff836f0c32aa93c22effe Mon Sep 17 00:00:00 2001 From: Jack Betteridge Date: Fri, 13 Mar 2026 19:17:49 +0000 Subject: [PATCH 05/11] ci: Use composite actions for tutorials --- .github/actions/docker-build/action.yaml | 1 - .github/workflows/tutorials.yaml | 314 +++++++++++++++++------ 2 files changed, 235 insertions(+), 80 deletions(-) diff --git a/.github/actions/docker-build/action.yaml b/.github/actions/docker-build/action.yaml index 004ee0d0d2..71feceb70e 100644 --- a/.github/actions/docker-build/action.yaml +++ b/.github/actions/docker-build/action.yaml @@ -14,7 +14,6 @@ inputs: base: description: "Base docker image to build on top of" required: true - # Update default if docker/Dockerfile.devito ever changes default default: "devitocodes/bases:cpu-gcc" outputs: diff --git a/.github/workflows/tutorials.yaml b/.github/workflows/tutorials.yaml index 8562647c4c..7c51b5b887 100644 --- a/.github/workflows/tutorials.yaml +++ b/.github/workflows/tutorials.yaml @@ -18,7 +18,7 @@ on: - main jobs: - tutorials: + tutorials-basic: name: ${{ matrix.name }} runs-on: ${{ matrix.os }} @@ -26,17 +26,13 @@ jobs: DEVITO_ARCH: "${{ matrix.compiler }}" DEVITO_LANGUAGE: ${{ matrix.language }} - outputs: - unique : ${{ steps.uniquetag.outputs.unique }} - strategy: # Prevent all build to stop if a single one fails fail-fast: false matrix: name: [ tutos-ubuntu-gcc-py310, - tutos-osx-clang-py311, - tutos-docker-gcc-py310 + tutos-osx-clang-py311 ] include: @@ -44,136 +40,296 @@ jobs: os: ubuntu-latest compiler: gcc language: "openmp" - pyver: "3.10" + python-version: "3.10" - name: tutos-osx-clang-py311 os: macos-latest compiler: clang language: "C" - pyver: "3.11" - - - name: tutos-docker-gcc-py310 - os: ubuntu-latest - compiler: gcc - language: "openmp" - pyver: "3.10" + python-version: "3.11" steps: - name: Checkout devito uses: actions/checkout@v6 - - name: Generate unique CI tag - id: uniquetag - run: | - UNIQUE=$(echo "${GITHUB_RUN_ID}_${GITHUB_RUN_ATTEMPT}" | cksum | cut -f 1 -d " ") - echo "Unique ID: ${UNIQUE}" - echo "unique=${UNIQUE}" >> "$GITHUB_OUTPUT" - - - name: Set up Python ${{ matrix.pyver }} - if: "!contains(matrix.name, 'docker')" + - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v6 with: - python-version: ${{ matrix.pyver }} + python-version: ${{ matrix.python-version }} - uses: maxim-lobanov/setup-xcode@v1 if: runner.os == 'macOS' with: xcode-version: latest-stable - - name: Build docker image - if: "contains(matrix.name, 'docker')" - env: - UNIQUE: ${{ steps.uniquetag.outputs.unique }} - run: | - docker build \ - --pull \ - --file docker/Dockerfile.devito \ - --tag "devito_img${UNIQUE}" \ - . - - - name: Set run prefix - env: - UNIQUE: ${{ steps.uniquetag.outputs.unique }} - run: | - if [ "${{ matrix.name }}" == 'tutos-docker-gcc-py310' ]; then - echo "RUN_CMD=docker run \ - --init -t --rm \ - --name testrun \ - devito_img${UNIQUE}" >> "$GITHUB_ENV" - else - echo "RUN_CMD=" >> "$GITHUB_ENV" - fi - id: set-run - - name: Install dependencies - if: matrix.name != 'tutos-docker-gcc-py310' run: | python -m pip install --upgrade pip pip install -e ".[tests,extras]" pip install blosc - - name: Check Docker image Python version - if: matrix.name == 'tutos-docker-gcc-py310' - run: | - declared_pyver="${{ matrix.pyver }}" - actual_pyver=$(${{ env.RUN_CMD }} python3 --version | cut -d' ' -f2 | cut -d'.' -f1,2) - echo "Declared Python version: $declared_pyver" - echo "Actual Python version: $actual_pyver" - if [ "$declared_pyver" != "$actual_pyver" ]; then - echo "Python version mismatch: declared $declared_pyver, image has $actual_pyver" - exit 1 - fi - - name: Seismic notebooks run: | - ${{ env.RUN_CMD }} py.test --nbval -k 'not dask' -k 'not synthetics' examples/seismic/tutorials/ - ${{ env.RUN_CMD }} py.test --nbval examples/seismic/acoustic/accuracy.ipynb + pytest --nbval -k 'not dask' -k 'not synthetics' examples/seismic/tutorials/ + pytest --nbval examples/seismic/acoustic/accuracy.ipynb - name: Failing notebooks continue-on-error: true run: | - ${{ env.RUN_CMD }} py.test --nbval examples/seismic/tutorials/14_creating_synthetics.ipynb + pytest --nbval examples/seismic/tutorials/14_creating_synthetics.ipynb - name: Dask notebooks if: runner.os != 'macOS' run: | - ${{ env.RUN_CMD }} py.test --nbval examples/seismic/tutorials/*dask*.ipynb + pytest --nbval examples/seismic/tutorials/*dask*.ipynb - name: Self-adjoint notebooks run: | - ${{ env.RUN_CMD }} py.test --nbval examples/seismic/self_adjoint/ + pytest --nbval examples/seismic/self_adjoint/ - name: CFD notebooks run: | - ${{ env.RUN_CMD }} py.test --nbval examples/cfd + pytest --nbval examples/cfd - name: User api notebooks run: | - ${{ env.RUN_CMD }} py.test --nbval examples/userapi + pytest --nbval examples/userapi - name: Compiler notebooks run: | - ${{ env.RUN_CMD }} py.test --nbval examples/compiler + pytest --nbval examples/compiler - name: Finance notebooks run: | - ${{ env.RUN_CMD }} py.test --nbval examples/finance + pytest --nbval examples/finance - name: Performance notebooks run: | - ${{ env.RUN_CMD }} py.test --nbval examples/performance + pytest --nbval examples/performance - name: ABC Notebooks run: | - ${{ env.RUN_CMD }} py.test --nbval examples/seismic/abc_methods + pytest --nbval examples/seismic/abc_methods - name: Timestepping Notebooks run: | - ${{ env.RUN_CMD }} py.test --nbval examples/timestepping + pytest --nbval examples/timestepping + + tutorials-docker: + name: ${{ matrix.name }} + runs-on: ${{ matrix.os }} + + env: + DEVITO_ARCH: "${{ matrix.compiler }}" + DEVITO_LANGUAGE: ${{ matrix.language }} + + strategy: + # Prevent all build to stop if a single one fails + fail-fast: false + matrix: + name: [ + tutos-docker-gcc-py310 + ] + + include: + - name: tutos-docker-gcc-py310 + os: ubuntu-latest + compiler: gcc + language: "openmp" + python-version: "3.10" + + steps: + - name: Checkout devito + uses: actions/checkout@v6 + + - id: build + name: Build docker image + uses: ./.github/actions/docker-build + with: + file: docker/Dockerfile.devito + tag: ${{ matrix.name }} + + - name: Get the Docker image Python version + uses: ./.github/actions/docker-run + with: + uid: ${{ steps.build.outputs.unique }} + tag: ${{ matrix.name }} + command: > + python3 --version | grep "Python " | cut -d' ' -f2 | cut -d'.' -f1,2 > dockerpythonversion.txt - - name: Cleanup Docker - if: "contains(matrix.name, 'docker')" - env: - UNIQUE: ${{ steps.uniquetag.outputs.unique }} + - name: Check Docker image Python version run: | - docker image rm -f "devito_img${UNIQUE}" + declared_pyver="${{ matrix.python-version }}" + actual_pyver=$(cat dockerpythonversion.txt) + echo "Declared Python version: $declared_pyver" + echo "Actual Python version: $actual_pyver" + if [ "$declared_pyver" != "$actual_pyver" ]; then + echo "Python version mismatch: declared $declared_pyver, image has $actual_pyver" + exit 1 + fi + + - name: Test seismic notebooks + uses: ./.github/actions/docker-run + with: + uid: ${{ steps.build.outputs.unique }} + tag: ${{ matrix.name }} + env: | + CI=true + CODECOV_TOKEN=${{ secrets.CODECOV_TOKEN }} + DEVITO_LOGGING=DEBUG + PYTHONFAULTHANDLER=1 + command: | + pytest \ + --nbval \ + -k 'not dask' \ + -k 'not synthetics' \ + examples/seismic/tutorials/ + + - name: Test seismic accuracy notebook + uses: ./.github/actions/docker-run + with: + uid: ${{ steps.build.outputs.unique }} + tag: ${{ matrix.name }} + env: | + CI=true + CODECOV_TOKEN=${{ secrets.CODECOV_TOKEN }} + DEVITO_LOGGING=DEBUG + PYTHONFAULTHANDLER=1 + command: | + pytest --nbval examples/seismic/acoustic/accuracy.ipynb + + - name: Failing notebooks + continue-on-error: true + uses: ./.github/actions/docker-run + with: + uid: ${{ steps.build.outputs.unique }} + tag: ${{ matrix.name }} + env: | + CI=true + CODECOV_TOKEN=${{ secrets.CODECOV_TOKEN }} + DEVITO_LOGGING=DEBUG + PYTHONFAULTHANDLER=1 + command: | + pytest --nbval examples/seismic/tutorials/14_creating_synthetics.ipynb + + - name: Dask notebooks + uses: ./.github/actions/docker-run + with: + uid: ${{ steps.build.outputs.unique }} + tag: ${{ matrix.name }} + env: | + CI=true + CODECOV_TOKEN=${{ secrets.CODECOV_TOKEN }} + DEVITO_LOGGING=DEBUG + PYTHONFAULTHANDLER=1 + command: | + pytest --nbval examples/seismic/tutorials/*dask*.ipynb + + - name: Self-adjoint notebooks + uses: ./.github/actions/docker-run + with: + uid: ${{ steps.build.outputs.unique }} + tag: ${{ matrix.name }} + env: | + CI=true + CODECOV_TOKEN=${{ secrets.CODECOV_TOKEN }} + DEVITO_LOGGING=DEBUG + PYTHONFAULTHANDLER=1 + command: | + pytest --nbval examples/seismic/self_adjoint/ + + - name: CFD notebooks + uses: ./.github/actions/docker-run + with: + uid: ${{ steps.build.outputs.unique }} + tag: ${{ matrix.name }} + env: | + CI=true + CODECOV_TOKEN=${{ secrets.CODECOV_TOKEN }} + DEVITO_LOGGING=DEBUG + PYTHONFAULTHANDLER=1 + command: | + pytest --nbval examples/cfd + + - name: User api notebooks + uses: ./.github/actions/docker-run + with: + uid: ${{ steps.build.outputs.unique }} + tag: ${{ matrix.name }} + env: | + CI=true + CODECOV_TOKEN=${{ secrets.CODECOV_TOKEN }} + PYTHONFAULTHANDLER=1 + command: | + pytest --nbval examples/userapi + + - name: Compiler notebooks + uses: ./.github/actions/docker-run + with: + uid: ${{ steps.build.outputs.unique }} + tag: ${{ matrix.name }} + env: | + CI=true + CODECOV_TOKEN=${{ secrets.CODECOV_TOKEN }} + DEVITO_LOGGING=DEBUG + PYTHONFAULTHANDLER=1 + command: | + pytest --nbval examples/compiler + + - name: Finance notebooks + uses: ./.github/actions/docker-run + with: + uid: ${{ steps.build.outputs.unique }} + tag: ${{ matrix.name }} + env: | + CI=true + CODECOV_TOKEN=${{ secrets.CODECOV_TOKEN }} + DEVITO_LOGGING=DEBUG + PYTHONFAULTHANDLER=1 + command: | + pytest --nbval examples/finance + + - name: Performance notebooks + uses: ./.github/actions/docker-run + with: + uid: ${{ steps.build.outputs.unique }} + tag: ${{ matrix.name }} + env: | + CI=true + CODECOV_TOKEN=${{ secrets.CODECOV_TOKEN }} + PYTHONFAULTHANDLER=1 + command: | + pytest --nbval examples/performance + + - name: ABC Notebooks + uses: ./.github/actions/docker-run + with: + uid: ${{ steps.build.outputs.unique }} + tag: ${{ matrix.name }} + env: | + CI=true + CODECOV_TOKEN=${{ secrets.CODECOV_TOKEN }} + DEVITO_LOGGING=DEBUG + PYTHONFAULTHANDLER=1 + command: | + pytest --nbval examples/seismic/abc_methods + + - name: Timestepping Notebooks + uses: ./.github/actions/docker-run + with: + uid: ${{ steps.build.outputs.unique }} + tag: ${{ matrix.name }} + env: | + CI=true + CODECOV_TOKEN=${{ secrets.CODECOV_TOKEN }} + DEVITO_LOGGING=DEBUG + PYTHONFAULTHANDLER=1 + command: | + pytest --nbval examples/timestepping + + - name: Cleanup docker image + if: always() + uses: ./.github/actions/docker-clean + with: + uid: ${{ steps.build.outputs.unique }} + tag: ${{ matrix.name }} From b49219f4d25ecde14a33a26333bd255a3d6cc426 Mon Sep 17 00:00:00 2001 From: Jack Betteridge Date: Mon, 16 Mar 2026 14:54:55 +0000 Subject: [PATCH 06/11] misc: Make base optional --- .github/actions/docker-build/action.yaml | 17 ++++------------- docker/Dockerfile.devito | 1 - 2 files changed, 4 insertions(+), 14 deletions(-) diff --git a/.github/actions/docker-build/action.yaml b/.github/actions/docker-build/action.yaml index 71feceb70e..ded026c54f 100644 --- a/.github/actions/docker-build/action.yaml +++ b/.github/actions/docker-build/action.yaml @@ -13,8 +13,7 @@ inputs: required: true base: description: "Base docker image to build on top of" - required: true - default: "devitocodes/bases:cpu-gcc" + default: "" outputs: unique: @@ -35,20 +34,12 @@ runs: - id: dockerbuild name: "Build docker container" shell: bash + env: + BASE: ${{ inputs.base }} run: | docker build \ --pull \ --file ${{ inputs.file }} \ --tag ${{ inputs.tag }}_${{ steps.uniquetag.outputs.unique }} \ - --build-arg base=${{ inputs.base }} \ + ${BASE:+--build-arg base=$BASE} \ . - -# Do we need to be more specific? -#~ docker buildx build . \ - #~ --builder "${RUNNER_NAME// /_}" \ - #~ --load \ - #~ --label ci-run="$GITHUB_RUN_ID" \ - #~ --rm --pull \ - #~ --file docker/Dockerfile.devito \ - #~ --tag "${DOCKER_IMAGE}" \ - #~ --build-arg base="${{ matrix.base }}" diff --git a/docker/Dockerfile.devito b/docker/Dockerfile.devito index b23e26dcf0..672daf0a9b 100644 --- a/docker/Dockerfile.devito +++ b/docker/Dockerfile.devito @@ -3,7 +3,6 @@ ############################################################## # Base image with compilers -# Update default in .github/actions/docker-build if this value ever changes ARG base=devitocodes/bases:cpu-gcc FROM $base AS builder From 1dc1857dabdb3e0c25fc2158f69bdd5bc65c200f Mon Sep 17 00:00:00 2001 From: Jack Betteridge Date: Mon, 16 Mar 2026 16:15:28 +0000 Subject: [PATCH 07/11] ci: Add optional name parameter to the run composite action --- .github/actions/docker-run/action.yaml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/actions/docker-run/action.yaml b/.github/actions/docker-run/action.yaml index b435db8242..12fdd1410b 100644 --- a/.github/actions/docker-run/action.yaml +++ b/.github/actions/docker-run/action.yaml @@ -18,6 +18,9 @@ inputs: tag: description: "Tag of the built image to use" required: true + name: + description: "Name for docker to use when running the command" + default: "" command: description: "Command to execute inside of the docker container" required: true @@ -45,11 +48,13 @@ runs: - id: dockerrun name: "Run command ${{ inputs.command }} in ${{ inputs.tag }} docker container" shell: bash + env: + NAME: ${{ input.name }} run: | docker run \ --init -t --rm \ ${{ inputs.args }} \ - --name "ci-${{ inputs.name }}-${{ inputs.uid }}" \ + --name "ci-${NAME:-${{ inputs.tag }}}-${{ inputs.uid }}" \ --env-file=docker/coverage.env \ ${{ steps.processenv.outputs.env }} \ "${{ inputs.tag }}_${{ inputs.uid }}" \ From f8abdae4afd51bb97e1a46a340064025fe653989 Mon Sep 17 00:00:00 2001 From: Jack Betteridge Date: Mon, 16 Mar 2026 17:10:14 +0000 Subject: [PATCH 08/11] misc: Tweaks --- .github/actions/docker-run/action.yaml | 14 +++++++------- .github/workflows/examples-mpi.yaml | 3 +-- .github/workflows/examples.yaml | 5 ++--- .github/workflows/lint.yaml | 3 +-- .github/workflows/pytest-core-mpi.yaml | 3 +-- .github/workflows/pytest-core-nompi.yaml | 7 +++---- .github/workflows/pytest-gpu.yaml | 2 +- .github/workflows/tutorials.yaml | 7 +++---- 8 files changed, 19 insertions(+), 25 deletions(-) diff --git a/.github/actions/docker-run/action.yaml b/.github/actions/docker-run/action.yaml index 12fdd1410b..cca9599200 100644 --- a/.github/actions/docker-run/action.yaml +++ b/.github/actions/docker-run/action.yaml @@ -7,20 +7,20 @@ inputs: uid: description: "Unique identifier output from docker-build action" required: true + tag: + description: "Tag of the built image to use" + required: true + name: + description: "Name substring for docker to use when running the command" + default: "" args: - description: "Arguments to pass to `docker run`" + description: "Arguments to pass to `docker run`, `--init -t --rm` are always added" required: true default: "" env: description: "Environment variables to set inside the docker container, one environment variable per line" required: true default: "" - tag: - description: "Tag of the built image to use" - required: true - name: - description: "Name for docker to use when running the command" - default: "" command: description: "Command to execute inside of the docker container" required: true diff --git a/.github/workflows/examples-mpi.yaml b/.github/workflows/examples-mpi.yaml index 034a3f3663..d096b7bfff 100644 --- a/.github/workflows/examples-mpi.yaml +++ b/.github/workflows/examples-mpi.yaml @@ -14,8 +14,7 @@ concurrency: cancel-in-progress: true on: - # Trigger the workflow on push or pull request, - # but only for the main branch + # Trigger the workflow on push or pull request, but only for the main branch push: branches: - main diff --git a/.github/workflows/examples.yaml b/.github/workflows/examples.yaml index 827a205b37..93726541af 100644 --- a/.github/workflows/examples.yaml +++ b/.github/workflows/examples.yaml @@ -8,8 +8,7 @@ concurrency: cancel-in-progress: true on: - # Trigger the workflow on push or pull request, - # but only for the main branch + # Trigger the workflow on push or pull request, but only for the main branch push: branches: - main @@ -32,7 +31,7 @@ jobs: DEVITO_LANGUAGE: "openmp" strategy: - # Prevent all build to stop if a single one fails + # Prevent cancellation if a single workflow fails fail-fast: false steps: diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml index d2495741ac..5b5c038a86 100644 --- a/.github/workflows/lint.yaml +++ b/.github/workflows/lint.yaml @@ -7,8 +7,7 @@ concurrency: cancel-in-progress: true on: - # Trigger the workflow on push or pull request, - # but only for the main branch + # Trigger the workflow on push or pull request, but only for the main branch push: branches: - main diff --git a/.github/workflows/pytest-core-mpi.yaml b/.github/workflows/pytest-core-mpi.yaml index f21851b0a8..b9feaf88a1 100644 --- a/.github/workflows/pytest-core-mpi.yaml +++ b/.github/workflows/pytest-core-mpi.yaml @@ -8,8 +8,7 @@ concurrency: cancel-in-progress: true on: - # Trigger the workflow on push or pull request, - # but only for the main branch + # Trigger the workflow on push or pull request, but only for the main branch push: branches: - main diff --git a/.github/workflows/pytest-core-nompi.yaml b/.github/workflows/pytest-core-nompi.yaml index 70f31fa7d6..7517ddad8b 100644 --- a/.github/workflows/pytest-core-nompi.yaml +++ b/.github/workflows/pytest-core-nompi.yaml @@ -8,8 +8,7 @@ concurrency: cancel-in-progress: true on: - # Trigger the workflow on push or pull request, - # but only for the main branch + # Trigger the workflow on push or pull request, but only for the main branch push: branches: - main @@ -28,7 +27,7 @@ jobs: OMP_NUM_THREADS: 2 strategy: - # Prevent all build to stop if a single one fails + # Prevent cancellation if a single workflow fails fail-fast: false matrix: @@ -185,7 +184,7 @@ jobs: OMP_NUM_THREADS: 2 strategy: - # Prevent all build to stop if a single one fails + # Prevent cancellation if a single workflow fails fail-fast: false matrix: diff --git a/.github/workflows/pytest-gpu.yaml b/.github/workflows/pytest-gpu.yaml index 1faa0fa666..f9359c7aa1 100644 --- a/.github/workflows/pytest-gpu.yaml +++ b/.github/workflows/pytest-gpu.yaml @@ -56,7 +56,7 @@ jobs: runner_label: amdgpu base: "devitocodes/bases:amd" test_drive_cmd: "rocm-smi" - # Unchanged, still passes through required /dev nodes etc. + # Passes through required /dev nodes etc. dockerflags: >- --network=host --device=/dev/kfd --device=/dev/dri diff --git a/.github/workflows/tutorials.yaml b/.github/workflows/tutorials.yaml index 7c51b5b887..a10d8b56b6 100644 --- a/.github/workflows/tutorials.yaml +++ b/.github/workflows/tutorials.yaml @@ -8,8 +8,7 @@ concurrency: cancel-in-progress: true on: - # Trigger the workflow on push or pull request, - # but only for the main branch + # Trigger the workflow on push or pull request, but only for the main branch push: branches: - main @@ -27,7 +26,7 @@ jobs: DEVITO_LANGUAGE: ${{ matrix.language }} strategy: - # Prevent all build to stop if a single one fails + # Prevent cancellation if a single workflow fails fail-fast: false matrix: name: [ @@ -124,7 +123,7 @@ jobs: DEVITO_LANGUAGE: ${{ matrix.language }} strategy: - # Prevent all build to stop if a single one fails + # Prevent cancellation if a single workflow fails fail-fast: false matrix: name: [ From e722b59575c9b4fde20b5e34c15c3c051d74e29f Mon Sep 17 00:00:00 2001 From: Jack Betteridge Date: Mon, 16 Mar 2026 19:58:02 +0000 Subject: [PATCH 09/11] ci: Build needs to take additional args for pyarmored builds --- .github/actions/docker-build/action.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/actions/docker-build/action.yaml b/.github/actions/docker-build/action.yaml index ded026c54f..e1df6baf8d 100644 --- a/.github/actions/docker-build/action.yaml +++ b/.github/actions/docker-build/action.yaml @@ -14,6 +14,10 @@ inputs: base: description: "Base docker image to build on top of" default: "" + args: + description: "Arguments to pass to `docker build`" + required: true + default: "" outputs: unique: @@ -42,4 +46,5 @@ runs: --file ${{ inputs.file }} \ --tag ${{ inputs.tag }}_${{ steps.uniquetag.outputs.unique }} \ ${BASE:+--build-arg base=$BASE} \ + ${{ inputs.args }} \ . From b4035ec95b6938f723684fc84c249af5727cf92b Mon Sep 17 00:00:00 2001 From: Jack Betteridge Date: Fri, 20 Mar 2026 15:04:12 +0000 Subject: [PATCH 10/11] misc: Address review comments --- .github/actions/docker-run/action.yaml | 4 +- .github/actions/readme.md | 109 +++++++++++++ .github/workflows/docker-devito.yaml | 4 +- .github/workflows/examples-mpi.yaml | 12 +- .github/workflows/pytest-core-mpi.yaml | 44 ++++-- .github/workflows/pytest-core-nompi.yaml | 21 +-- .github/workflows/pytest-gpu.yaml | 13 +- .github/workflows/tutorials.yaml | 192 ++--------------------- 8 files changed, 175 insertions(+), 224 deletions(-) create mode 100644 .github/actions/readme.md diff --git a/.github/actions/docker-run/action.yaml b/.github/actions/docker-run/action.yaml index cca9599200..0da4e3c906 100644 --- a/.github/actions/docker-run/action.yaml +++ b/.github/actions/docker-run/action.yaml @@ -43,7 +43,7 @@ runs: done <<< "$ENV_INPUT" # Remove the leading space from the first concatenation ENV_STRING="${ENV_STRING# }" - echo "env=$ENV_STRING" >> "$GITHUB_OUTPUT" + echo "docker_environment_args=$ENV_STRING" >> "$GITHUB_OUTPUT" - id: dockerrun name: "Run command ${{ inputs.command }} in ${{ inputs.tag }} docker container" @@ -56,6 +56,6 @@ runs: ${{ inputs.args }} \ --name "ci-${NAME:-${{ inputs.tag }}}-${{ inputs.uid }}" \ --env-file=docker/coverage.env \ - ${{ steps.processenv.outputs.env }} \ + ${{ steps.processenv.outputs.docker_environment_args }} \ "${{ inputs.tag }}_${{ inputs.uid }}" \ ${{ inputs.command }} diff --git a/.github/actions/readme.md b/.github/actions/readme.md new file mode 100644 index 0000000000..d6f249963d --- /dev/null +++ b/.github/actions/readme.md @@ -0,0 +1,109 @@ +# Devito Actions + +The Devito actions can either be used from the Github interface: +```yaml +uses: devitocodes/devito/.github/actions/docker-build@main +``` + +Or internally from within the repository: +```yaml +uses: ./.github/actions/docker-run +``` + +## `docker-build` + +Inputs: + +- `file`: Dockerfile containing build instructions (default: `Dockerfile`) +- `tag`: Tag to add to the built image +- `base`: Base docker image to build on top of +- `args`: Arguments to pass to `docker build` + +Outputs: + +- `unique`: Unique identifier for the CI run (ie: `${{ steps.build.outputs.unique }}`) + +Example: + +```yaml +jobs: + test: + steps: + - id: build + name: Build docker image for Devito + uses: ./.github/actions/docker-build + with: + file: docker/Dockerfile.devito + tag: tag-related-to-test-config + base: base-image-to-build-from + args: "--more-docker-build-args" +``` + +## `docker-run` + +Inputs: + +- `uid`: Unique identifier output from docker-build action +- `tag`: Tag of the built image to use +- `name`: Name substring for docker to use when running the command (optional) +- `args`: Arguments to pass to `docker run`, `--init -t --rm` are always added +- `env`: Environment variables to set inside the docker container, one environment variable per line +-command: Command to execute inside of the docker container + +### Notes + +- The UID must be unique, easily obtained from build action +- The tag must match built image, easily obtained from build action +- If you provide a custom name `foo` the container name will be `ci-foo-UUUUUUUUUU` where UUUUUUUUUU is the UID +- The default args `--init -t --rm` are _always_ added +- Environment variables must be passed a single environment variable per line, best achieved with the (`|`) syntax in yaml +- Only a single command is executed, not a list of commands. Using `;` or `&&` will result in subsequent commands being executed outside of the docker environment + +Example: + +```yaml +jobs: + test: + steps: + - name: Run a command in a previously built Docker container + uses: ./.github/actions/docker-run + with: + uid: ${{ steps.build.outputs.unique }} + tag: tag-related-to-test-config + name: completely-optional-name + args: "--more-docker-run-args" + env: | + FOO=value1 + BAR=value2 + command: | + mpiexec -n 4 \ + python \ + my_complicated_script.py --arg1 -v +``` + +## `docker-clean` + +Inputs: + +- `uid`: Unique identifier output from docker-build action +- `tag`: Tag of the built image to use + +### Notes + +- UID must be unique, easily obtained from build action +- Tag must match built image, easily obtained from build action +- Use `if: always()` to always clean up the image, even if the workflow fails + +Example: + +```yaml +jobs: + test: + steps: + - name: Cleanup Docker image + if: always() + uses: ./.github/actions/docker-clean + with: + uid: ${{ steps.build.outputs.unique }} + tag: tag-related-to-test-config +``` diff --git a/.github/workflows/docker-devito.yaml b/.github/workflows/docker-devito.yaml index 2d3211953d..bff8a6e470 100644 --- a/.github/workflows/docker-devito.yaml +++ b/.github/workflows/docker-devito.yaml @@ -178,7 +178,7 @@ jobs: run: | docker run ${{ matrix.flag }} --rm -t --name "${CONTAINER_NAME}" \ devitocodes/devito:${{ matrix.tag }}-dev \ - pytest ${{ matrix.test }} + pytest -v ${{ matrix.test }} deploy-devito-manifest: needs: deploy-devito @@ -292,4 +292,4 @@ jobs: run: | docker run ${{ matrix.flag }} --rm -t --name "${CONTAINER_NAME}" \ devitocodes/devito:${{ matrix.tag }}-dev \ - pytest ${{ matrix.test }} + pytest -v ${{ matrix.test }} diff --git a/.github/workflows/examples-mpi.yaml b/.github/workflows/examples-mpi.yaml index d096b7bfff..1b63466341 100644 --- a/.github/workflows/examples-mpi.yaml +++ b/.github/workflows/examples-mpi.yaml @@ -71,15 +71,17 @@ jobs: ipcluster start --profile=mpi --engines=mpi -n 4 --daemonize # A few seconds to ensure workers are ready sleep 10 - py.test --nbval examples/mpi + pytest -v --nbval examples/mpi ipcluster stop --profile=mpi - name: Test seismic examples run: | - mpirun ${{ matrix.mpiarg }} pytest examples/seismic/tti/tti_example.py - mpirun ${{ matrix.mpiarg }} pytest examples/seismic/elastic/elastic_example.py - mpirun ${{ matrix.mpiarg }} pytest examples/seismic/viscoacoustic/viscoacoustic_example.py - mpirun ${{ matrix.mpiarg }} pytest examples/seismic/viscoelastic/viscoelastic_example.py + mpirun ${{ matrix.mpiarg }} \ + pytest -v \ + examples/seismic/tti/tti_example.py \ + examples/seismic/elastic/elastic_example.py \ + examples/seismic/viscoacoustic/viscoacoustic_example.py \ + examples/seismic/viscoelastic/viscoelastic_example.py - name: Test fwi examples with mpi run: | diff --git a/.github/workflows/pytest-core-mpi.yaml b/.github/workflows/pytest-core-mpi.yaml index b9feaf88a1..934c4e3abb 100644 --- a/.github/workflows/pytest-core-mpi.yaml +++ b/.github/workflows/pytest-core-mpi.yaml @@ -47,13 +47,22 @@ jobs: - name: Test with pytest run: | python3 scripts/clear_devito_cache.py - python3 -m pytest --cov --cov-config=.coveragerc --cov-report=xml -m parallel tests/ + python3 -m pytest \ + -v \ + --cov \ + --cov-config=.coveragerc \ + --cov-report=xml \ + -m parallel \ + tests/ - name: Test examples with MPI run: | python3 scripts/clear_devito_cache.py - DEVITO_MPI=1 mpirun -n 2 python3 -m pytest examples/seismic/acoustic - DEVITO_MPI=1 mpirun -n 2 python3 -m pytest examples/seismic/tti + DEVITO_MPI=1 mpirun -n 2 \ + python3 -m pytest \ + -v \ + examples/seismic/acoustic \ + examples/seismic/tti - name: Upload coverage to Codecov uses: codecov/codecov-action@v5 @@ -104,19 +113,15 @@ jobs: env: | CODECOV_TOKEN=${{ secrets.CODECOV_TOKEN }} OMP_NUM_THREADS=1 - command: "pytest tests/test_mpi.py" - - - name: Test acoustic example with MPI - uses: ./.github/actions/docker-run - with: - uid: ${{ steps.build.outputs.unique }} - tag: ${{ matrix.name }} - env: | - DEVITO_MPI=1 - OMP_NUM_THREADS=1 - command: "mpiexec -n 2 pytest examples/seismic/acoustic" - - - name: Test tti example with MPI + command: | + pytest \ + -v \ + --cov \ + --cov-config=.coveragerc \ + --cov-report=xml \ + tests/test_mpi.py + + - name: Test acoustic and TTI examples with MPI uses: ./.github/actions/docker-run with: uid: ${{ steps.build.outputs.unique }} @@ -124,7 +129,12 @@ jobs: env: | DEVITO_MPI=1 OMP_NUM_THREADS=1 - command: "mpiexec -n 2 pytest examples/seismic/tti" + command: | + mpiexec -n 2 \ + pytest \ + -v \ + examples/seismic/acoustic \ + examples/seismic/tti - name: Cleanup docker image if: always() diff --git a/.github/workflows/pytest-core-nompi.yaml b/.github/workflows/pytest-core-nompi.yaml index 7517ddad8b..bae57248f7 100644 --- a/.github/workflows/pytest-core-nompi.yaml +++ b/.github/workflows/pytest-core-nompi.yaml @@ -161,6 +161,7 @@ jobs: - name: Test with pytest run: | pytest \ + -v \ -k "${{ matrix.test-set }}" \ -m "not parallel" \ --cov \ @@ -235,25 +236,6 @@ jobs: tag: ${{ matrix.name }} base: devitocodes/bases:cpu-${{ matrix.arch }} - - name: Get the Docker image Python version - uses: ./.github/actions/docker-run - with: - uid: ${{ steps.build.outputs.unique }} - tag: ${{ matrix.name }} - command: > - python3 --version | grep "Python " | cut -d' ' -f2 | cut -d'.' -f1,2 > dockerpythonversion.txt - - - name: Check Docker image Python version - run: | - declared_pyver="${{ matrix.python-version }}" - actual_pyver=$(cat dockerpythonversion.txt) - echo "Declared Python version: $declared_pyver" - echo "Actual Python version: $actual_pyver" - if [ "$declared_pyver" != "$actual_pyver" ]; then - echo "Python version mismatch: declared $declared_pyver, image has $actual_pyver" - exit 1 - fi - - name: Test with pytest uses: ./.github/actions/docker-run with: @@ -263,6 +245,7 @@ jobs: CODECOV_TOKEN=${{ secrets.CODECOV_TOKEN }} command: | pytest \ + -v \ -k "${{ matrix.test-set }}" \ -m "not parallel" \ --cov \ diff --git a/.github/workflows/pytest-gpu.yaml b/.github/workflows/pytest-gpu.yaml index f9359c7aa1..78d25ff254 100644 --- a/.github/workflows/pytest-gpu.yaml +++ b/.github/workflows/pytest-gpu.yaml @@ -115,8 +115,10 @@ jobs: uid: ${{ steps.build.outputs.unique }} tag: ${{ matrix.name }} args: ${{ matrix.dockerflags }} - env: | - command: pytest ${{ matrix.test_examples }} + command: | + pytest \ + -v \ + ${{ matrix.test_examples }} - name: Test examples with MPI uses: ./.github/actions/docker-run @@ -125,9 +127,14 @@ jobs: tag: ${{ matrix.name }} args: ${{ matrix.dockerflags }} env: | + CI=true DEVITO_LOGGING=DEBUG DEVITO_MPI=1 - command: mpiexec -n 2 pytest ${{ matrix.test_examples }} + command: | + mpiexec -n 2 \ + pytest \ + -v \ + ${{ matrix.test_examples }} - name: Cleanup docker image if: always() diff --git a/.github/workflows/tutorials.yaml b/.github/workflows/tutorials.yaml index a10d8b56b6..0bc2d03ff3 100644 --- a/.github/workflows/tutorials.yaml +++ b/.github/workflows/tutorials.yaml @@ -67,52 +67,26 @@ jobs: pip install -e ".[tests,extras]" pip install blosc - - name: Seismic notebooks + - name: Test all notebooks run: | - pytest --nbval -k 'not dask' -k 'not synthetics' examples/seismic/tutorials/ - pytest --nbval examples/seismic/acoustic/accuracy.ipynb + pytest \ + -v \ + --nbval \ + -k 'not dask' \ + -k 'not synthetics' \ + -k 'not mpi' \ + examples - name: Failing notebooks + # TODO: Fix or skip this continue-on-error: true run: | - pytest --nbval examples/seismic/tutorials/14_creating_synthetics.ipynb + pytest -v --nbval examples/seismic/tutorials/14_creating_synthetics.ipynb - name: Dask notebooks if: runner.os != 'macOS' run: | - pytest --nbval examples/seismic/tutorials/*dask*.ipynb - - - name: Self-adjoint notebooks - run: | - pytest --nbval examples/seismic/self_adjoint/ - - - name: CFD notebooks - run: | - pytest --nbval examples/cfd - - - name: User api notebooks - run: | - pytest --nbval examples/userapi - - - name: Compiler notebooks - run: | - pytest --nbval examples/compiler - - - name: Finance notebooks - run: | - pytest --nbval examples/finance - - - name: Performance notebooks - run: | - pytest --nbval examples/performance - - - name: ABC Notebooks - run: | - pytest --nbval examples/seismic/abc_methods - - - name: Timestepping Notebooks - run: | - pytest --nbval examples/timestepping + pytest -v --nbval examples/seismic/tutorials/*dask*.ipynb tutorials-docker: name: ${{ matrix.name }} @@ -148,25 +122,6 @@ jobs: file: docker/Dockerfile.devito tag: ${{ matrix.name }} - - name: Get the Docker image Python version - uses: ./.github/actions/docker-run - with: - uid: ${{ steps.build.outputs.unique }} - tag: ${{ matrix.name }} - command: > - python3 --version | grep "Python " | cut -d' ' -f2 | cut -d'.' -f1,2 > dockerpythonversion.txt - - - name: Check Docker image Python version - run: | - declared_pyver="${{ matrix.python-version }}" - actual_pyver=$(cat dockerpythonversion.txt) - echo "Declared Python version: $declared_pyver" - echo "Actual Python version: $actual_pyver" - if [ "$declared_pyver" != "$actual_pyver" ]; then - echo "Python version mismatch: declared $declared_pyver, image has $actual_pyver" - exit 1 - fi - - name: Test seismic notebooks uses: ./.github/actions/docker-run with: @@ -175,27 +130,15 @@ jobs: env: | CI=true CODECOV_TOKEN=${{ secrets.CODECOV_TOKEN }} - DEVITO_LOGGING=DEBUG PYTHONFAULTHANDLER=1 command: | pytest \ + -v \ --nbval \ -k 'not dask' \ -k 'not synthetics' \ - examples/seismic/tutorials/ - - - name: Test seismic accuracy notebook - uses: ./.github/actions/docker-run - with: - uid: ${{ steps.build.outputs.unique }} - tag: ${{ matrix.name }} - env: | - CI=true - CODECOV_TOKEN=${{ secrets.CODECOV_TOKEN }} - DEVITO_LOGGING=DEBUG - PYTHONFAULTHANDLER=1 - command: | - pytest --nbval examples/seismic/acoustic/accuracy.ipynb + -k 'not mpi' \ + examples - name: Failing notebooks continue-on-error: true @@ -206,10 +149,9 @@ jobs: env: | CI=true CODECOV_TOKEN=${{ secrets.CODECOV_TOKEN }} - DEVITO_LOGGING=DEBUG PYTHONFAULTHANDLER=1 command: | - pytest --nbval examples/seismic/tutorials/14_creating_synthetics.ipynb + pytest -v --nbval examples/seismic/tutorials/14_creating_synthetics.ipynb - name: Dask notebooks uses: ./.github/actions/docker-run @@ -222,109 +164,7 @@ jobs: DEVITO_LOGGING=DEBUG PYTHONFAULTHANDLER=1 command: | - pytest --nbval examples/seismic/tutorials/*dask*.ipynb - - - name: Self-adjoint notebooks - uses: ./.github/actions/docker-run - with: - uid: ${{ steps.build.outputs.unique }} - tag: ${{ matrix.name }} - env: | - CI=true - CODECOV_TOKEN=${{ secrets.CODECOV_TOKEN }} - DEVITO_LOGGING=DEBUG - PYTHONFAULTHANDLER=1 - command: | - pytest --nbval examples/seismic/self_adjoint/ - - - name: CFD notebooks - uses: ./.github/actions/docker-run - with: - uid: ${{ steps.build.outputs.unique }} - tag: ${{ matrix.name }} - env: | - CI=true - CODECOV_TOKEN=${{ secrets.CODECOV_TOKEN }} - DEVITO_LOGGING=DEBUG - PYTHONFAULTHANDLER=1 - command: | - pytest --nbval examples/cfd - - - name: User api notebooks - uses: ./.github/actions/docker-run - with: - uid: ${{ steps.build.outputs.unique }} - tag: ${{ matrix.name }} - env: | - CI=true - CODECOV_TOKEN=${{ secrets.CODECOV_TOKEN }} - PYTHONFAULTHANDLER=1 - command: | - pytest --nbval examples/userapi - - - name: Compiler notebooks - uses: ./.github/actions/docker-run - with: - uid: ${{ steps.build.outputs.unique }} - tag: ${{ matrix.name }} - env: | - CI=true - CODECOV_TOKEN=${{ secrets.CODECOV_TOKEN }} - DEVITO_LOGGING=DEBUG - PYTHONFAULTHANDLER=1 - command: | - pytest --nbval examples/compiler - - - name: Finance notebooks - uses: ./.github/actions/docker-run - with: - uid: ${{ steps.build.outputs.unique }} - tag: ${{ matrix.name }} - env: | - CI=true - CODECOV_TOKEN=${{ secrets.CODECOV_TOKEN }} - DEVITO_LOGGING=DEBUG - PYTHONFAULTHANDLER=1 - command: | - pytest --nbval examples/finance - - - name: Performance notebooks - uses: ./.github/actions/docker-run - with: - uid: ${{ steps.build.outputs.unique }} - tag: ${{ matrix.name }} - env: | - CI=true - CODECOV_TOKEN=${{ secrets.CODECOV_TOKEN }} - PYTHONFAULTHANDLER=1 - command: | - pytest --nbval examples/performance - - - name: ABC Notebooks - uses: ./.github/actions/docker-run - with: - uid: ${{ steps.build.outputs.unique }} - tag: ${{ matrix.name }} - env: | - CI=true - CODECOV_TOKEN=${{ secrets.CODECOV_TOKEN }} - DEVITO_LOGGING=DEBUG - PYTHONFAULTHANDLER=1 - command: | - pytest --nbval examples/seismic/abc_methods - - - name: Timestepping Notebooks - uses: ./.github/actions/docker-run - with: - uid: ${{ steps.build.outputs.unique }} - tag: ${{ matrix.name }} - env: | - CI=true - CODECOV_TOKEN=${{ secrets.CODECOV_TOKEN }} - DEVITO_LOGGING=DEBUG - PYTHONFAULTHANDLER=1 - command: | - pytest --nbval examples/timestepping + pytest -v --nbval examples/seismic/tutorials/*dask*.ipynb - name: Cleanup docker image if: always() From 55cae25cac9b6a3541745573f5a8a105e1d9ba18 Mon Sep 17 00:00:00 2001 From: Jack Betteridge Date: Fri, 20 Mar 2026 19:36:04 +0000 Subject: [PATCH 11/11] ci: Multiple -k args to pytest --- .github/workflows/tutorials.yaml | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/.github/workflows/tutorials.yaml b/.github/workflows/tutorials.yaml index 0bc2d03ff3..067c9ab815 100644 --- a/.github/workflows/tutorials.yaml +++ b/.github/workflows/tutorials.yaml @@ -72,9 +72,7 @@ jobs: pytest \ -v \ --nbval \ - -k 'not dask' \ - -k 'not synthetics' \ - -k 'not mpi' \ + -k 'not dask and not synthetics and not mpi' \ examples - name: Failing notebooks @@ -135,9 +133,7 @@ jobs: pytest \ -v \ --nbval \ - -k 'not dask' \ - -k 'not synthetics' \ - -k 'not mpi' \ + -k 'not dask and not synthetics and not mpi' \ examples - name: Failing notebooks