xref: /petsc/.gitlab-ci.yml (revision 6f33641175f69f1db294cc9ba81c3f4ad4f81d49)

#
# stage-1 take only a few minutes; they do not run the full test suite or external packages.
#
# stage-2 runs on MCS systems and may take 10 to 15 minutes. They run the full test suite but with limited mixture of external packages
#
# stage-3 runs on MCS systems and may take an hour or more. They run the full test suite and heavily test external packages, utilize valgrind etc
#
# The stage-(n) tests are only started if all of the stage-(n-1) tests run without error
#   You can limit the testing by using the variable STAGE with value 1 or 2
#

stages:
  - stage-1
  - stage-2
  - stage-3
  - stage-4
variables:
  GIT_STRATEGY: fetch
  GIT_CLEAN_FLAGS: -ffdxq
  TIMEOUT: 450
  EXTRA_OPTIONS: -nox -nox_warning -malloc_dump

#
# The most basic template that most tests will expand upon
#

.test-basic:
  interruptible: true
  only:
    refs:
#     Set with CI/CD Shedules - New Schedule
      - schedules
      - api
#     Set with CI/CD Pipelines - Run Pipeline
      - web
      - merge_requests
  dependencies: []

.test:
  extends: .test-basic
  except:
    variables:
      # Skip if the docs-only label is attached to a merge request
      - $CI_MERGE_REQUEST_LABELS =~ /(^|,)docs-only($|,)/
      - $PETSC_CI_SCHEDULED =~ /yes/

check-ci-settings:
  extends: .test-basic
  stage: .pre
  tags:
    - gce-stage1
  script:
    - lib/petsc/bin/maint/check-ci-settings.sh

#
#  This provides the basic order of operations and options template for cloud based stage 1 tests.
#  Not all test-short need to follow this template but most will.
#

.stage-1:
  extends: .test
  stage: stage-1
  tags:
    - gce-u22-stage1
  before_script:
    - date
    - hostname
    - grep PRETTY_NAME /etc/os-release
    - nproc
    - lscpu
    - ccache --zero-stats
  script:
    - if [ ! -z ${LOAD_MODULES+x} ]; then module --trace load ${LOAD_MODULES}; module list; fi
    - if [ ! -z ${INIT_SCRIPT+x} ]; then echo "sourcing ${INIT_SCRIPT}"; if [ -f ${HOME}/${INIT_SCRIPT} ]; then source ${HOME}/${INIT_SCRIPT}; else source ${INIT_SCRIPT}; fi; fi
    - printf "PATH:$PATH\n"
    - printf "PYTHONPATH:$PYTHONPATH\n"
    - printf "CONFIG_OPTS:${CONFIG_OPTS}\n"
    - printf "PETSC_OPTIONS:$PETSC_OPTIONS\n"
    - if [ ! -z ${CI_MERGE_REQUEST_SOURCE_BRANCH_NAME+x} ]; then printf "CI_MERGE_REQUEST_SOURCE_BRANCH_NAME:$CI_MERGE_REQUEST_SOURCE_BRANCH_NAME\n" ;fi
    - ${PYTHON} ./configure --with-debugging=0 COPTFLAGS='-O3 -march=native' CXXOPTFLAGS='-O3 -march=native' FOPTFLAGS='-O3 -march=native' ${CONFIG_OPTS}
    - make CFLAGS="${MAKE_CFLAGS}" CXXFLAGS="${MAKE_CXXFLAGS}" FFLAGS="${MAKE_FFLAGS}" CUDAFLAGS="${MAKE_CUDAFLAGS}"
    - if [ ! -z ${ENABLE_CHECK+x} ]; then make CFLAGS="${MAKE_CFLAGS}" CXXFLAGS="${MAKE_CXXFLAGS}" FFLAGS="${MAKE_FFLAGS}" CUDAFLAGS="${MAKE_CUDAFLAGS}" check; fi
    - make CFLAGS="${MAKE_TEST_CFLAGS}" CXXFLAGS="${MAKE_TEST_CXXFLAGS}" FFLAGS="${MAKE_TEST_FFLAGS}" CUDAFLAGS="${MAKE_CUDAFLAGS}" allgtests-tap gmakesearch="${TEST_SEARCH}" TIMEOUT=${TIMEOUT}
  variables:
    PYTHON: python3
    MAKE_CFLAGS: -Werror
    MAKE_CXXFLAGS: -Werror -Wzero-as-null-pointer-constant
    MAKE_FFLAGS: -Werror
    MAKE_TEST_CFLAGS: -Werror
    MAKE_TEST_CXXFLAGS: -Werror
    MAKE_TEST_FFLAGS: -Werror
  after_script:
    - date
    - ccache --show-stats
  artifacts:
    reports:
      junit: arch-*/tests/testresults.xml
    name: "$CI_JOB_NAME"
    when: always
    paths:
    - arch-*/lib/petsc/conf/*.log
    - arch-*/lib/pkgconfig/petsc.pc
    - arch-*/tests/testresults.xml
    - arch-*/tests/test_*_tap.log
    - arch-*/tests/test_*_err.log
    expire_in: 4 days

#
# The following tests run on the cloud as part of stage-1.
#

ompi-cuda:
  extends: .stage-1
  tags:
  - cuda-stage1
  variables:
    CONFIG_OPTS: --with-mpi-dir=/nfs/gce/software/custom/linux-ubuntu22.04-x86_64/spack/opt/spack/linux-ubuntu22.04-x86_64/gcc-11.3.0/openmpi-4.1.3-qrpnszy --with-cuda=1 --with-cuda-arch=all-major --CUDAPPFLAGS=-Wno-deprecated-gpu-targets --with-fc=0
    TEST_SEARCH: snes_tutorials-ex19_cuda%
    ENABLE_CHECK: 1
    MAKE_CXXFLAGS: -Werror
    MAKE_CUDAFLAGS: -Xcompiler -Wall -Xcompiler -Werror

mpich-cxx-py3:
  extends: .stage-1
  variables:
    CONFIG_OPTS: --with-clanguage=cxx --with-fc=0 CXXFLAGS=-std=c++11 --with-strict-petscerrorcode
    TEST_SEARCH: snes_tutorials-ex48%
    LOAD_MODULES: gcc/12.1.0 mpich/4.1-gcc-12.1.0

mpich-cxx-mlib:
  extends: .stage-1
  variables:
    CONFIG_OPTS: --with-clanguage=cxx --with-single-library=0 --with-log=0 --with-info=0 --with-ctable=0 --with-is-color-value-type=short CXXFLAGS=-std=c++14
    TEST_SEARCH: snes_tutorials-ex48%
    LOAD_MODULES: gcc/12.1.0 mpich/4.1-gcc-12.1.0

uni-complex-float-int64:
  extends: .stage-1
  variables:
    CONFIG_OPTS: --with-mpi=0 --with-scalar-type=complex --with-precision=single --with-64-bit-indices
    TEST_SEARCH: ts_tutorials-ex11_adv_2d_quad_%
    LOAD_MODULES: gcc/12.1.0 mpich/4.1-gcc-12.1.0

c99-mlib-static-py:
  extends: .stage-1
  variables:
    CONFIG_OPTS: --with-single-library=0 --with-shared-libraries=0 --with-strict-petscerrorcode CFLAGS=-std=c99 FFLAGS= CXXFLAGS=
    ENABLE_CHECK: 1
    TEST_SEARCH: snes_tutorials-ex19% snes_tutorials-ex5f90t%
    MAKE_TEST_FFLAGS: -Werror -std=f2008
    LOAD_MODULES: gcc/12.1.0 mpich/4.1-gcc-12.1.0

clang-uni:
  extends: .stage-1
  variables:
    CONFIG_OPTS: --with-cc=clang --with-cxx=clang++ --with-mpi=0 --with-strict-petscerrorcode
    TEST_SEARCH: snes_tutorials-ex48%

gcc-lto:
  extends: .stage-1
  variables:
    CONFIG_OPTS: CFLAGS= CXXFLAGS= FFLAGS=
    MAKE_CFLAGS: -Werror -flto=4 -Werror=odr -Werror=lto-type-mismatch
    MAKE_CXXFLAGS: -Werror -flto=4 -Werror=odr -Werror=lto-type-mismatch
    MAKE_FFLAGS: -Werror -flto=4 -Werror=odr -Werror=lto-type-mismatch
    TEST_SEARCH: snes_tutorials-ex19% snes_tutorials-ex5f%
    LOAD_MODULES: gcc/12.1.0 mpich/4.1-gcc-12.1.0

petsc4py-install:
  extends: .stage-2
  tags:
    - gce-u22-stage1
  variables:
    PYTHON: python3
    T_PREFIX: petsc-install
    T_DESTDIR: petsc-destdir
    T_PETSC4PY: src/binding/petsc4py
    T_VALIDATE_CMD: lib/petsc/bin/maint/validate-petsc4py.sh
  script:
    - printf "PATH:$PATH\n"
    - mkdir hide
    - if [ ! -z ${CI_MERGE_REQUEST_SOURCE_BRANCH_NAME+x} ]; then printf "CI_MERGE_REQUEST_SOURCE_BRANCH_NAME:$CI_MERGE_REQUEST_SOURCE_BRANCH_NAME\n" ;fi
    - ${PYTHON} ./configure --prefix="${PWD}/${T_PREFIX}" --with-petsc4py=1 --with-debugging=0
    - make CFLAGS=-Werror CXXFLAGS="-Werror -Wzero-as-null-pointer-constant" FFLAGS=-Werror
    - printf "====== Test A. Install into staging directory then into prefix directory (mimics distro package recipe) =====\n"
    - make install DESTDIR="${PWD}/${T_DESTDIR}"
    - test "$(find ${PWD}/${T_PREFIX} -mindepth 1 | wc -l)" -eq 0
    - test "$(find ${PWD}/${T_DESTDIR} -mindepth 1 | wc -l)" -gt 0
    - (cd "${PWD}/${T_DESTDIR}" && find * -exec install -v -D "{}" "/{}" \; )
    - mv arch-* hide/ && PETSC_DIR="${PWD}/${T_PREFIX}" ${T_VALIDATE_CMD} && mv hide/arch-* ./
    - rm -r "${PWD}/${T_PREFIX}" "${PWD}/${T_DESTDIR}" && mkdir "${PWD}/${T_PREFIX}"
    - printf "====== Test B. Install directly into prefix directory =====\n"
    - make install
    - test "$(find ${PWD}/${T_PREFIX} -mindepth 1 | wc -l)" -gt 0
    - mv arch-* hide/ && PETSC_DIR="${PWD}/${T_PREFIX}" ${T_VALIDATE_CMD} && mv hide/arch-* ./
    - rm -r "${PWD}/${T_PREFIX}"/*/petsc4py*
    - printf "====== Test C. Install manually with setuptools =====\n"
    - export PETSC_DIR="${PWD}/${T_PREFIX}" && (cd "${T_PETSC4PY}" && ${PYTHON} setup.py build)
    - export PETSC_DIR="${PWD}/${T_PREFIX}" P="${PWD}" && (cd "${T_PETSC4PY}" && ${PYTHON} setup.py install --install-lib="${P}/${T_PREFIX}/lib")
    - mv arch-* hide/ && PETSC_DIR="${PWD}/${T_PREFIX}" ${T_VALIDATE_CMD} && mv hide/arch-* ./
    - rm -r "${PWD}/${T_PREFIX}"/*/petsc4py*
    - printf "====== Test D. Install manually with setuptools with staging =====\n"
    - export PETSC_DIR="${PWD}/${T_PREFIX}" P="${PWD}" && (cd "${T_PETSC4PY}" && ${PYTHON} setup.py install --root="${P}/${T_DESTDIR}" --install-lib="${P}/${T_PREFIX}/lib")
    - (cd "${PWD}/${T_DESTDIR}" && find * -exec install -v -D "{}" "/{}" \; )
    - mv arch-* hide/ && PETSC_DIR="${PWD}/${T_PREFIX}" ${T_VALIDATE_CMD} && mv hide/arch-* ./

petsc4py-pip:
  extends: .stage-2
  tags:
    - gce-u22-stage1
  variables:
    PETSC_CONFIGURE_OPTIONS: --with-mpi=0 --with-fc=0
    CFLAGS: -O0
  script:
    - python3 -m venv pip-builds
    - source pip-builds/bin/activate
    - python -m pip install --upgrade pip
    - unset PETSC_DIR PETSC_ARCH
    - export PIP_WHEEL_DIR="${PWD}/wheelhouse"
    - export PIP_FIND_LINKS="${PIP_WHEEL_DIR}"
    - mkdir -p "${PIP_WHEEL_DIR}"
    - python -m pip wheel .
    - python -m pip wheel src/binding/petsc4py
    - python -m pip install --pre --no-index --no-cache-dir petsc4py
    - python -m pip list
    - python -m petsc --prefix
    - python -m petsc4py

checksource:
  extends: .test-basic
  stage: .pre
  tags:
    - gce-u22-stage1
  script:
    - python3 ./configure --with-mpi=0 --with-fc=0 --with-cxx=0 --with-x=0 --with-bison=0 --with-cmake=0 --with-pthread=0 --with-regex=0
    - vermin --version
    - make vermin
    - PATH=/nfs/gce/projects/petsc/soft/u22.04/spack-2024-03-06/opt/spack/linux-ubuntu22.04-x86_64/gcc-11.4.0/llvm-18.1.0-7hsf4hzma6aebqsd4ifb5brchcooktkp/bin:$PATH make checkclangformat SHELL=bash
    - make checkbadSource SHELL=bash
    - make checkbadFileChange SHELL=bash
    - make -f gmakefile check_output SHELL=bash
    - make check_petsc4py_rst
    - make -C src/binding/petsc4py lint
  artifacts:
    name: "$CI_JOB_NAME"
    when: always
    paths:
    - arch-*/lib/petsc/conf/*.patch
    - arch-*/lib/petsc/conf/*.log
    expire_in: 4 days

pause-for-approval:
  extends: .test
  stage: .pre
  tags:
    - gce-stage1
  only:
    refs:
      - merge_requests
    variables:
      - $CI_MERGE_REQUEST_EVENT_TYPE == "merged_result"
      - $CI_MERGE_REQUEST_EVENT_TYPE == "detached"
  script:
    - echo "pause-for-approval has no script to run"
  variables:
    GIT_STRATEGY: none
  when: manual
  allow_failure: false

#
# Basic template for code coverage generation. Tests should additionally inherit from the more
# specialized OS-specific versions instead of this one.
#

.coverage-generate:
  variables:
    RUN_GCOV: 1
    RUN_GCOV_OPTIONS: --decisions

.coverage-disable:
  variables:
    RUN_GCOV: 0

#
# This provides the basic order of operations and options template for stage-2,3 tests.
# Not all stage-2,3 need to follow this template, but most will.
#
.stage-23:
  extends:
    - .test
    - .coverage-generate
  script:
    - if [ ! -z ${LOAD_MODULES+x} ]; then module --trace load ${LOAD_MODULES}; module list; fi
    - if [ ! -z ${GCOV_MODULES+x} ]; then module --trace load ${GCOV_MODULES}; module list; fi
    - if [ ! -z ${INIT_SCRIPT+x} ]; then echo "sourcing ${INIT_SCRIPT}"; if [ -f ${HOME}/${INIT_SCRIPT} ]; then source ${HOME}/${INIT_SCRIPT}; else source ${INIT_SCRIPT}; fi; fi
    - if [ "${RUN_GCOV}" == 1 ]; then
        printf "RUN_GCOV_OPTIONS:$RUN_GCOV_OPTIONS\n";
        gcovr --version;
      fi
    - if [ -d ${HOME}/petsc-hash-pkgs/ ]; then find ${HOME}/petsc-hash-pkgs/ -maxdepth 1 -mindepth 1 -type d -mtime +25 -exec rm -rf {} \; ; fi
    - printf "PATH:$PATH\n"
    - printf "PYTHONPATH:$PYTHONPATH\n"
    - printf "PETSC_OPTIONS:$PETSC_OPTIONS\n"
    - if [ ! -z ${CI_MERGE_REQUEST_SOURCE_BRANCH_NAME+x} ]; then printf "CI_MERGE_REQUEST_SOURCE_BRANCH_NAME:$CI_MERGE_REQUEST_SOURCE_BRANCH_NAME\n" ;fi
    - printf "./config/examples/${TEST_ARCH}.py\n"
    - cat ./config/examples/${TEST_ARCH}.py
    - ./config/examples/${TEST_ARCH}.py || if [ -f naglicenseproblem ]; then exit 126; else exit 1; fi
    - make CFLAGS="${MAKE_CFLAGS}" CXXFLAGS="${MAKE_CXXFLAGS}" FFLAGS="${MAKE_FFLAGS}" CUDAFLAGS="${MAKE_CUDAFLAGS}" || if [ -f naglicenseproblem ]; then exit 126; else exit 1; fi
    - export ASAN_OPTIONS="log_path=stdout:detect_leaks=false" UBSAN_OPTIONS="suppressions=${PWD}/share/petsc/suppressions/ubsan" # only needed for -fsanitize=undefined (but harmless otherwise)
    - make CFLAGS="${MAKE_CFLAGS}" CXXFLAGS="${MAKE_CXXFLAGS}" FFLAGS="${MAKE_FFLAGS}" CUDAFLAGS="${MAKE_CUDAFLAGS}" check || if [ -f naglicenseproblem ]; then exit 126; else exit 1; fi
    - make updatedatafiles
    - if [ -z ${DISABLE_PETSC4PY_TESTS+x} ]; then if [ -z ${ENABLE_PETSC4PY_LDPRELOAD+x} ]; then echo "NOT using LD_PRELOAD"; make petsc4pytest; else echo "using LD_PRELOAD"; LD_PRELOAD=${PWD}/${TEST_ARCH}/lib/libpetsc.so make petsc4pytest; fi; fi
    - if [ -z ${DISABLE_TESTS+x} ]; then make cleantest && make CFLAGS="${MAKE_CFLAGS}" CXXFLAGS="${MAKE_CXXFLAGS}" FFLAGS="${MAKE_FFLAGS}" CUDAFLAGS="${MAKE_CUDAFLAGS}" allgtests-tap TIMEOUT=${TIMEOUT} ${TEST_OPTS} || if [ -f naglicenseproblem ]; then exit 126; else exit 1; fi; fi
    - if [ "${RUN_GCOV}" == 1 ]; then make gcov PETSC_GCOV_OPTIONS="${RUN_GCOV_OPTIONS}"; fi
  artifacts:
    reports:
      junit: arch-*/tests/testresults.xml
    name: "$CI_JOB_NAME"
    when: always
    paths:
    - arch-*/lib/petsc/conf/*.log
    - arch-*/lib/pkgconfig/petsc.pc
    - arch-*/tests/testresults.xml
    - arch-*/tests/test_arch-*_tap.log
    - arch-*/tests/test_arch-*_err.log
    - arch-*/arch-*-gcovr-report.json.tar.bz2
    expire_in: 4 days
  variables:
    OPENBLAS_NUM_THREADS: 1

.stage-2:
  extends: .stage-23
  stage: stage-2

.stage-3:
  extends: .stage-23
  stage: stage-3

#
# The following provide templates for various OSes for pre/post info
#

.linux_test_noflags:
  before_script:
    - date
    - hostname
    - grep PRETTY_NAME /etc/os-release
    - nproc
    - lscpu
    - ccache --zero-stats
  after_script:
    - date
    - ccache --show-stats

.linux_test_nofflags:
  extends: .linux_test_noflags
  variables:
    MAKE_CFLAGS: -Werror
    MAKE_CXXFLAGS: -Werror

.linux_test:
  extends: .linux_test_noflags
  variables:
    MAKE_CFLAGS: -Werror
    MAKE_CXXFLAGS: -Werror
    MAKE_FFLAGS: -Werror

.freebsd_test:
  extends:
    - .coverage-disable
  variables:
    MAKE_CFLAGS: -Werror
    MAKE_CXXFLAGS: -Werror
    MAKE_FFLAGS: -Werror
  before_script:
    - date
    - hostname
    - freebsd-version
    - echo $(sysctl -n hw.ncpu)
    - ccache --zero-stats
  after_script:
    - date
    - ccache --show-stats

.osx_test:
  variables:
    MAKE_CFLAGS: -Werror
    MAKE_CXXFLAGS: -Werror
    MAKE_FFLAGS: -Werror
  before_script:
    - date
    - hostname
    - sw_vers -productVersion
    - echo $(sysctl -n hw.ncpu)
    - ccache --zero-stats
  after_script:
    - date
    - ccache --show-stats

.mswin_test:
  extends:
    - .coverage-disable
  before_script:
    - date
    - hostname
    - uname -a
    - nproc
  after_script:
    - date

#
# The following tests run as part of stage-2.
#
# The tags variable used in the tests below connects the particular test with the runners
# listed on the left hand side of https://gitlab.com/petsc/petsc/-/settings/ci_cd.
# For example the test 'linux-sycl-double' any runner that has the tags 'gce-u22-stage3, linux-fast'
# (in a blue box beneath it)
#

osx-arm:
  extends:
    - .stage-3
    - .osx_test
    - .coverage-disable
  tags:
    - os:macos-arm
  variables:
    TEST_ARCH: arch-ci-osx-arm
    PETSC_OPTIONS: -check_pointer_intensity 0 -error_output_stdout
    INIT_SCRIPT: .zprofile

freebsd-cxx-cmplx-64idx-dbg:
  extends:
    - .stage-2
    - .freebsd_test
  tags:
    - os:fbsd
  variables:
    TEST_ARCH: arch-ci-freebsd-cxx-cmplx-64idx-dbg

freebsd-c-single-opt:
  extends:
    - .stage-2
    - .freebsd_test
  tags:
    - os:fbsd
  variables:
    TEST_ARCH: arch-ci-freebsd-c-single-opt

mswin-opt-impi:
  extends:
    - .stage-3
    - .mswin_test
  tags:
    - mswin-stage3
  variables:
    TEST_ARCH: arch-ci-mswin-opt-impi
    TEST_OPTS: search='ksp_ksp_tests*'

linux-gcc-quad-64idx-dbg:
  extends:
    - .stage-2
    - .linux_test
  tags:
    - gce-stage2
  variables:
    TEST_ARCH: arch-ci-linux-gcc-quad-64idx-dbg

linux-gcc-pkgs-opt:
  extends:
    - .stage-2
    - .linux_test
  tags:
    - gce-stage2
  variables:
    TEST_ARCH: arch-ci-linux-gcc-pkgs-opt
  except:
    variables:
      - $PETSC_CI_SCHEDULED =~ /yes/

linux-gcc-complex-opt-32bit:
  extends:
    - .stage-2
    - .linux_test
  tags:
    - gce-stage2
  variables:
    TEST_ARCH: arch-ci-linux-gcc-complex-opt-32bit

linux-emcc:
  extends:
    - .stage-2
    - .linux_test
    - .coverage-disable
  tags:
    - gce-stage2
  variables:
    TEST_ARCH: arch-ci-linux-emcc
    INIT_SCRIPT: /nfs/gce/projects/petsc/soft/u22.04/emsdk/emsdk_env.sh
    DISABLE_TESTS: 1

#
# The following tests run as part of stage-3.
#

freebsd-cxx-cmplx-pkgs-dbg:
  extends:
    - .stage-3
    - .freebsd_test
  tags:
    - os:fbsd
  variables:
    TEST_ARCH: arch-ci-freebsd-cxx-cmplx-pkgs-dbg

freebsd-cxx-pkgs-opt:
  extends:
    - .stage-3
    - .freebsd_test
  tags:
    - os:fbsd
  variables:
    PETSC_OPTIONS: -vecscatter_mpi1 false -options_left false
    TEST_ARCH: arch-ci-freebsd-cxx-pkgs-opt

freebsd-pkgs-opt:
  extends:
    - .stage-3
    - .freebsd_test
  tags:
    - os:fbsd
  variables:
    TEST_ARCH: arch-ci-freebsd-pkgs-opt
    MAKE_CFLAGS: -DPETSC_SKIP_COMPLEX -Werror

linux-hip-double:
  extends:
    - .stage-3
    - .linux_test_noflags
    - .coverage-disable
  tags:
    - gpu:amd, os:linux
  variables:
    TEST_ARCH: arch-ci-linux-hip-double

linux-sycl-double:
  extends:
    - .stage-3
    - .linux_test_nofflags
    - .coverage-disable
  tags:
    - gce-u22-stage3, linux-fast
  variables:
    MODULEPATH: /nfs/gce/projects/petsc/soft/u22.04/oneapi/modulefiles
    TEST_ARCH: arch-ci-linux-sycl-double
    LOAD_MODULES: compiler/latest mpi/latest
    TEST_OPTS: query=requires queryval=kokkos_kernels
    MAKE_CFLAGS: -Wno-expected-file-type -Werror

linux-cuda-pkgs:
  extends:
    - .stage-3
    - .linux_test
  tags:
    - cuda-stage3
  variables:
    TEST_ARCH: arch-ci-linux-cuda-pkgs
    # run all tests that:
    # 1. require cuda, OR
    # 2. require device, OR
    # 3. require hypre, OR
    # 4. require kokkos[_kernels], AND lives in the vec subdirectory
    #
    # need to use the alternate OR spelling since '|' is interpreted as a pipe by the
    # shell, and I could not manage to escape it enough times...
    TEST_OPTS: -j3 queryval='cuda,device,hypre,kokkos*,vec*' query='requires,requires,requires,requires%OR%name'
    MAKE_CUDAFLAGS: --compiler-options -Wall,-Werror

linux-cuda112-omp:
  extends:
    - .stage-3
    - .linux_test
    - .coverage-disable
  tags:
    - cuda-stage3
  variables:
    TEST_ARCH: arch-ci-linux-cuda112-omp
    TEST_OPTS: -j4 query=requires queryval=kokkos_kernels
    LOAD_MODULES: cuda/11.2.2 u-gcc-9/9.4.0
    OMP_PROC_BIND: "false"
    OMP_NUM_THREADS: 1
    MAKE_CUDAFLAGS: --compiler-options -Wall,-Werror

linux-cuda-cmplx:
  extends:
    - .stage-3
    - .linux_test
  tags:
    - cuda-stage3
  variables:
    TEST_ARCH: arch-ci-linux-cuda-cmplx
    MAKE_CUDAFLAGS: --compiler-options -Wall,-Werror

linux-cuda-double-64idx:
  extends:
    - .stage-3
    - .linux_test
  tags:
    - cuda12
  variables:
    TEST_ARCH: arch-ci-linux-cuda-double-64idx
    TEST_OPTS: -j3 query=requires queryval=cuda
    MAKE_CUDAFLAGS: --compiler-options -Wall,-Werror

linux-cuda-single-cxx:
  extends:
    - .stage-3
    - .linux_test_noflags
    - .coverage-disable
  tags:
    - cuda-stage3
  variables:
    TEST_ARCH: arch-ci-linux-cuda-single-cxx
    MODULEPATH: /nfs/gce/projects/petsc/soft/u22.04/spack-2023-02-02/share/spack/modules/linux-ubuntu22.04-ivybridge
    LOAD_MODULES: nvhpc-22.11-gcc-11.3.0-vmvmbrx
    TEST_OPTS: -j15 query=requires queryval=cuda

linux-cuda-uni-pkgs:
  extends:
    - .stage-3
    - .linux_test
  tags:
    - cuda-stage3
  variables:
    TEST_ARCH: arch-ci-linux-cuda-uni-pkgs
    TEST_OPTS: -j20 query=requires queryval=cuda
    MAKE_CUDAFLAGS: --compiler-options -Wall,-Werror

linux-clang-cuda:
  extends:
    - .stage-3
    - .linux_test
    - .coverage-disable
  tags:
    - cuda-stage3
  variables:
    TEST_ARCH: arch-ci-linux-clang-cuda
    TEST_OPTS: -j 1 queryval='cuda*,snes*' query='requires%OR%name'
    MODULEPATH: /nfs/gce/projects/petsc/soft/u22.04/spack-2023-02-02/share/spack/modules/linux-ubuntu22.04-ivybridge
    LOAD_MODULES: llvm-15.0.7-gcc-11.3.0-pjldtxf cuda-11.5.2-gcc-11.3.0-jkoottx gcc-11.3.0-gcc-11.3.0-pgrvke5
    MAKE_CUDAFLAGS: -Wno-pass-failed -Werror

linux-viennacl:
  extends:
    - .stage-3
    - .linux_test
  tags:
    - cuda-stage3
  variables:
    TEST_ARCH: arch-ci-linux-viennacl
    MAKE_CXXFLAGS: -Werror -Wno-ignored-attributes
    TEST_OPTS: -j20 query=requires queryval=viennacl

linux-without-fc:
  extends:
    - .stage-3
    - .linux_test
  tags:
    - gce-u22-stage3
  variables:
    TEST_ARCH: arch-ci-linux-without-fc

linux-cmplx-single-arm:
  extends:
    - .stage-3
    - .linux_test
    # gcovr does not work on arm/linux
    - .coverage-disable
  tags:
    - arch:arm, os:linux
  variables:
    FLEXIBLAS: netlib
    TEST_ARCH: arch-ci-linux-cmplx-single-arm

linux-gcc-cxx-avx2:
  extends:
    - .stage-3
    - .linux_test
  tags:
    - name:petsc-knl-01
  variables:
    TEST_ARCH: arch-ci-linux-gcc-cxx-avx2

linux-clang-avx:
  extends:
    - .stage-3
    - .linux_test
  tags:
    - name:petsc-knl-01
  variables:
    TEST_ARCH: arch-ci-linux-clang-avx

linux-knl:
  extends:
    - .stage-3
    - .linux_test
    - .coverage-disable
  tags:
    - name:petsc-knl-01
  variables:
    MAKE_FFLAGS: -warn errors
    TEST_ARCH: arch-ci-linux-knl
    LOAD_MODULES: intel-oneapi-compilers/2022.0.2 intel-oneapi-mkl/2022.0.2 intel-oneapi-mpi/2021.5.1

linux-intel-mkl-single:
  extends:
    - .stage-3
    - .linux_test_nofflags
    - .coverage-disable
  tags:
    - name:petsc-knl-01
  variables:
    TEST_ARCH: arch-ci-linux-intel-mkl-single
    LOAD_MODULES: intel-oneapi-compilers/2022.0.2 intel-oneapi-mkl/2022.0.2 intel-oneapi-mpi/2021.5.1

linux-cxx-cmplx-pkgs-64idx:
  extends:
    - .stage-3
    - .linux_test
  tags:
    - gce-u22-stage3
  variables:
    TEST_ARCH: arch-ci-linux-cxx-cmplx-pkgs-64idx
    PETSC_OPTIONS: -fp_trap

# Set env vars for 'Kokkos + OpenMP' tests, so that they won't give warnings
linux-pkgs-dbg-ftn-interfaces:
  extends:
    - .stage-3
    - .linux_test
    - .coverage-disable
  tags:
    - gce-u22-stage3
  variables:
    TEST_ARCH: arch-ci-linux-pkgs-dbg-ftn-interfaces
    OMP_PROC_BIND: "false"
    OMP_NUM_THREADS: 4
    PETSC_OPTIONS: -fp_trap

linux-pkgs-cxx-mlib:
  extends:
    - .stage-3
    - .linux_test
  tags:
    - gce-u22-stage3
  variables:
    TEST_ARCH: arch-ci-linux-pkgs-cxx-mlib
    PETSC_OPTIONS: -fp_trap

linux-pkgs-valgrind:
  extends:
    - .stage-3
    - .linux_test
  tags:
    - gce-valgrind
  variables:
    TEST_ARCH: arch-ci-linux-pkgs-valgrind
    TIMEOUT: 7200

linux-pkgs-opt:
  extends:
    - .stage-3
    - .linux_test
  tags:
    - gce-u22-stage3, linux-fast
  variables:
    TEST_ARCH: arch-ci-linux-pkgs-opt
    LOAD_MODULES: gcc/12.1.0
    MAKE_CXXFLAGS: -Wno-deprecated-declarations -Werror

linux-pkgs-dbg:
  extends:
    - .stage-3
    - .linux_test
  tags:
    - gce-u22-stage3, linux-gcov
  variables:
    TEST_ARCH: arch-ci-linux-pkgs-dbg
    PETSC_OPTIONS: -fp_trap

linux-matlab-ilp64:
  extends:
    - .stage-3
    - .linux_test
  tags:
    - gce-u22-stage3, linux-gcov
  variables:
    TEST_ARCH: arch-ci-linux-matlab-ilp64
    LOAD_MODULES: gcc/9.4.0 matlab/R2022a
  allow_failure: true

linux-pgi:
  extends:
    - .stage-3
    - .linux_test_noflags
    - .coverage-disable
  tags:
    - gce-u22-stage3, linux-mcpu
  variables:
    TEST_ARCH: arch-ci-linux-pgi
    LOAD_MODULES: nvhpc/22.3

# If the Nag license server fails then ${PETSC_DIR}/naglicenseproblem is created and the job is marked as failed but allow_failure
linux-nagfor:
  extends:
    - .stage-3
    - .linux_test_nofflags
  tags:
    - gce-stage2, linux-mcpu
  variables:
    LOAD_MODULES: nag/6.1
    TEST_ARCH: arch-ci-linux-nagfor
  allow_failure:
    exit_codes: 126

linux-intel-cmplx:
  extends:
    - .stage-3
    - .linux_test_nofflags
    - .coverage-disable
  tags:
    - gce-u22-stage3
  variables:
    TEST_ARCH: arch-ci-linux-intel-cmplx
    LOAD_MODULES: intel-oneapi-compilers/2023.0.0 intel-oneapi-mkl/2022.0.2

linux-xsdk-dbg:
  extends:
    - .stage-3
    - .linux_test
  tags:
    - gce-u22-stage3
  variables:
    TEST_ARCH: arch-ci-linux-xsdk-dbg
    LOAD_MODULES: gcc/12.1.0

check-each-commit:
  extends:
    - .stage-2
    - .linux_test
    - .coverage-disable
  tags:
    - gce-u22-stage2
  script:
    - export PATH=/nfs/gce/projects/petsc/soft/u22.04/spack-2024-03-06/opt/spack/linux-ubuntu22.04-x86_64/gcc-11.4.0/llvm-18.1.0-7hsf4hzma6aebqsd4ifb5brchcooktkp/bin:$PATH
    - lib/petsc/bin/maint/check-each-commit.sh
  except:
    variables:
      - $PETSC_CI_SCHEDULED =~ /yes/
  allow_failure: true

linux-analyzer:
  extends:
    - .stage-2
    - .linux_test
    - .coverage-disable
  tags:
    - gce-u22-stage2
  script:
    - printf "PATH:$PATH\n"
    - printf "PYTHONPATH:$PYTHONPATH\n"
    - python3 --version
    - git --version
    - mypy --version
    - vermin --version
    - printf "PETSC_OPTIONS:$PETSC_OPTIONS\n"
    - if [ ! -z ${CI_MERGE_REQUEST_SOURCE_BRANCH_NAME+x} ]; then printf "CI_MERGE_REQUEST_SOURCE_BRANCH_NAME:$CI_MERGE_REQUEST_SOURCE_BRANCH_NAME\n" ;fi
    - printf "./config/examples/${TEST_ARCH}.py\n"
    - cat ./config/examples/${TEST_ARCH}.py
    - ./config/examples/${TEST_ARCH}.py
    - make LINTER_OPTIONS="${LINTER_OPTIONS}" test-lint
    - make V=3 LINTER_OPTIONS="${LINTER_OPTIONS} -- ./src" lint
  variables:
    TEST_ARCH: arch-ci-linux-analyzer
    LINTER_OPTIONS: "--PETSC_ARCH=${TEST_ARCH} --clang_lib=`llvm-config-14 --libdir`/libclang.so.1 --werror 1"
  artifacts:
    paths:
    - arch-*/lib/petsc/conf/*.log
    - ./petscLintPatches/*.patch
    expire_in: 4 days
  except:
    variables:
      - $PETSC_CI_SCHEDULED =~ /yes/

linux-intel:
  extends:
    - .stage-3
    - .linux_test_nofflags
    - .coverage-disable
  tags:
    - gce-u22-stage3
  variables:
    TEST_ARCH: arch-ci-linux-intel
    LOAD_MODULES: intel-oneapi-compilers/2023.0.0 intel-oneapi-mkl/2022.0.2

linux-clang-ubsan:
  extends:
    - .stage-2
    - .linux_test
    - .coverage-disable
  tags:
    - gce-u22-stage2
  variables:
    TEST_ARCH: arch-ci-linux-clang-ubsan
    LOAD_MODULES: llvm/17.0.4

linux-opt-arm:
  extends:
    - .stage-3
    - .linux_test
    # gcovr does not work on arm/linux
    - .coverage-disable
  tags:
    - arch:arm, os:linux
  variables:
    TEST_ARCH: arch-ci-linux-opt-arm

linux-pkgs-64idx:
  extends:
    - .stage-3
    - .linux_test
  tags:
    - gce-u22-stage3, linux-mcpu
  variables:
    TEST_ARCH: arch-ci-linux-pkgs-64idx
    LOAD_MODULES: intel-oneapi-mkl/2022.0.2

linux-64idx-i8:
  extends:
    - .stage-3
    - .linux_test
  tags:
    - gce-u22-stage3, linux-mcpu
  variables:
    TEST_ARCH: arch-ci-linux-64idx-i8

linux-gcc-ifc-cmplx:
  extends:
    - .stage-3
    - .linux_test_nofflags
  tags:
    - gce-u22-stage3
  variables:
    TEST_ARCH: arch-ci-linux-gcc-ifc-cmplx
    LOAD_MODULES: intel-oneapi-compilers/2022.2.1

linux-opt-cxx-quad:
  extends:
    - .stage-3
    - .linux_test
  tags:
    - gce-u22-stage3
  variables:
    TEST_ARCH: arch-ci-linux-opt-cxx-quad

linux-ILP64:
  extends:
    - .stage-3
    - .linux_test
  tags:
    - gce-u22-stage3
  variables:
    TEST_ARCH: arch-ci-linux-ILP64
    LOAD_MODULES: gcc/8.5.0 mpich/2-1.5-gcc-8.5.0
    MAKE_CFLAGS: -Wno-discarded-qualifiers -Werror

linux-64idx-i8-uni:
  extends:
    - .stage-3
    - .linux_test
  tags:
    - gce-u22-stage3, linux-mcpu
  variables:
    TEST_ARCH: arch-ci-linux-64idx-i8-uni

linux-misc-32bit:
  extends:
    - .stage-3
    - .linux_test
  tags:
    - gce-u22-stage3
  variables:
    TEST_ARCH: arch-ci-linux-misc-32bit

mswin-intel-cxx-cmplx:
  extends:
    - .stage-3
    - .mswin_test
  tags:
    - mswin-stage3
  variables:
    TEST_ARCH: arch-ci-mswin-intel-cxx-cmplx
    TIMEOUT: 150

mswin-uni:
  extends:
    - .stage-3
    - .mswin_test
  tags:
    - mswin-stage3
  variables:
    PETSC_OPTIONS: -check_pointer_intensity 0 -error_output_stdout
    TEST_ARCH: arch-ci-mswin-uni

mswin-gnu:
  extends:
    - .stage-3
    - .mswin_test
  tags:
    - mswin-stage3
  variables:
    TEST_ARCH: arch-ci-mswin-gnu
    MAKE_CFLAGS: -Werror
    MAKE_CXXFLAGS: -Werror
    MAKE_FFLAGS: -Werror
    DISABLE_TESTS: 1

#mswin-intel:
#  extends:
#    - .stage-3
#    - .mswin_test
#  tags:
#    - os:win
#  variables:
#    TEST_ARCH: arch-ci-mswin-intel

osx-cxx-cmplx-pkgs-dbg:
  extends:
    - .stage-3
    - .osx_test
    - .coverage-disable
  tags:
    - os:macos-x64
  variables:
    TEST_ARCH: arch-ci-osx-cxx-cmplx-pkgs-dbg
    INIT_SCRIPT: .zprofile

osx-cxx-pkgs-opt-arm:
  extends:
    - .stage-3
    - .osx_test
    - .coverage-disable
  tags:
    - os:macos-arm
  variables:
    PETSC_OPTIONS: -vecscatter_mpi1 false -options_left false
    TEST_ARCH: arch-ci-osx-cxx-pkgs-opt-arm
    MAKE_CXXFLAGS: -Werror -Wno-pass-failed
    INIT_SCRIPT: .zprofile

osx-dbg:
  extends:
    - .stage-3
    - .osx_test
    - .coverage-disable
  tags:
    - os:macos-x64
  variables:
    TEST_ARCH: arch-ci-osx-dbg
    ASAN_OPTIONS: log_path=stdout
    INIT_SCRIPT: .zprofile

osx-xsdk-opt:
  extends:
    - .stage-3
    - .osx_test
    - .coverage-disable
  tags:
    - os:macos-x64
  variables:
    TEST_ARCH: arch-ci-osx-xsdk-opt
    INIT_SCRIPT: .zprofile

# job for analyzing the coverage results and generating the various reports
coverage-analyze:
  extends: .test
  stage: stage-4
  tags:
    - gce-u22-short
  dependencies:
    # stage-2
    - linux-gcc-quad-64idx-dbg
    - linux-gcc-pkgs-opt
    - linux-gcc-complex-opt-32bit
    # stage-3
    - linux-pkgs-dbg
    - linux-matlab-ilp64
    - linux-cuda-pkgs
    - linux-cxx-cmplx-pkgs-64idx
    - linux-cuda-cmplx
    - linux-cuda-double-64idx
    - linux-cuda-uni-pkgs
    - linux-viennacl
    - linux-without-fc
    - linux-gcc-cxx-avx2
    - linux-clang-avx
    - linux-pkgs-cxx-mlib
    - linux-pkgs-valgrind
    - linux-nagfor
    - linux-xsdk-dbg
    - linux-pkgs-64idx
    - linux-pkgs-opt
    - linux-64idx-i8
    - linux-gcc-ifc-cmplx
    - linux-opt-cxx-quad
    - linux-ILP64
    - linux-64idx-i8-uni
  variables:
    PYTHON: python3
    PETSC_ARCH: arch-ci-analyze-pipeline
  before_script:
    - date
    - hostname
  script:
    - if [ ! -z ${LOAD_MODULES+x} ]; then module --trace load ${LOAD_MODULES}; module list; fi
    - ${PYTHON} --version
    - gcovr --version
    - printf "PATH:$PATH\n"
    - printf "PYTHONPATH:$PYTHONPATH\n"
    - if [ ! -z ${CI_MERGE_REQUEST_SOURCE_BRANCH_NAME+x} ]; then printf "CI_MERGE_REQUEST_SOURCE_BRANCH_NAME:$CI_MERGE_REQUEST_SOURCE_BRANCH_NAME\n" ;fi
    - ${PYTHON} ./configure --with-mpi=0 --with-cxx=0 --download-sowing --with-fortran-bindings
    - make mergegcov PETSC_GCOV_OPTIONS='--ci-mode --verbose'
  coverage: /^\s*lines:\s*\d+.\d+\%/
  artifacts:
    name: "$CI_JOB_NAME"
    when: always
    paths:
    - arch-ci-analyze-pipeline/gcovr/*
    - arch-ci-analyze-pipeline/lib/petsc/conf/*.log
    expire_in: 4 days
    reports:
      coverage_report:
        coverage_format: cobertura
        path: arch-ci-analyze-pipeline/gcovr/xml/*.xml

# template for the coverage review jobs, these must be separate jobs as you cannot deploy multiple
# environments from one job...
.coverage-review:
  extends: .test
  stage: .post
  tags:
    - gce-u22-short
  dependencies:
    - coverage-analyze
  script:
    - date
    - hostname
  artifacts:
    name: "$CI_JOB_NAME"
    when: always
    paths:
    - arch-ci-analyze-pipeline/gcovr/*
    - arch-ci-analyze-pipeline/lib/petsc/conf/*.log
    expire_in: 4 days

coverage-total-review:
  extends: .coverage-review
  environment:
    name: coverage/all/$CI_COMMIT_REF_SLUG
    url: https://$CI_PROJECT_NAMESPACE.gitlab.io/-/$CI_PROJECT_NAME/-/jobs/$CI_JOB_ID/artifacts/arch-ci-analyze-pipeline/gcovr/html/report.html
    auto_stop_in: one week
    deployment_tier: testing

coverage-untested-review:
  extends: .coverage-review
  environment:
    name: coverage/untested/$CI_COMMIT_REF_SLUG
    url: https://$CI_PROJECT_NAMESPACE.gitlab.io/-/$CI_PROJECT_NAME/-/jobs/$CI_JOB_ID/artifacts/arch-ci-analyze-pipeline/gcovr/html_untested/report_untested.html
    auto_stop_in: one week
    deployment_tier: testing

analyze-pipeline:
  extends: .coverage-review
  script:
    - date
    - hostname
    # This file is created by lib/petsc/bin/maint/gcov.py. If it exists then the CI
    # pipeline should fail
    - if [ -f ./arch-ci-analyze-pipeline/gcovr/.CI_FAIL ]; then
        cat ./arch-ci-analyze-pipeline/gcovr/merge_gcov_errors.log;
        exit 1;
      fi
  allow_failure: true

#
# Base job for a documentation build (runs in venv to allow installing additional packages without root)
#
.docs:
  stage: stage-2
  tags:
    - linux-docs
  before_script:
    - git fetch --unshallow --tags origin +release:remotes/origin/release +main:remotes/origin/main
    - module --trace load python-3.11.2-gcc-9.4.0-6p2ktlj
    - printf "PATH:$PATH\n"
    - printf "PYTHONPATH:$PYTHONPATH\n"
    - VENV=venv-petsc-docs &&
      python3 -m venv $VENV &&
      . $VENV/bin/activate &&
      cd doc &&
      python -m pip install -r requirements.txt

#
# Build documentation and make available for review using GitLab pages
#
docs-review:
  extends:
    - .docs
    - .test-basic
  script:
    - (time make html BUILDDIR="../public" SPHINXOPTS="-T -E -W --keep-going -j 2") || (printf "Sphinx build failed (warnings count as errors)\n" && false)
    - (time make latexpdf SPHINXOPTS="-T -E -W --keep-going -j 2") || (printf "Sphinx build failed (warnings count as errors)\n" && false)
    - cp _build/latex/manual.pdf ../public/html/manual/
  environment:
    name: review/$CI_COMMIT_REF_NAME
    url: https://$CI_PROJECT_NAMESPACE.gitlab.io/-/$CI_PROJECT_NAME/-/jobs/$CI_JOB_ID/artifacts/public/html/index.html
    auto_stop_in: one week
    deployment_tier: development
  artifacts:
    paths:
      - public
    expire_in: 4 days
  except:
    variables:
      - $PETSC_CI_SCHEDULED =~ /yes/

#
# Deploy documentation using GitLab pages
#
pages: # this job name has special meaning to GitLab
  extends: .docs
  interruptible: true
  script:
    - mkdir -p ../public/ && cp public/* ../public/
    - (git checkout origin/main && pip install -r requirements.txt && make website-deploy SPHINXOPTS="-j 1" BUILDDIR="../public/main" && make latexpdf && mkdir -p ../public/main/manual && cp _build/latex/manual.pdf ../public/main/manual/) || (printf "Sphinx build failed (warnings count as errors)\n" && false)
    - cd .. && git clean -fdxq -e public -e venv-petsc-docs && cd doc && rm -rf images
    - (git checkout origin/release && pip install -r requirements.txt && make website-deploy SPHINXOPTS="-j 1" BUILDDIR="../public/release" && make latexpdf && mkdir -p ../public/release/manual && cp _build/latex/manual.pdf ../public/release/manual/) || (printf "Sphinx build failed (warnings count as errors)\n" && false)
    - rm -rf ../public/*/.doctrees
  only:
    variables:
      - $PETSC_CI_SCHEDULED == "yes"
  artifacts:
    paths:
      - public
    expire_in: 4 days

#
#
# The following tests are experimental; more tests by users at other sites may be added below this. Experimental test
# that fail produce a warning, but do not block execution of a pipeline.
#

.test-experimental:
  extends: .test
  allow_failure: true